diff --git a/doc/coro-performance-baseline.md b/doc/coro-performance-baseline.md index 990ceaa9f7..6ee94bb7ff 100644 --- a/doc/coro-performance-baseline.md +++ b/doc/coro-performance-baseline.md @@ -1963,3 +1963,61 @@ not on the file/syscall path. Small standard-file, direct-syscall, sole-M blocking-pipe/timer, and loopback-TCP execution gates pass with `GOMAXPROCS=1`, as do separately linked same-M callback, locked-thread compensation, deferred replacement, and worker-capability E2Es. + +### Demand-lazy replacement and compiler-task route checkpoint + +The 2026-08-15 follow-up starts at merge +`95acde8e415516d07f2caea74578d91125a0997f`; the measured implementation is +`13bdf13c4`. Profiling the scalar-ABI binary showed that a quick native syscall +still reserved and initialized a replacement directory slot before entering +the kernel, even though almost every call returned before another task or event +needed a physical owner. The same compiler-owned call window also recovered the +current driver repeatedly through its complete registry, source-catalog, +poll-transaction, and owner-local lifecycle validator. + +The deferred handoff gate now has two demand-lazy phases. `Armed` and +`Starting` contain no replacement slot. A durable executor request must first +win the stable parent's `Armed -> Starting` CAS; only that unique publisher +reads the released execution-domain generation, allocates a directory slot, +and publishes `Queued` or `Started` with the slot. A quick returning syscall +wins `Withdraw` without scanning, initializing, or recycling replacement +storage. The request side retains no pointer to the caller's stack, G, P, +driver, LLVM handle, or coroutine frame. The post-Arm durable-demand recheck +and request-tail activation still close the request-before-Arm race. + +Generated physical coroutine code already carries the exact hidden task. The +native boundary now consumes `CurrentExecutorDriverForCompilerTask`, freezes +its route for the no-suspend detach interval, and uses route-authenticated +native-owner and managed-quota operations. The detach transition still checks +the mutable task, frame, park, action, lock, preemption, and P fields it changes. +TLS/reentry, retained, and otherwise non-compiler callers retain the original +complete validator; no annotation or function-address reverse lookup was +introduced. + +The parent and candidate were built from the same `io_workload` source with +independent caches, full LTO, stripped output, Go 1.26.5, LLVM 22.1.8, and +process-start `GOMAXPROCS=1` on Darwin arm64. Fifteen three-way rotated process +runs gave: + +| Workload | scalar parent median [range] | candidate median [range] | Go median [range] | candidate delta / Go | +| --- | ---: | ---: | ---: | ---: | +| cache-hot 4 KiB standard file round trip, 5,000 operations | 42.343 ms [41.093, 44.261] | 38.376 ms [37.189, 39.208] | 7.765 ms [7.644, 7.946] | -9.37%; 4.94x | +| direct `syscall` file round trip, 5,000 operations | 18.485 ms [18.149, 19.471] | 15.826 ms [15.384, 16.718] | 7.658 ms [7.467, 7.891] | -14.39%; 2.07x | +| loopback TCP echo, 500 operations | 19.980 ms [17.879, 21.272] | 19.343 ms [18.015, 21.213] | 9.486 ms [8.745, 10.183] | -3.19%; 2.04x | + +The TCP ranges overlap, so the table establishes no network speedup beyond a +no-regression observation. In a three-second direct-syscall sample, the kernel +`write` leaf accounted for about 77% of samples below `syscall.Write`, versus +about 62% before the route-capability change; complete source-set validation no +longer appears in the ordinary begin/release/reenter path. The stripped binary +is 7,016,288 bytes versus 7,014,768 (+1,520, 0.022%), and Mach-O `__text` is +3,140,944 bytes versus 3,137,244 (+3,700, 0.118%). + +The full runtime module, architecture debt gate, native target-plan gate, +same-M scheduler-progress E2E, locked compensation E2E, request-driven direct +channel replacement E2E, and real standard-file/direct-syscall/sole-M blocking +pipe with timer/loopback-TCP executions pass. Sampling now identifies the +remaining standard-file gap primarily in synchronous child coroutine +allocation, frame publication, inline-await completion, and destroy/consume +transactions through `syscall`, `internal/poll`, `os.File`, and `io`, rather +than in replacement-slot or full driver-route recovery. diff --git a/internal/build/coro_native_fleet_e2e_test.go b/internal/build/coro_native_fleet_e2e_test.go index 4a65814436..752c6f3f72 100644 --- a/internal/build/coro_native_fleet_e2e_test.go +++ b/internal/build/coro_native_fleet_e2e_test.go @@ -1930,7 +1930,7 @@ func TestCoroNativeFleetLockedForeignReleasesQuotaBeforeReplacementStarts(t *tes source := string(raw) release := strings.Index( source, - "if releaseManaged && !coroTargetReleaseManagedExecutionV1(boundary.driver)", + "if releaseManaged && !coroTargetReleaseManagedExecutionAtRouteV1(", ) request := strings.Index( source, diff --git a/runtime/internal/coro/deferred_executor_handoff.go b/runtime/internal/coro/deferred_executor_handoff.go index 6932156e3c..0d4940fd5e 100644 --- a/runtime/internal/coro/deferred_executor_handoff.go +++ b/runtime/internal/coro/deferred_executor_handoff.go @@ -42,25 +42,28 @@ func (*deferredExecutorHandoffNoCopy) Lock() {} func (*deferredExecutorHandoffNoCopy) Unlock() {} // DeferredExecutorHandoff is a pointer-free, stable-address dispatch gate for -// a replacement which has already obtained an exact ExecutionDomainHandoff -// and directory slot but has not yet consumed a physical thread. Arm happens -// only after the blocked owner releases its managed-execution permit. A -// durable target request and the returning owner then race one CAS: +// a replacement whose exact ExecutionDomainHandoff has been published but +// whose directory slot and physical thread are both still demand-lazy. Arm +// happens only after the blocked owner releases its managed-execution permit. +// A durable target request and the returning owner then race one CAS: // // - BeginStart wins and must publish Queued or Started; // - Withdraw wins and proves that no physical owner was dispatched. // -// Starting is a short publication interval, not a scheduler wait state. The -// returning owner may yield until the request publisher records the outcome. -// Queued distinguishes an asynchronously dispatched cached thread, which can -// still be canceled before C-to-Go dispatch, from a synchronously acknowledged -// start. Complete is called only after the ordinary generation-bound return and +// Armed and Starting carry no directory slot. The unique Starting publisher +// allocates one and includes it only when publishing Queued or Started. Starting +// is a short publication interval, not a scheduler wait state; the returning +// owner may yield until the request publisher records the outcome. Queued +// distinguishes an asynchronously dispatched cached thread, which can still be +// canceled before C-to-Go dispatch, from a synchronously acknowledged start. +// Complete is called only after the ordinary generation-bound return and // strong-recycle protocol has finished. // // Slot is deliberately a routing hint rather than a generation capability. A -// delayed accepted request may start a later armed use of the same slot; that -// is a safe coalesced compensation request. ExecutionDomainHandoff remains the -// authority which prevents a stale physical owner from claiming a later call. +// delayed accepted request may start a later armed use of the same stable +// parent; that is a safe coalesced compensation request. ExecutionDomainHandoff +// remains the authority which prevents a stale physical owner from claiming a +// later call. type DeferredExecutorHandoff struct { noCopy deferredExecutorHandoffNoCopy state uint32 @@ -76,56 +79,61 @@ func deferredExecutorHandoffUnpack(state uint32) (uint32, DeferredExecutorHandof } func deferredExecutorHandoffValid(slot uint32, phase DeferredExecutorHandoffPhase) bool { - if phase == DeferredExecutorHandoffIdle { + switch phase { + case DeferredExecutorHandoffIdle, + DeferredExecutorHandoffArmed, + DeferredExecutorHandoffStarting: return slot == 0 + case DeferredExecutorHandoffQueued, + DeferredExecutorHandoffStarted: + return slot != 0 && slot <= deferredExecutorHandoffSlotMask + default: + return false } - return slot != 0 && slot <= deferredExecutorHandoffSlotMask && - phase <= DeferredExecutorHandoffStarted } -// Arm publishes one prepared replacement after its managed-execution permit -// has been released. The zero value is reusable Idle. -func (handoff *DeferredExecutorHandoff) Arm(slot uint32) bool { - return handoff != nil && deferredExecutorHandoffValid(slot, DeferredExecutorHandoffArmed) && - preemptCompareAndSwap( - &handoff.state, - 0, - deferredExecutorHandoffPack(slot, DeferredExecutorHandoffArmed), - ) +// Arm publishes one demand-lazy replacement after its managed-execution +// permit has been released. The zero value is reusable Idle. +func (handoff *DeferredExecutorHandoff) Arm() bool { + return handoff != nil && preemptCompareAndSwap( + &handoff.state, + 0, + deferredExecutorHandoffPack(0, DeferredExecutorHandoffArmed), + ) } // BeginStart lets one accepted durable executor request become the unique // physical-start publisher. A false result means there is no armed replacement // to start; Idle and an already-starting/started request are both benign. -func (handoff *DeferredExecutorHandoff) BeginStart() (slot uint32, started bool) { +func (handoff *DeferredExecutorHandoff) BeginStart() bool { if handoff == nil { - return 0, false + return false } state := preemptLoad(&handoff.state) slot, phase := deferredExecutorHandoffUnpack(state) if !deferredExecutorHandoffValid(slot, phase) || phase != DeferredExecutorHandoffArmed { - return 0, false + return false } - return slot, preemptCompareAndSwap( + return preemptCompareAndSwap( &handoff.state, state, - deferredExecutorHandoffPack(slot, DeferredExecutorHandoffStarting), + deferredExecutorHandoffPack(0, DeferredExecutorHandoffStarting), ) } // PublishStart completes the unique Starting interval. queued records whether // the cached-thread dispatch can still be withdrawn through its C token. func (handoff *DeferredExecutorHandoff) PublishStart(slot uint32, queued bool) bool { - if handoff == nil || !deferredExecutorHandoffValid(slot, DeferredExecutorHandoffStarting) { - return false - } phase := DeferredExecutorHandoffStarted if queued { phase = DeferredExecutorHandoffQueued } + if handoff == nil || !deferredExecutorHandoffValid(slot, phase) { + return false + } return preemptCompareAndSwap( &handoff.state, - deferredExecutorHandoffPack(slot, DeferredExecutorHandoffStarting), + deferredExecutorHandoffPack(0, DeferredExecutorHandoffStarting), deferredExecutorHandoffPack(slot, phase), ) } @@ -133,23 +141,21 @@ func (handoff *DeferredExecutorHandoff) PublishStart(slot uint32, queued bool) b // RetryStart returns a failed physical-start publication to Armed. The durable // request caller must report failure; a later request may retry, while a // concurrently returning owner may withdraw the restored arm. -func (handoff *DeferredExecutorHandoff) RetryStart(slot uint32) bool { - return handoff != nil && deferredExecutorHandoffValid(slot, DeferredExecutorHandoffStarting) && - preemptCompareAndSwap( - &handoff.state, - deferredExecutorHandoffPack(slot, DeferredExecutorHandoffStarting), - deferredExecutorHandoffPack(slot, DeferredExecutorHandoffArmed), - ) +func (handoff *DeferredExecutorHandoff) RetryStart() bool { + return handoff != nil && preemptCompareAndSwap( + &handoff.state, + deferredExecutorHandoffPack(0, DeferredExecutorHandoffStarting), + deferredExecutorHandoffPack(0, DeferredExecutorHandoffArmed), + ) } // Withdraw wins only before a durable request has begun physical dispatch. -func (handoff *DeferredExecutorHandoff) Withdraw(slot uint32) bool { - return handoff != nil && deferredExecutorHandoffValid(slot, DeferredExecutorHandoffArmed) && - preemptCompareAndSwap( - &handoff.state, - deferredExecutorHandoffPack(slot, DeferredExecutorHandoffArmed), - 0, - ) +func (handoff *DeferredExecutorHandoff) Withdraw() bool { + return handoff != nil && preemptCompareAndSwap( + &handoff.state, + deferredExecutorHandoffPack(0, DeferredExecutorHandoffArmed), + 0, + ) } // Observe returns one atomic state snapshot. ok rejects an impossible packed diff --git a/runtime/internal/coro/deferred_executor_handoff_test.go b/runtime/internal/coro/deferred_executor_handoff_test.go index 48fee0840f..8008342cfb 100644 --- a/runtime/internal/coro/deferred_executor_handoff_test.go +++ b/runtime/internal/coro/deferred_executor_handoff_test.go @@ -23,13 +23,13 @@ import ( func TestDeferredExecutorHandoffWithdraw(t *testing.T) { var handoff DeferredExecutorHandoff - if !handoff.Idle() || !handoff.Arm(17) || handoff.Arm(18) { + if !handoff.Idle() || !handoff.Arm() || handoff.Arm() { t.Fatal("arm deferred executor handoff") } - if slot, phase, ok := handoff.Observe(); !ok || slot != 17 || phase != DeferredExecutorHandoffArmed { + if slot, phase, ok := handoff.Observe(); !ok || slot != 0 || phase != DeferredExecutorHandoffArmed { t.Fatalf("armed snapshot = (%d, %d, %t)", slot, phase, ok) } - if handoff.Withdraw(18) || !handoff.Withdraw(17) || !handoff.Idle() { + if !handoff.Withdraw() || handoff.Withdraw() || !handoff.Idle() { t.Fatal("withdraw deferred executor handoff") } } @@ -37,11 +37,11 @@ func TestDeferredExecutorHandoffWithdraw(t *testing.T) { func TestDeferredExecutorHandoffStartOutcomes(t *testing.T) { for _, queued := range []bool{false, true} { var handoff DeferredExecutorHandoff - if !handoff.Arm(23) { + if !handoff.Arm() { t.Fatal("arm deferred executor handoff") } - slot, begun := handoff.BeginStart() - if !begun || slot != 23 || handoff.Withdraw(23) || + const slot = uint32(23) + if !handoff.BeginStart() || handoff.Withdraw() || !handoff.PublishStart(slot, queued) { t.Fatalf("publish deferred start queued=%t", queued) } @@ -60,14 +60,13 @@ func TestDeferredExecutorHandoffStartOutcomes(t *testing.T) { func TestDeferredExecutorHandoffRetry(t *testing.T) { var handoff DeferredExecutorHandoff - if !handoff.Arm(29) { + if !handoff.Arm() { t.Fatal("arm deferred executor handoff") } - slot, begun := handoff.BeginStart() - if !begun || slot != 29 || !handoff.RetryStart(slot) { + if !handoff.BeginStart() || !handoff.RetryStart() { t.Fatal("retry deferred executor start") } - if !handoff.Withdraw(slot) || !handoff.Idle() { + if !handoff.Withdraw() || !handoff.Idle() { t.Fatal("withdraw retried deferred executor handoff") } } @@ -76,7 +75,7 @@ func TestDeferredExecutorHandoffStartWithdrawRace(t *testing.T) { const iterations = 2_000 for iteration := 0; iteration < iterations; iteration++ { var handoff DeferredExecutorHandoff - if !handoff.Arm(31) { + if !handoff.Arm() { t.Fatal("arm deferred executor handoff") } var wait sync.WaitGroup @@ -85,15 +84,15 @@ func TestDeferredExecutorHandoffStartWithdrawRace(t *testing.T) { withdrawn := make(chan bool, 1) go func() { defer wait.Done() - slot, ok := handoff.BeginStart() - if ok && !handoff.PublishStart(slot, true) { + ok := handoff.BeginStart() + if ok && !handoff.PublishStart(31, true) { t.Errorf("publish winning start at iteration %d", iteration) } started <- ok }() go func() { defer wait.Done() - withdrawn <- handoff.Withdraw(31) + withdrawn <- handoff.Withdraw() }() wait.Wait() startWon, withdrawWon := <-started, <-withdrawn @@ -113,10 +112,14 @@ func TestDeferredExecutorHandoffStartWithdrawRace(t *testing.T) { func TestDeferredExecutorHandoffRejectsInvalidSlots(t *testing.T) { var handoff DeferredExecutorHandoff - if handoff.Arm(0) || handoff.Arm(deferredExecutorHandoffSlotMask+1) { - t.Fatal("accepted invalid deferred executor slot") - } - if _, begun := handoff.BeginStart(); begun { + if handoff.BeginStart() { t.Fatal("started idle deferred executor handoff") } + if !handoff.Arm() || !handoff.BeginStart() { + t.Fatal("cannot begin deferred executor handoff") + } + if handoff.PublishStart(0, false) || + handoff.PublishStart(deferredExecutorHandoffSlotMask+1, false) { + t.Fatal("accepted invalid deferred executor slot") + } } diff --git a/runtime/internal/coro/executor_resume_handoff.go b/runtime/internal/coro/executor_resume_handoff.go index 4423ec60e6..41002ee822 100644 --- a/runtime/internal/coro/executor_resume_handoff.go +++ b/runtime/internal/coro/executor_resume_handoff.go @@ -106,28 +106,23 @@ func validForeignWaitingExecutorTask(p *P, task *G) bool { active.header.Lifecycle == uint16(FrameActive) } -// DetachExecutorResume removes one exact issued ActionResume from its P and -// driver while leaving the LLVM frame active on the calling locked M. The -// caller must publish an ExecutionDomainHandoff only after this succeeds. +// detachExecutorResumeCertified removes one owner-certified issued +// ActionResume from its P and driver while leaving the LLVM frame active on +// the calling M. The caller must publish an ExecutionDomainHandoff only after +// this succeeds. // // No coroutine transition or scheduler action is executed here. The task // remains rooted by handoff, keeps runP as its cancellation ownership domain, // and enters GForeignWaiting. Clearing P.osThreadLockOwner lets a replacement // M run unrelated work on the same P without weakening the original G-to-M // affinity. -func DetachExecutorResume( +func detachExecutorResumeCertified( handoff *ExecutorResumeHandoff, driver *ExecutorDriver, task *G, mode ExecutorResumeHandoffMode, ) bool { - if !emptyExecutorResumeHandoff(handoff) || driver == nil || task == nil || - (mode != ExecutorResumeHandoffLockedForeign && - mode != ExecutorResumeHandoffSameMForeign) { - return false - } - current, _, _, ownerOK := CurrentExecutorDriver(task) - if !ownerOK || current != driver || !enterCriticalContext(task) || + if !enterCriticalContext(task) || driver.run.issued != ActionCheckResume { return false } @@ -176,6 +171,55 @@ func DetachExecutorResume( return true } +func validExecutorResumeHandoffDetachRequest( + handoff *ExecutorResumeHandoff, + driver *ExecutorDriver, + task *G, + mode ExecutorResumeHandoffMode, +) bool { + return emptyExecutorResumeHandoff(handoff) && driver != nil && task != nil && + (mode == ExecutorResumeHandoffLockedForeign || + mode == ExecutorResumeHandoffSameMForeign) +} + +// DetachExecutorResume removes one exact issued ActionResume after performing +// the complete retained/native owner audit. Targets which do not hold the +// compiler's hidden task capability must use this entry. +func DetachExecutorResume( + handoff *ExecutorResumeHandoff, + driver *ExecutorDriver, + task *G, + mode ExecutorResumeHandoffMode, +) bool { + if !validExecutorResumeHandoffDetachRequest(handoff, driver, task, mode) { + return false + } + current, _, _, ownerOK := CurrentExecutorDriver(task) + return ownerOK && current == driver && + detachExecutorResumeCertified(handoff, driver, task, mode) +} + +// DetachExecutorResumeForCompilerTask consumes the hidden task capability +// carried by generated physical coroutine code. CurrentExecutorDriverForCompilerTask +// has already frozen the exact no-suspend G/P/driver relation, so this path does +// not repeat the registry, complete source-catalog, or owner-local queue audits +// required when a target recovers a task through TLS or retained native state. +// The concrete detach transition still validates every mutable task, action, +// lock, park and preemption field which it changes. +func DetachExecutorResumeForCompilerTask( + handoff *ExecutorResumeHandoff, + driver *ExecutorDriver, + task *G, + mode ExecutorResumeHandoffMode, +) bool { + if !validExecutorResumeHandoffDetachRequest(handoff, driver, task, mode) { + return false + } + current, _, ownerOK := CurrentExecutorDriverForCompilerTask(task) + return ownerOK && current == driver && + detachExecutorResumeCertified(handoff, driver, task, mode) +} + // ExecutorResumeHandoffReturnable reports the necessary target-neutral // physical-owner boundary. A replacement owner may finish its // ExecutionDomainHandoff return only here: no physical action or source diff --git a/runtime/internal/coro/executor_resume_handoff_test.go b/runtime/internal/coro/executor_resume_handoff_test.go index 7e0de95c4a..881dafad44 100644 --- a/runtime/internal/coro/executor_resume_handoff_test.go +++ b/runtime/internal/coro/executor_resume_handoff_test.go @@ -110,6 +110,65 @@ func (fixture *executorResumeHandoffFixture) restore(t *testing.T) { } } +func TestCompilerTaskExecutorResumeHandoffUsesHiddenOwnerCapability(t *testing.T) { + p := new(P) + driver, _, _ := bindTestExecutorDriver(t, p) + task := newYieldingTestG(t, "compiler-foreign-wait-owner") + if !Enqueue(p, task.g) { + t.Fatal("enqueue compiler foreign-wait owner") + } + step := runnerNextPhysicalAction(t, driver, task, ActionCheckResume) + resume, ok := Checked(p, task.g, step.Action, false) + if !ok || resume.Kind != ActionResume || resume.Handle != task.handle { + t.Fatalf("check compiler foreign-wait owner = (%+v, %t)", resume, ok) + } + takeNormalRunnerDecision(t, task.g) + task.frame.header.SuspendReason = uint16(SuspendNone) + task.frame.header.Lifecycle = uint16(FrameActive) + + var handoff ExecutorResumeHandoff + savedCurrent := p.current + p.current = nil + if DetachExecutorResumeForCompilerTask( + &handoff, + driver, + task.g, + ExecutorResumeHandoffSameMForeign, + ) { + t.Fatal("compiler detach accepted a hidden task outside its physical resume") + } + p.current = savedCurrent + if !emptyExecutorResumeHandoff(&handoff) || + !DetachExecutorResumeForCompilerTask( + &handoff, + driver, + task.g, + ExecutorResumeHandoffSameMForeign, + ) { + t.Fatal("detach compiler-task executor resume") + } + if !handoff.Detached() || task.g.state != GForeignWaiting || + p.current != nil || p.inResume || driver.run.issued != ActionInvalid { + t.Fatal("compiler-task executor resume did not detach exact owner state") + } + if !RestoreExecutorResume(&handoff) || p.current != task.g || + !p.inResume || driver.run.issued != ActionCheckResume { + t.Fatal("restore compiler-task executor resume") + } + + task.frame.header.SuspendReason = uint16(SuspendYield) + task.frame.header.Lifecycle = uint16(FrameSuspended) + if !PrepareYield(task.g, task.handle, task.frame.header) { + t.Fatal("prepare compiler foreign-wait owner yield") + } + next, resumed := Resumed(p, task.g, resume) + if !resumed || next.Kind != ActionYield || + !CommitExecutorRunAction(driver, task.g, next) { + t.Fatalf("finish compiler foreign-wait owner = (%+v, %t)", next, resumed) + } + runtime.KeepAlive(task.frame.memory) +} + func TestExecutorResumeHandoffRunsReplacementAndRestoresExactResume(t *testing.T) { p := new(P) driver, registry, executor := bindTestExecutorDriver(t, p) diff --git a/runtime/internal/runtime/coro_execution_quota_native_llgo.go b/runtime/internal/runtime/coro_execution_quota_native_llgo.go index 0180465980..e9b8b543a6 100644 --- a/runtime/internal/runtime/coro_execution_quota_native_llgo.go +++ b/runtime/internal/runtime/coro_execution_quota_native_llgo.go @@ -38,7 +38,22 @@ func coroNativeFleetExecutionDomainV1( return nil, 0, false } route, ok := driver.Route() - if !ok || uint32(route) > coroNativeFleetV1State.domainCount { + if !ok { + return nil, 0, false + } + return coroNativeFleetExecutionDomainAtRouteV1(driver, route) +} + +// coroNativeFleetExecutionDomainAtRouteV1 consumes a route frozen from the +// current compiler-task capability. It still authenticates the target domain, +// driver identity and fleet handle, but does not repeat the driver's complete +// source-catalog and owner-local lifecycle audit. +func coroNativeFleetExecutionDomainAtRouteV1( + driver *coro.ExecutorDriver, + route coro.RouteID, +) (*coroNativeFleetDomainV1, coro.RouteID, bool) { + if driver == nil || !route.Valid() || + uint32(route) > coroNativeFleetV1State.domainCount { return nil, 0, false } domain := &coroNativeFleetV1State.domains[uint32(route)-1] @@ -75,11 +90,37 @@ func coroTargetAcquireManagedExecutionV1(driver *coro.ExecutorDriver) (bool, boo return coroNativeFleetV1State.execution.TryAcquire(route) } +func coroTargetAcquireManagedExecutionAtRouteV1( + driver *coro.ExecutorDriver, + route coro.RouteID, +) (bool, bool) { + _, route, ok := coroNativeFleetExecutionDomainAtRouteV1(driver, route) + if !ok { + return false, false + } + return coroNativeFleetV1State.execution.TryAcquire(route) +} + func coroTargetReleaseManagedExecutionV1(driver *coro.ExecutorDriver) bool { _, route, ok := coroNativeFleetExecutionDomainV1(driver) if !ok { return false } + return coroTargetReleaseManagedExecutionRouteV1(route) +} + +func coroTargetReleaseManagedExecutionAtRouteV1( + driver *coro.ExecutorDriver, + route coro.RouteID, +) bool { + _, route, ok := coroNativeFleetExecutionDomainAtRouteV1(driver, route) + if !ok { + return false + } + return coroTargetReleaseManagedExecutionRouteV1(route) +} + +func coroTargetReleaseManagedExecutionRouteV1(route coro.RouteID) bool { wake, released := coroNativeFleetV1State.execution.Release(route) if !released || !wake { return released @@ -117,6 +158,18 @@ func coroTargetWaitManagedExecutionV1(driver *coro.ExecutorDriver) bool { return ok } +func coroTargetWaitManagedExecutionAtRouteV1( + driver *coro.ExecutorDriver, + route coro.RouteID, +) bool { + domain, _, ok := coroNativeFleetExecutionDomainAtRouteV1(driver, route) + if !ok { + return false + } + _, ok = domain.doorbell.WaitBounded(corodoorbell.PollFaultContainmentMilliseconds) + return ok +} + // coroTargetReenterManagedExecutionV1 restores an outer bounded run slice's // exact P lease after its replacement M has returned and been strongly joined. // The caller restores the detached active resume only after this succeeds. @@ -135,6 +188,24 @@ func coroTargetReenterManagedExecutionV1(driver *coro.ExecutorDriver) bool { } } +func coroTargetReenterManagedExecutionAtRouteV1( + driver *coro.ExecutorDriver, + route coro.RouteID, +) bool { + for { + acquired, ok := coroTargetAcquireManagedExecutionAtRouteV1(driver, route) + if !ok { + return false + } + if acquired { + return true + } + if !coroTargetWaitManagedExecutionAtRouteV1(driver, route) { + return false + } + } +} + // CoroGOMAXPROCS implements the standard runtime.GOMAXPROCS query/set contract // over the logical managed-execution quota. Values above the current bounded // physical fleet are retained and reported; the physical topology merely diff --git a/runtime/internal/runtime/coro_native_deferred_replacement_llgo.go b/runtime/internal/runtime/coro_native_deferred_replacement_llgo.go index 19e1a53ed6..97745fb07c 100644 --- a/runtime/internal/runtime/coro_native_deferred_replacement_llgo.go +++ b/runtime/internal/runtime/coro_native_deferred_replacement_llgo.go @@ -23,8 +23,9 @@ import "github.com/goplus/llgo/runtime/internal/coro" // coroNativeMActivateDeferredReplacementV1 is the request-side half of a // demand-free native syscall handoff. It resolves only stable directory state; // neither the caller's stack boundary nor an LLVM coroutine handle is -// published. The exact execution-domain generation remains authoritative in -// parent.handoff and replacement.baton. +// published. It allocates a replacement directory slot only after winning the +// stable parent's Armed-to-Starting CAS. The exact execution-domain generation +// remains authoritative in parent.handoff and replacement.baton. func coroNativeMActivateDeferredReplacementV1( domain *coroNativeFleetDomainV1, ) bool { @@ -59,23 +60,25 @@ func coroNativeMActivateDeferredReplacementV1( activeSlot != parentSlot { return false } - startedSlot, won := parent.deferred.BeginStart() - if !won { + if slot != 0 || !parent.deferred.BeginStart() { continue } // BeginStart reloads the one-word gate and is authoritative. A delayed // accepted request may safely coalesce into a later armed call on the - // same stable parent even when that call obtained a different slot. - slot = startedSlot - if slot > coroNativeMDirectoryCapacityV1 { - _ = parent.deferred.RetryStart(startedSlot) + // same stable parent; the currently released baton selects the exact + // generation before this publisher materializes a directory slot. + released, releasedOK := parent.handoff.Released() + if !releasedOK || !released.Valid() || + released.OwnerEpoch == 0 { + _ = parent.deferred.RetryStart() return false } - replacement, replacementOK := coroNativeMOwnerForSlotV1(slot) - released, releasedOK := parent.handoff.Released() + slot, replacement, replacementOK := coroNativeMAllocateReplacementV1( + parentSlot, + domain.handle, + released, + ) if !replacementOK || replacement == nil || - coroNativeMOwnerLifecycleLoadV1(replacement) != - coroNativeMOwnerReplacementPublishedV1 || replacement.parentSlot != parentSlot || replacement.predecessorSlot != 0 || replacement.lineageRootSlot != slot || @@ -84,14 +87,18 @@ func coroNativeMActivateDeferredReplacementV1( replacement.thread != nil || replacement.self != nil || replacement.token != 0 || replacement.resume.Detached() || !replacement.handoff.Idle() || !replacement.deferred.Idle() || - !releasedOK || released != replacement.baton || + released != replacement.baton || replacement.ownerEpoch != released.OwnerEpoch { - _ = parent.deferred.RetryStart(slot) + if replacementOK && slot != 0 { + _ = coroNativeMReleaseUnstartedReplacementV1(slot) + } + _ = parent.deferred.RetryStart() return false } queued, started := coroNativeMRequestPhysicalOwnerV1(replacement, slot) if !started { - if !parent.deferred.RetryStart(slot) { + if !coroNativeMReleaseUnstartedReplacementV1(slot) || + !parent.deferred.RetryStart() { coroRuntimeAbort("native deferred replacement retry publication failed") } return false diff --git a/runtime/internal/runtime/coro_native_m_owner_llgo.go b/runtime/internal/runtime/coro_native_m_owner_llgo.go index e846690f69..91eb3851a8 100644 --- a/runtime/internal/runtime/coro_native_m_owner_llgo.go +++ b/runtime/internal/runtime/coro_native_m_owner_llgo.go @@ -383,6 +383,34 @@ func coroNativeMActiveOwnerV1( if !domainOK || domain == nil || !route.Valid() { return nil, nil, 0, 0, false } + return coroNativeMActiveOwnerInDomainV1(domain, route) +} + +func coroNativeMActiveOwnerAtRouteV1( + driver *coro.ExecutorDriver, + route coro.RouteID, +) ( + owner *coroNativeMOwnerV1, + domain *coroNativeFleetDomainV1, + slot, epoch uint32, + ok bool, +) { + domain, route, domainOK := coroNativeFleetExecutionDomainAtRouteV1(driver, route) + if !domainOK || domain == nil || !route.Valid() { + return nil, nil, 0, 0, false + } + return coroNativeMActiveOwnerInDomainV1(domain, route) +} + +func coroNativeMActiveOwnerInDomainV1( + domain *coroNativeFleetDomainV1, + route coro.RouteID, +) ( + owner *coroNativeMOwnerV1, + resolved *coroNativeFleetDomainV1, + slot, epoch uint32, + ok bool, +) { slot = coroNativeAtomicLoadV1(&coroNativeMDirectoryV1State.active[uint32(route)-1]) owner, ownerOK := coroNativeMOwnerForSlotV1(slot) if !ownerOK || owner.handle != domain.handle || owner.self == nil { @@ -463,6 +491,22 @@ func coroNativeMCurrentOwnerV1( return owner, domain, slot, epoch, true } +func coroNativeMCurrentOwnerAtRouteV1( + driver *coro.ExecutorDriver, + route coro.RouteID, +) ( + owner *coroNativeMOwnerV1, + domain *coroNativeFleetDomainV1, + slot, epoch uint32, + ok bool, +) { + owner, domain, slot, epoch, ok = coroNativeMActiveOwnerAtRouteV1(driver, route) + if !ok || pthread.Equal(owner.self, pthread.Self()) == 0 { + return nil, nil, 0, 0, false + } + return owner, domain, slot, epoch, true +} + func coroNativeMAllocateSuccessorV1( predecessorSlot uint32, predecessor *coroNativeMOwnerV1, diff --git a/runtime/internal/runtime/coro_os_thread_foreign_llgo.go b/runtime/internal/runtime/coro_os_thread_foreign_llgo.go index d7d46b5298..38ce30b343 100644 --- a/runtime/internal/runtime/coro_os_thread_foreign_llgo.go +++ b/runtime/internal/runtime/coro_os_thread_foreign_llgo.go @@ -40,6 +40,7 @@ type coroNativeForeignBoundaryV1 struct { parent *coroNativeMOwnerV1 domain *coroNativeFleetDomainV1 replacement *coroNativeMOwnerV1 + route coro.RouteID parentSlot uint32 replacementSlot uint32 @@ -118,7 +119,10 @@ func (boundary *coroNativeForeignBoundaryV1) startReplacementV1( _ = rolledBack return false } - if releaseManaged && !coroTargetReleaseManagedExecutionV1(boundary.driver) { + if releaseManaged && !coroTargetReleaseManagedExecutionAtRouteV1( + boundary.driver, + boundary.route, + ) { // Release may have already dropped the quota before a required waiter // doorbell failed. Its boolean result therefore cannot authorize restoring // the detached resume or releasing the handoff as though the lease were @@ -135,7 +139,10 @@ func (boundary *coroNativeForeignBoundaryV1) startReplacementV1( boundary.parent.handoff.Complete(baton) && coroNativeMReleaseUnstartedReplacementV1(slot) if releaseManaged { - rollback = coroTargetReenterManagedExecutionV1(boundary.driver) && + rollback = coroTargetReenterManagedExecutionAtRouteV1( + boundary.driver, + boundary.route, + ) && rollback } _ = rollback @@ -148,46 +155,36 @@ func (boundary *coroNativeForeignBoundaryV1) startReplacementV1( return true } -// prepareDeferredReplacementV1 publishes the logical handoff and replacement -// directory slot without consuming a physical M. Arm happens only after the -// managed-execution permit is released. The post-Arm demand recheck closes the -// request-before-Arm window; a later request races Withdraw directly in the -// stable parent owner. +// prepareDeferredReplacementV1 publishes the logical handoff without eagerly +// consuming either a replacement directory slot or a physical M. Arm happens +// only after the managed-execution permit is released. The post-Arm demand +// recheck closes the request-before-Arm window; a later request races Withdraw +// directly in the stable parent owner and allocates a slot only after it wins. func (boundary *coroNativeForeignBoundaryV1) prepareDeferredReplacementV1() bool { if boundary == nil || !boundary.active || boundary.driver == nil || boundary.parent == nil || boundary.domain == nil || boundary.replacement != nil || boundary.replacementSlot != 0 || boundary.baton.Valid() || boundary.replacementQueued || - boundary.replacementDeferred || !boundary.parent.deferred.Idle() { + boundary.replacementDeferred || !boundary.route.Valid() || + !boundary.parent.deferred.Idle() { return false } baton, begun := boundary.parent.handoff.Begin(boundary.ownerEpoch) if !begun { return false } - slot, replacement, allocated := coroNativeMAllocateReplacementV1( - boundary.parentSlot, - boundary.domain.handle, - baton, - ) - if !allocated { - rolledBack := boundary.parent.handoff.RequestReturn(baton) == - coro.ExecutionDomainHandoffReturnUnclaimed && - boundary.parent.handoff.Complete(baton) - _ = rolledBack - return false - } - if !coroTargetReleaseManagedExecutionV1(boundary.driver) { + if !coroTargetReleaseManagedExecutionAtRouteV1( + boundary.driver, + boundary.route, + ) { // Release may already have published the free permit before a waiter // doorbell failure. Nothing can safely restore ownership from this result. coroRuntimeAbort("native deferred foreign execution quota release failed") return false } - boundary.replacement = replacement - boundary.replacementSlot = slot boundary.baton = baton boundary.replacementDeferred = true - if !boundary.parent.deferred.Arm(slot) { + if !boundary.parent.deferred.Arm() { coroRuntimeAbort("native deferred replacement arm failed") return false } @@ -215,14 +212,19 @@ func (boundary *coroNativeForeignBoundaryV1) beginV1( boundary.parentSlot != 0 || boundary.replacementSlot != 0 || boundary.ownerEpoch != 0 || boundary.baton.Valid() || boundary.replacementQueued || boundary.replacementDeferred || - boundary.callbackAcquired { + boundary.callbackAcquired || boundary.route.Valid() { return false } - driver, _, _, ownerOK := coro.CurrentExecutorDriver(task) + driver, route, ownerOK := coro.CurrentExecutorDriverForCompilerTask(task) parent, domain, parentSlot, ownerEpoch, physicalOK := - coroNativeMCurrentOwnerV1(driver) + coroNativeMCurrentOwnerAtRouteV1(driver, route) if !ownerOK || !physicalOK || - !coro.DetachExecutorResume(&boundary.resume, driver, task, mode) { + !coro.DetachExecutorResumeForCompilerTask( + &boundary.resume, + driver, + task, + mode, + ) { return false } boundary.driver = driver @@ -231,6 +233,7 @@ func (boundary *coroNativeForeignBoundaryV1) beginV1( boundary.domain = domain boundary.parentSlot = parentSlot boundary.ownerEpoch = ownerEpoch + boundary.route = route boundary.active = true if lazyCompensation { if boundary.prepareDeferredReplacementV1() { @@ -247,6 +250,7 @@ func (boundary *coroNativeForeignBoundaryV1) beginV1( boundary.domain = nil boundary.parentSlot = 0 boundary.ownerEpoch = 0 + boundary.route = 0 boundary.active = false _ = restored return false @@ -411,42 +415,56 @@ func (boundary *coroNativeForeignBoundaryV1) completeDeferredReplacementV1( // Started reuse the ordinary generation-bound reclaim path. func (boundary *coroNativeForeignBoundaryV1) resolveDeferredReplacementV1() bool { if boundary == nil || !boundary.replacementDeferred || - boundary.parent == nil || boundary.replacement == nil || - boundary.replacementSlot == 0 || !boundary.baton.Valid() { + boundary.parent == nil || boundary.domain == nil || + boundary.replacement != nil || boundary.replacementSlot != 0 || + !boundary.baton.Valid() { return false } for { slot, phase, valid := boundary.parent.deferred.Observe() - if !valid || slot != boundary.replacementSlot { + if !valid { return false } switch phase { case coro.DeferredExecutorHandoffArmed: - if !boundary.parent.deferred.Withdraw(slot) { + if slot != 0 { + return false + } + if !boundary.parent.deferred.Withdraw() { continue } rolledBack := boundary.parent.handoff.RequestReturn(boundary.baton) == coro.ExecutionDomainHandoffReturnUnclaimed && - boundary.parent.handoff.Complete(boundary.baton) && - coroNativeMReleaseUnstartedReplacementV1(slot) + boundary.parent.handoff.Complete(boundary.baton) if !rolledBack { return false } - boundary.replacement = nil - boundary.replacementSlot = 0 boundary.baton = coro.ExecutionDomainHandoffHandle{} boundary.replacementQueued = false boundary.replacementDeferred = false return true case coro.DeferredExecutorHandoffStarting: + if slot != 0 { + return false + } if corofleet.Yield() != 0 { return false } - case coro.DeferredExecutorHandoffQueued: - boundary.replacementQueued = true - return true - case coro.DeferredExecutorHandoffStarted: - boundary.replacementQueued = false + case coro.DeferredExecutorHandoffQueued, + coro.DeferredExecutorHandoffStarted: + replacement, replacementOK := coroNativeMOwnerForSlotV1(slot) + if !replacementOK || replacement == nil || slot == 0 || + replacement.handle != boundary.domain.handle || + replacement.baton != boundary.baton || + replacement.parentSlot != boundary.parentSlot || + replacement.lineageRootSlot != slot || + coroNativeAtomicLoadV1(&replacement.lineageSlot) != slot || + replacement.ownerEpoch != boundary.baton.OwnerEpoch { + return false + } + boundary.replacement = replacement + boundary.replacementSlot = slot + boundary.replacementQueued = phase == coro.DeferredExecutorHandoffQueued return true default: return false @@ -474,7 +492,10 @@ func (boundary *coroNativeForeignBoundaryV1) finishV1() bool { boundary.replacementDeferred { return false } - if !coroTargetReenterManagedExecutionV1(boundary.driver) { + if !coroTargetReenterManagedExecutionAtRouteV1( + boundary.driver, + boundary.route, + ) { coroRuntimeAbort("native direct foreign execution quota reentry failed") } if !coro.RestoreExecutorResume(&boundary.resume) { @@ -486,6 +507,7 @@ func (boundary *coroNativeForeignBoundaryV1) finishV1() bool { boundary.domain = nil boundary.parentSlot = 0 boundary.ownerEpoch = 0 + boundary.route = 0 boundary.active = false return true } diff --git a/runtime/poll_worker_source_test.go b/runtime/poll_worker_source_test.go index 253afb7c7c..69009b1750 100644 --- a/runtime/poll_worker_source_test.go +++ b/runtime/poll_worker_source_test.go @@ -544,11 +544,11 @@ func TestRuntimeCoroWorkerBlockingCallHasOnlyScalarScratchSameMEntrance(t *testi "func coroNativeForeignBoundaryTLSStartV1() bool", "tls.AllocStatic[*coroNativeForeignBoundaryV1]()", "!coroNativeForeignBoundaryTLSReadyV1", - "coro.CurrentExecutorDriver(task)", - "coro.DetachExecutorResume(", + "coro.CurrentExecutorDriverForCompilerTask(task)", + "coro.DetachExecutorResumeForCompilerTask(", "boundary.parent.handoff.Begin(boundary.ownerEpoch)", "coroNativeMAllocateReplacementV1(", - "coroTargetReleaseManagedExecutionV1(boundary.driver)", + "coroTargetReleaseManagedExecutionAtRouteV1(", "coroNativeMRequestPhysicalOwnerV1(replacement, slot)", "corofleet.CancelReuseOwner(", "if !boundary.beginV1(task, mode, lazyCompensation)", @@ -560,7 +560,7 @@ func TestRuntimeCoroWorkerBlockingCallHasOnlyScalarScratchSameMEntrance(t *testi "boundary.parent.handoff.RequestReturn(boundary.baton)", "coroNativeMReplacementLineageOwnerV1(", "coroNativeMRecycleReplacementV1(returnedSlot)", - "coroTargetReenterManagedExecutionV1(boundary.driver)", + "coroTargetReenterManagedExecutionAtRouteV1(", "coro.RestoreExecutorResume(&boundary.resume)", "boundary.finishV1()", "//export __llgo_coro_foreign_reentry_acquire_v1", @@ -597,7 +597,7 @@ func TestRuntimeCoroWorkerBlockingCallHasOnlyScalarScratchSameMEntrance(t *testi ) } } - detach := strings.Index(entrance, "coro.DetachExecutorResume(") + detach := strings.Index(entrance, "coro.DetachExecutorResumeForCompilerTask(") startEntry := strings.Index(entrance, "func (boundary *coroNativeForeignBoundaryV1) startReplacementV1(") prepareEntry := strings.Index(entrance, "func (boundary *coroNativeForeignBoundaryV1) prepareDeferredReplacementV1()") beginEntry := strings.Index(entrance, "func (boundary *coroNativeForeignBoundaryV1) beginV1(") @@ -632,20 +632,22 @@ func TestRuntimeCoroWorkerBlockingCallHasOnlyScalarScratchSameMEntrance(t *testi } } helperFinish := strings.LastIndex(entrance, "boundary.finishV1()") - immediateRelease := strings.Index(immediate, "coroTargetReleaseManagedExecutionV1(boundary.driver)") + immediateRelease := strings.Index(immediate, "coroTargetReleaseManagedExecutionAtRouteV1(") immediateCreate := strings.Index(immediate, "coroNativeMRequestPhysicalOwnerV1(replacement, slot)") - preparedRelease := strings.Index(prepared, "coroTargetReleaseManagedExecutionV1(boundary.driver)") - preparedArm := strings.Index(prepared, "boundary.parent.deferred.Arm(slot)") + preparedRelease := strings.Index(prepared, "coroTargetReleaseManagedExecutionAtRouteV1(") + preparedArm := strings.Index(prepared, "boundary.parent.deferred.Arm()") preparedRecheck := strings.Index(prepared, "coro.ExecutorResumeHandoffCompensationRequired(&boundary.resume)") + preparedAllocate := strings.Index(prepared, "coroNativeMAllocateReplacementV1(") reclaimCancel := strings.Index(reclaim, "corofleet.CancelReuseOwner(") reclaimRequest := strings.Index(reclaim, "boundary.parent.handoff.RequestReturn(boundary.baton)") reclaimRecycle := strings.Index(reclaim, "coroNativeMRecycleReplacementV1(returnedSlot)") finishResolve := strings.Index(finishBody, "boundary.resolveDeferredReplacementV1()") finishReclaim := strings.Index(finishBody, "boundary.reclaimReplacementV1()") - finishReenter := strings.Index(finishBody, "coroTargetReenterManagedExecutionV1(boundary.driver)") + finishReenter := strings.Index(finishBody, "coroTargetReenterManagedExecutionAtRouteV1(") finishRestore := strings.Index(finishBody, "coro.RestoreExecutorResume(&boundary.resume)") if detach < beginEntry || immediateRelease < 0 || immediateCreate <= immediateRelease || preparedRelease < 0 || preparedArm <= preparedRelease || preparedRecheck <= preparedArm || + preparedAllocate >= 0 || reclaimCancel < 0 || reclaimRequest <= reclaimCancel || reclaimRecycle <= reclaimRequest || resolveEntry <= completeEntry || finishResolve < 0 || finishReclaim <= finishResolve || finishReenter <= finishReclaim || finishRestore <= finishReenter || @@ -659,10 +661,10 @@ func TestRuntimeCoroWorkerBlockingCallHasOnlyScalarScratchSameMEntrance(t *testi } quota := readRuntimePollFile(t, "internal/runtime/coro_execution_quota_native_llgo.go") for _, required := range []string{ - "func coroTargetReleaseManagedExecutionV1(driver *coro.ExecutorDriver) bool", - "func coroTargetReenterManagedExecutionV1(driver *coro.ExecutorDriver) bool", - "acquired, ok := coroTargetAcquireManagedExecutionV1(driver)", - "if !coroTargetWaitManagedExecutionV1(driver)", + "func coroTargetReleaseManagedExecutionAtRouteV1(", + "func coroTargetReenterManagedExecutionAtRouteV1(", + "acquired, ok := coroTargetAcquireManagedExecutionAtRouteV1(driver, route)", + "if !coroTargetWaitManagedExecutionAtRouteV1(driver, route)", } { if !strings.Contains(quota, required) { t.Errorf("native execution quota lacks same-M compensation marker %q", required) @@ -692,9 +694,9 @@ func TestRuntimeNativeSyscallDeferredReplacementUsesStableRequestGate(t *testing for _, required := range []string{ "type DeferredExecutorHandoff struct", "state uint32", - "func (handoff *DeferredExecutorHandoff) Arm(slot uint32) bool", + "func (handoff *DeferredExecutorHandoff) Arm() bool", "func (handoff *DeferredExecutorHandoff) BeginStart()", - "func (handoff *DeferredExecutorHandoff) Withdraw(slot uint32) bool", + "func (handoff *DeferredExecutorHandoff) Withdraw() bool", "func (handoff *DeferredExecutorHandoff) Complete(slot uint32) bool", } { if !strings.Contains(core, required) { @@ -707,9 +709,12 @@ func TestRuntimeNativeSyscallDeferredReplacementUsesStableRequestGate(t *testing release, arm, recheck := -1, -1, -1 if prepare >= 0 { prepared := boundary[prepare:] - release = strings.Index(prepared, "coroTargetReleaseManagedExecutionV1(boundary.driver)") - arm = strings.Index(prepared, "boundary.parent.deferred.Arm(slot)") + release = strings.Index(prepared, "coroTargetReleaseManagedExecutionAtRouteV1(") + arm = strings.Index(prepared, "boundary.parent.deferred.Arm()") recheck = strings.Index(prepared, "coro.ExecutorResumeHandoffCompensationRequired(&boundary.resume)") + if recheck > 0 && strings.Index(prepared[:recheck], "coroNativeMAllocateReplacementV1(") >= 0 { + t.Error("deferred native syscall eagerly allocates a replacement slot") + } } if prepare < 0 || release < 0 || arm <= release || recheck <= arm { t.Error("deferred native syscall does not release, arm, then recheck durable demand") @@ -719,6 +724,7 @@ func TestRuntimeNativeSyscallDeferredReplacementUsesStableRequestGate(t *testing for _, required := range []string{ "coroNativeMActiveOwnerV1(domain.driverOwnerV1())", "parent.deferred.BeginStart()", + "coroNativeMAllocateReplacementV1(", "coroNativeMRequestPhysicalOwnerV1(replacement, slot)", "parent.deferred.PublishStart(slot, queued)", } {