From 4c6f8c4ffac628b11877463fcbf12df3bd918b62 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Fri, 14 Aug 2026 19:57:09 +0800 Subject: [PATCH 1/2] benchmark: isolate coroutine worker file cost --- benchmark/coro_core/README.md | 4 ++ .../coro_core/testdata/io_workload/main.go | 44 ++++++++++++++++++- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/benchmark/coro_core/README.md b/benchmark/coro_core/README.md index 61339eb4c2..81d4ad0310 100644 --- a/benchmark/coro_core/README.md +++ b/benchmark/coro_core/README.md @@ -34,6 +34,10 @@ pollute the core artifact. Its modes are: - `file`: one persistent temporary file, with cache-hot 4 KiB seek/write/seek/read round trips; +- `file-syscall`: the same physical file transaction using direct + `syscall.Seek`, `syscall.Write`, and `syscall.Read` calls inside the measured + loop, isolating worker-boundary cost from the `os.File`/`internal/poll` + wrapper chain; - `tcp`: one persistent loopback TCP connection, with 4 KiB request/echo round trips between two goroutines. diff --git a/benchmark/coro_core/testdata/io_workload/main.go b/benchmark/coro_core/testdata/io_workload/main.go index cbd5f2e041..5df4138c9d 100644 --- a/benchmark/coro_core/testdata/io_workload/main.go +++ b/benchmark/coro_core/testdata/io_workload/main.go @@ -23,6 +23,7 @@ import ( "io" "net" "os" + "syscall" "time" ) @@ -84,6 +85,45 @@ func fileRoundTrip(count, rounds int) int { return checksum } +// fileSyscallRoundTrip performs the same persistent-file transaction as +// fileRoundTrip but deliberately bypasses os.File and internal/poll inside the +// measured loop. Keeping both modes in one source fixture separates the +// compiler/runtime worker boundary from coroutine frames introduced by the +// standard-library wrapper chain without changing the physical syscalls. +func fileSyscallRoundTrip(count, rounds int) int { + file, err := os.CreateTemp("", "llgo-coro-benchmark-syscall-*") + if err != nil { + panic(err) + } + path := file.Name() + defer os.Remove(path) + defer file.Close() + + fd := int(file.Fd()) + payload := make([]byte, payloadSize) + readback := make([]byte, payloadSize) + fillPayload(payload) + checksum := 0 + for round := range rounds { + for index := range count { + if _, err := syscall.Seek(fd, 0, 0); err != nil { + panic(err) + } + if n, err := syscall.Write(fd, payload); err != nil || n != len(payload) { + panic("short syscall file write") + } + if _, err := syscall.Seek(fd, 0, 0); err != nil { + panic(err) + } + if n, err := syscall.Read(fd, readback); err != nil || n != len(readback) { + panic("short syscall file read") + } + checksum += int(readback[(round*count+index)%len(readback)]) + } + } + return checksum +} + func writeFull(conn *net.TCPConn, payload []byte) { for len(payload) != 0 { n, err := conn.Write(payload) @@ -146,7 +186,7 @@ func tcpRoundTrip(count, rounds int) int { func main() { if len(os.Args) != 4 { - panic("usage: io_workload ") + panic("usage: io_workload ") } mode := os.Args[1] count, ok := parsePositive(os.Args[2]) @@ -163,6 +203,8 @@ func main() { switch mode { case "file": result = fileRoundTrip(count, rounds) + case "file-syscall": + result = fileSyscallRoundTrip(count, rounds) case "tcp": result = tcpRoundTrip(count, rounds) default: From ab3704e7c5769536f5f6e6fb92834de64675dff9 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Fri, 14 Aug 2026 21:35:02 +0800 Subject: [PATCH 2/2] runtime: demand-gate native syscall compensation --- benchmark/coro_core/README.md | 3 + .../coro_core/testdata/io_workload/main.go | 46 ++- cl/coro_worker.go | 124 ++++---- cl/coro_worker_cgo.go | 2 + cl/coro_worker_foreign.go | 1 + cl/coro_worker_test.go | 61 ++-- doc/coro-performance-baseline.md | 70 +++++ doc/llvm-coro-runtime-design.md | 48 ++++ internal/build/coro_native_fleet_e2e_test.go | 10 +- .../internal/coro/channel_operation_source.go | 5 +- .../coro/commit_capable_select_test.go | 2 +- .../internal/coro/executor_resume_handoff.go | 28 ++ .../coro/executor_resume_handoff_test.go | 94 ++++++ .../internal/coro/manual_operation_source.go | 2 +- .../internal/coro/owner_local_completion.go | 4 +- runtime/internal/coro/park_state_v2.go | 16 +- runtime/internal/coro/park_state_v2_test.go | 2 +- .../internal/coro/poll_operation_source.go | 2 +- .../coro/published_epoch_resolution_test.go | 4 +- .../internal/coro/run_decision_abi_test.go | 2 +- runtime/internal/coro/run_slice.go | 1 + .../coro/scalar_result_payload_test.go | 4 +- runtime/internal/coro/scheduler.go | 14 +- .../internal/coro/scheduler_park_v2_test.go | 20 +- runtime/internal/coro/task_cancel_test.go | 2 + runtime/internal/coro/task_control_source.go | 25 +- runtime/internal/coro/timer_registration.go | 2 +- runtime/internal/coro/wait_set_record.go | 37 ++- .../internal/coro/worker_operation_source.go | 2 +- runtime/internal/corofleet/_owner/owner.c | 137 ++++++++- runtime/internal/corofleet/_owner/owner.h | 4 + runtime/internal/corofleet/call_llgo.go | 16 ++ .../corofleet/native_factory_c_test.go | 59 +++- .../runtime/coro_native_m_owner_llgo.go | 120 ++++++-- .../coro_native_replacement_owner_llgo.go | 11 +- .../runtime/coro_os_thread_foreign_llgo.go | 271 ++++++++++++++---- runtime/poll_worker_source_test.go | 34 ++- 37 files changed, 1046 insertions(+), 239 deletions(-) diff --git a/benchmark/coro_core/README.md b/benchmark/coro_core/README.md index 81d4ad0310..575428a407 100644 --- a/benchmark/coro_core/README.md +++ b/benchmark/coro_core/README.md @@ -38,6 +38,9 @@ pollute the core artifact. Its modes are: `syscall.Seek`, `syscall.Write`, and `syscall.Read` calls inside the measured loop, isolating worker-boundary cost from the `os.File`/`internal/poll` wrapper chain; +- `pipe-block`: a scheduler-progress gate in which a direct pipe read blocks + the sole current M while a sleeping goroutine must be run by a compensation + M, wake on its timer, and write the byte which releases the reader; - `tcp`: one persistent loopback TCP connection, with 4 KiB request/echo round trips between two goroutines. diff --git a/benchmark/coro_core/testdata/io_workload/main.go b/benchmark/coro_core/testdata/io_workload/main.go index 5df4138c9d..2024ddbfab 100644 --- a/benchmark/coro_core/testdata/io_workload/main.go +++ b/benchmark/coro_core/testdata/io_workload/main.go @@ -124,6 +124,48 @@ func fileSyscallRoundTrip(count, rounds int) int { return checksum } +// blockingPipeRoundTrip is a scheduler-progress gate, not a throughput mode. +// The reader enters a genuinely blocking syscall on the current M. With +// GOMAXPROCS=1, the buffered armed handoff first proves that the writer has +// parked on its timer. Only a compensation M can then service that timer and +// issue the write which releases the reader. +func blockingPipeRoundTrip(count, rounds int) int { + fds := make([]int, 2) + if err := syscall.Pipe(fds); err != nil { + panic(err) + } + defer syscall.Close(fds[0]) + defer syscall.Close(fds[1]) + + readback := []byte{0} + checksum := 0 + for round := range rounds { + for index := range count { + value := byte(round*count + index + 1) + armed := make(chan struct{}, 1) + done := make(chan error, 1) + go func() { + armed <- struct{}{} + time.Sleep(time.Millisecond) + written, err := syscall.Write(fds[1], []byte{value}) + if err == nil && written != 1 { + err = syscall.EIO + } + done <- err + }() + <-armed + if read, err := syscall.Read(fds[0], readback); err != nil || read != 1 { + panic("blocking pipe read failed") + } + if err := <-done; err != nil { + panic(err) + } + checksum += int(readback[0]) + } + } + return checksum +} + func writeFull(conn *net.TCPConn, payload []byte) { for len(payload) != 0 { n, err := conn.Write(payload) @@ -186,7 +228,7 @@ func tcpRoundTrip(count, rounds int) int { func main() { if len(os.Args) != 4 { - panic("usage: io_workload ") + panic("usage: io_workload ") } mode := os.Args[1] count, ok := parsePositive(os.Args[2]) @@ -205,6 +247,8 @@ func main() { result = fileRoundTrip(count, rounds) case "file-syscall": result = fileSyscallRoundTrip(count, rounds) + case "pipe-block": + result = blockingPipeRoundTrip(count, rounds) case "tcp": result = tcpRoundTrip(count, rounds) default: diff --git a/cl/coro_worker.go b/cl/coro_worker.go index 6a18b17830..464f8c6b09 100644 --- a/cl/coro_worker.go +++ b/cl/coro_worker.go @@ -34,6 +34,7 @@ const ( coroHostOperationDeadlineResumeHookV1 = "__llgo_coro_host_operation_deadline_resume_v1" coroOSThreadLockedHookV1 = "__llgo_coro_os_thread_locked_v1" coroOSThreadForeignCallHookV1 = "__llgo_coro_os_thread_foreign_call_v1" + coroNativeSyscallCallHookV1 = "__llgo_coro_native_syscall_call_v1" ) const ( @@ -225,6 +226,7 @@ func (p *context) compileCoroHostOperation( physicalWords, keepaliveSlots, &coroHostWordOperationV1{metadata: metadata}, + false, ) return b.Aggregate( p.type_(results, llssa.InGo), @@ -245,9 +247,11 @@ func (p *context) compileCoroWorkerWordCall( args []llssa.Expr, keepaliveSlots []llssa.Expr, host *coroHostWordOperationV1, + nativeSyscall bool, ) coroWorkerWordResultV1 { body := p.requireCoroWorkerBody(b) if function.IsNil() || host == nil && traceTarget.IsNil() || host != nil && !traceTarget.IsNil() || + host != nil && nativeSyscall || len(args) > coroWorkerMaxArgsV1 || host != nil && len(host.metadata) != 0 && len(host.metadata) != coroHostOperationDeadlineMetadataWordsV1 { @@ -301,34 +305,34 @@ func (p *context) compileCoroWorkerWordCall( resumeHook = coroHostOperationDeadlineResumeHookV1 } } - state := b.Alloc(p.prog.RuntimeType(stateType), false) r1 := b.Alloc(p.prog.Uintptr(), false) r2 := b.Alloc(p.prog.Uintptr(), false) errno := b.Alloc(p.prog.Uintptr(), false) zero := p.prog.Zero(p.prog.Uintptr()) - physicalArgs := make([]llssa.Expr, 0, 7+len(metadata)+coroWorkerMaxArgsV1) - physicalArgs = append(physicalArgs, - body.task, - body.coro.Handle(), - b.Convert(b.Prog.VoidPtr(), body.header), - b.Convert(b.Prog.VoidPtr(), state), - function, - ) - if host == nil { - physicalArgs = append(physicalArgs, traceTarget) - } argcValue := p.prog.IntVal(uint64(len(args)), p.prog.Uint32()) - physicalArgs = append(physicalArgs, argcValue) - physicalArgs = append(physicalArgs, metadata...) - for index := 0; index < coroWorkerMaxArgsV1; index++ { - if index < len(args) { - physicalArgs = append(physicalArgs, args[index]) - } else { - physicalArgs = append(physicalArgs, zero) - } - } emitPark := func(worker llssa.Builder) { + state := worker.Alloc(p.prog.RuntimeType(stateType), false) + physicalArgs := make([]llssa.Expr, 0, 7+len(metadata)+coroWorkerMaxArgsV1) + physicalArgs = append(physicalArgs, + body.task, + body.coro.Handle(), + worker.Convert(worker.Prog.VoidPtr(), body.header), + worker.Convert(worker.Prog.VoidPtr(), state), + function, + ) + if host == nil { + physicalArgs = append(physicalArgs, traceTarget) + } + physicalArgs = append(physicalArgs, argcValue) + physicalArgs = append(physicalArgs, metadata...) + for index := 0; index < coroWorkerMaxArgsV1; index++ { + if index < len(args) { + physicalArgs = append(physicalArgs, args[index]) + } else { + physicalArgs = append(physicalArgs, zero) + } + } body.emitCoroParkOperation(p, worker, coroParkOperation{ prepare: func(active llssa.Builder, _, _ uint32) llssa.Expr { return active.Prog.BoolVal(true) @@ -360,17 +364,7 @@ func (p *context) compileCoroWorkerWordCall( return coroWorkerWordResultV1{r1: b.Load(r1), r2: b.Load(r2), errno: b.Load(errno)} } - lockedHook := p.pkg.NewFunc(coroOSThreadLockedHookV1, coroOSThreadLockedSignature(), llssa.InC) - locked := b.Call(lockedHook.Expr, body.task) - directBlock := b.Func.MakeBlock() - workerBlock := b.Func.MakeBlock() join := b.Func.MakeBlock() - b.If(locked, directBlock, workerBlock) - - b.SetBlockEx(directBlock, llssa.AtEnd, false) - direct := p.pkg.NewFunc( - coroOSThreadForeignCallHookV1, coroOSThreadForeignCallSignature(), llssa.InC, - ) directArgs := make([]llssa.Expr, 0, 4+coroWorkerMaxArgsV1+3) directArgs = append(directArgs, body.task, function, traceTarget, argcValue) for index := 0; index < coroWorkerMaxArgsV1; index++ { @@ -381,33 +375,47 @@ func (p *context) compileCoroWorkerWordCall( } } directArgs = append(directArgs, r1, r2, errno) - directStatus := b.Call(direct.Expr, directArgs...) - directNormal := b.Func.MakeBlock() - directMemoryFault := b.Func.MakeBlock() - directDivideFault := b.Func.MakeBlock() - directInvalid := b.Func.MakeBlock() - directDispatch := b.Switch(directStatus, directInvalid) - directDispatch.Case(p.prog.IntVal(coroWorkerResumeSuccessV1, p.prog.Uint32()), directNormal) - directDispatch.Case(p.prog.IntVal(coroWorkerResumeFaultMemoryV1, p.prog.Uint32()), directMemoryFault) - directDispatch.Case(p.prog.IntVal(coroWorkerResumeFaultDivideV1, p.prog.Uint32()), directDivideFault) - directDispatch.End(b) - b.SetBlockEx(directMemoryFault, llssa.AtEnd, false) - p.compileCoroTerminalFault(b, coroFaultNilV1) - b.SetBlockEx(directDivideFault, llssa.AtEnd, false) - p.compileCoroTerminalFault(b, coroFaultIntegerDivideByZeroV1) - b.SetBlockEx(directInvalid, llssa.AtEnd, false) - b.Unreachable() - b.SetBlockEx(directNormal, llssa.AtEnd, false) - b.Jump(join) - - b.SetBlockEx(workerBlock, llssa.AtEnd, false) - emitPark(b) - b.Jump(join) + emitDirect := func(hookName string) { + direct := p.pkg.NewFunc( + hookName, coroOSThreadForeignCallSignature(), llssa.InC, + ) + directStatus := b.Call(direct.Expr, directArgs...) + directNormal := b.Func.MakeBlock() + directMemoryFault := b.Func.MakeBlock() + directDivideFault := b.Func.MakeBlock() + directInvalid := b.Func.MakeBlock() + directDispatch := b.Switch(directStatus, directInvalid) + directDispatch.Case(p.prog.IntVal(coroWorkerResumeSuccessV1, p.prog.Uint32()), directNormal) + directDispatch.Case(p.prog.IntVal(coroWorkerResumeFaultMemoryV1, p.prog.Uint32()), directMemoryFault) + directDispatch.Case(p.prog.IntVal(coroWorkerResumeFaultDivideV1, p.prog.Uint32()), directDivideFault) + directDispatch.End(b) + b.SetBlockEx(directMemoryFault, llssa.AtEnd, false) + p.compileCoroTerminalFault(b, coroFaultNilV1) + b.SetBlockEx(directDivideFault, llssa.AtEnd, false) + p.compileCoroTerminalFault(b, coroFaultIntegerDivideByZeroV1) + b.SetBlockEx(directInvalid, llssa.AtEnd, false) + b.Unreachable() + b.SetBlockEx(directNormal, llssa.AtEnd, false) + b.Jump(join) + } + if nativeSyscall { + emitDirect(coroNativeSyscallCallHookV1) + } else { + lockedHook := p.pkg.NewFunc(coroOSThreadLockedHookV1, coroOSThreadLockedSignature(), llssa.InC) + locked := b.Call(lockedHook.Expr, body.task) + directBlock := b.Func.MakeBlock() + workerBlock := b.Func.MakeBlock() + b.If(locked, directBlock, workerBlock) + b.SetBlockEx(directBlock, llssa.AtEnd, false) + emitDirect(coroOSThreadForeignCallHookV1) + b.SetBlockEx(workerBlock, llssa.AtEnd, false) + emitPark(b) + b.Jump(join) + } b.SetBlockContinuation(join) - // The worker queue deliberately contains only copied uintptr words. Keep - // every independently proved typed owner live until the physical completion - // acknowledgement has selected this normal resume path; llvm.fake.use emits - // no machine code but forces CoroSplit to retain the values in the frame. + // Keep every independently proved typed owner live until direct return or + // worker completion has selected this normal path; llvm.fake.use emits no + // machine code but forces CoroSplit to retain the values in the frame. p.emitCoroKeepaliveSlots(b, keepaliveSlots) return coroWorkerWordResultV1{r1: b.Load(r1), r2: b.Load(r2), errno: b.Load(errno)} } @@ -520,7 +528,7 @@ func (p *context) compileCoroWorkerSyscall( compiled[index] = p.compileValue(b, argument) } keepaliveSlots := p.compileCoroCallKeepaliveSlots(b, direct) - result := p.compileCoroWorkerWordCall(b, compiled[0], compiled[0], compiled[1:], keepaliveSlots, nil) + result := p.compileCoroWorkerWordCall(b, compiled[0], compiled[0], compiled[1:], keepaliveSlots, nil, true) errnoValue := p.filterSyscallErrno(b, result.r1, result.errno, convention) return b.Aggregate(p.type_(results, llssa.InGo), result.r1, result.r2, errnoValue) } diff --git a/cl/coro_worker_cgo.go b/cl/coro_worker_cgo.go index c40f9ad457..c6511b6576 100644 --- a/cl/coro_worker_cgo.go +++ b/cl/coro_worker_cgo.go @@ -875,6 +875,7 @@ func (p *context) compileCoroWorkerCgoErrnoCall( []llssa.Expr{b.Convert(p.prog.Uintptr(), record)}, nil, nil, + false, ) b.KeepAlive(record) p.cgoRet = b.LoadKnownNonNil(b.FieldAddr(record, shape.resultField)) @@ -927,6 +928,7 @@ func (p *context) compileCoroWorkerCgoTransaction( []llssa.Expr{b.Convert(p.prog.Uintptr(), record)}, keepaliveSlots, nil, + false, ) b.KeepAlive(record) if shape.result == nil { diff --git a/cl/coro_worker_foreign.go b/cl/coro_worker_foreign.go index e4af115507..c3c57d5c8c 100644 --- a/cl/coro_worker_foreign.go +++ b/cl/coro_worker_foreign.go @@ -1270,6 +1270,7 @@ func (p *context) compileCoroWorkerForeignTransaction( []llssa.Expr{b.Convert(p.prog.Uintptr(), record)}, keepaliveSlots, nil, + false, ) // The native queue carries the record address as an opaque uintptr. This // post-acknowledgement use forces CoroSplit to retain the complete typed diff --git a/cl/coro_worker_test.go b/cl/coro_worker_test.go index 6e9632c783..bf13734d8f 100644 --- a/cl/coro_worker_test.go +++ b/cl/coro_worker_test.go @@ -134,72 +134,54 @@ func TestCoroWorkerSyscallCurrentFrame(t *testing.T) { t.Fatalf("verify worker coroutine before CoroSplit: %v\n%s", err, module.String()) } body := requireCoroPhysicalFunction(t, module, "foo.Root").String() - assertCoroCancellationTerminalStatusPublication(t, requireCoroPhysicalFunction(t, module, "foo.Root")) - if got := strings.Count(body, "call i8 @llvm.coro.suspend"); got != 3 { - t.Fatalf("Root coro.suspend calls = %d, want initial + worker + final:\n%s", got, body) + if got := strings.Count(body, "call i8 @llvm.coro.suspend"); got != 2 { + t.Fatalf("Root coro.suspend calls = %d, want initial + final:\n%s", got, body) } - for _, symbol := range []string{coroWorkerParkHookV1, coroWorkerResumeHookV1} { - if got := strings.Count(body, "@"+symbol); got != 1 { - t.Fatalf("Root references to %q = %d, want 1:\n%s", symbol, got, body) - } + if got := strings.Count(body, "@"+coroNativeSyscallCallHookV1); got != 1 { + t.Fatalf("Root native syscall calls = %d, want 1:\n%s", got, body) } - for _, symbol := range []string{coroOSThreadLockedHookV1, coroOSThreadForeignCallHookV1} { - if got := strings.Count(body, "@"+symbol); got != 1 { - t.Fatalf("Root references to locked-thread branch %q = %d, want 1:\n%s", symbol, got, body) + for _, symbol := range []string{ + coroWorkerParkHookV1, + coroWorkerResumeHookV1, + coroOSThreadLockedHookV1, + coroOSThreadForeignCallHookV1, + } { + if got := strings.Count(body, "@"+symbol); got != 0 { + t.Fatalf("Root retained obsolete syscall path %q = %d:\n%s", symbol, got, body) } } - lockedQuery := strings.Index(body, "call i1 @"+coroOSThreadLockedHookV1) - directCall := strings.Index(body, "call i32 @"+coroOSThreadForeignCallHookV1) - workerPark := strings.Index(body, "call void @"+coroWorkerParkHookV1) - if lockedQuery < 0 || directCall < lockedQuery || workerPark < lockedQuery { - t.Fatalf("Root does not branch from the lock query to both direct and worker paths:\n%s", body) - } for _, forbidden := range []string{"@foo.raw", "@llgo.syscall"} { if strings.Contains(body, forbidden) { t.Fatalf("worker lowering retained ordinary intrinsic call %q:\n%s", forbidden, body) } } dispatch := regexp.MustCompile( - `(?s)call i32 @` + regexp.QuoteMeta(coroWorkerResumeHookV1) + `\([^\n]+\)\n\s+switch i32 [^\[]+\[(.*?)\]`, + `(?s)call i32 @` + regexp.QuoteMeta(coroNativeSyscallCallHookV1) + `\([^\n]+\)\n\s+switch i32 [^\[]+\[(.*?)\]`, ).FindStringSubmatch(body) if len(dispatch) != 2 { - t.Fatalf("Root has no isolated worker resume switch:\n%s", body) + t.Fatalf("Root has no isolated native syscall result switch:\n%s", body) } for _, status := range []uint64{ coroWorkerResumeSuccessV1, - coroWorkerResumeTaskAbortV1, - coroWorkerResumeShutdownV1, coroWorkerResumeFaultMemoryV1, coroWorkerResumeFaultDivideV1, } { if !regexp.MustCompile(`(?m)^\s+i32 ` + strconv.FormatUint(status, 10) + `, label `).MatchString(dispatch[1]) { - t.Fatalf("Root worker resume switch lacks status %d:\n%s", status, dispatch[0]) + t.Fatalf("Root native syscall switch lacks status %d:\n%s", status, dispatch[0]) } } - park := strings.Index(body, "call void @"+coroWorkerParkHookV1) - suspend := strings.Index(body[park:], "call i8 @llvm.coro.suspend") - resume := strings.Index(body[park:], "call i32 @"+coroWorkerResumeHookV1) - if park < 0 || suspend < 0 || resume < 0 || suspend >= resume { - t.Fatalf("Root does not publish worker park before suspend and consume after resume:\n%s", body) - } runCoroABITestPipeline(t, prog, module) resumeBody := module.NamedFunction("foo.Root$coro.resume") - if resumeBody.IsNil() || !strings.Contains(resumeBody.String(), "call i32 @"+coroWorkerResumeHookV1) { - t.Fatalf("CoroSplit lost worker resume dispatch:\n%s", module.String()) + if resumeBody.IsNil() || !strings.Contains(resumeBody.String(), "call i32 @"+coroNativeSyscallCallHookV1) { + t.Fatalf("CoroSplit lost native syscall dispatch:\n%s", module.String()) } - assertCoroCancellationTerminalStatusPublication(t, resumeBody) object, err := prog.TargetMachine().EmitToMemoryBuffer(module, llvm.ObjectFile) if err != nil { t.Fatalf("emit post-CoroSplit worker object: %v\n%s", err, module.String()) } defer object.Dispose() - for _, symbol := range []string{ - coroWorkerParkHookV1, - coroWorkerResumeHookV1, - coroOSThreadLockedHookV1, - coroOSThreadForeignCallHookV1, - } { + for _, symbol := range []string{coroNativeSyscallCallHookV1} { if len(object.Bytes()) == 0 || !bytes.Contains(object.Bytes(), []byte(symbol)) { t.Fatalf("post-CoroSplit object lost worker ABI symbol %q", symbol) } @@ -260,9 +242,12 @@ func TestCoroWorkerSyscallFailureConventionsShareLowering(t *testing.T) { t.Errorf("%s contains wrong failure predicate %s:\n%s", test.name, forbidden, text) } } + if got := strings.Count(text, "@"+coroNativeSyscallCallHookV1); got != 1 { + t.Errorf("%s native syscall calls = %d, want one direct lowering", test.name, got) + } for _, symbol := range []string{coroWorkerParkHookV1, coroWorkerResumeHookV1} { - if got := strings.Count(text, "@"+symbol); got != 1 { - t.Errorf("%s %q calls = %d, want one shared park/resume lowering", test.name, symbol, got) + if got := strings.Count(text, "@"+symbol); got != 0 { + t.Errorf("%s retained worker syscall hook %q = %d", test.name, symbol, got) } } } diff --git a/doc/coro-performance-baseline.md b/doc/coro-performance-baseline.md index 9d086e789d..be8455a887 100644 --- a/doc/coro-performance-baseline.md +++ b/doc/coro-performance-baseline.md @@ -1735,3 +1735,73 @@ architecture-debt, native target-plan, representative native E2E, and actual file/TCP execution gates pass. The full `internal/cabi` suite was deliberately stopped after an unrelated fixture reached about 5 GiB RSS; the exact changed CABI tests pass, so no full-suite result is claimed here. + +### Demand-gated native syscall compensation checkpoint + +The next 2026-08-14 LLVM 22-only checkpoint separates a compiler-certified +`llgo.syscall` boundary from the general C/worker path. The generated native +syscall call executes on the current M, but first detaches the active resume and +releases its managed-execution lease. A replacement M is requested only when +the stable detached boundary contains work which can progress independently of +the syscall: runnable work, an attached timer/poll/manual/worker/channel source, +an active TaskControl endpoint, an owner-local completion, or an already +published executor/source request. Ordinary C and cgo calls retain their worker +or locked-thread paths. + +The scheduler maintains one owner-only aggregate count for attached external +source operations. Activation adds `ParkState.attached`, and the common detach +path removes one unit for each operation, so adding a future source through the +common park protocol needs no compensation-specific bit or scan. TaskControl is +the one orthogonal source which can receive a foreign post without parking a G; +its exact active-endpoint count is checked separately. An unmatched direct +channel waiter is deliberately not counted: it cannot complete without an +executing producer, and its completed form already publishes a durable executor +request. Request-driven activation of a deferred replacement remains a separate +liveness follow-up for the cross-route dependency in which that producer needs +another G on the syscall owner's route to release the syscall itself. + +The native standby factory also gained an asynchronous request/cancel protocol. +The syscall owner may publish one clean cached M without waiting for its Go +dispatch; a quick return cancels a still-queued request, while a dispatch winner +is joined through the existing generation/owner-epoch handoff. The C owner +state machine linearizes `STARTING -> DISPATCHING -> RUNNING`, and cancellation +may win only before dispatch. Repeated native harness tests cover both sides of +that race. A failed managed-quota release is fail-stop because its boolean result +cannot distinguish a pre-release failure from a post-release doorbell failure; +restoring the old resume in that ambiguous state could create two owners for one +P. + +Two broader gates were measured and rejected. Scanning every source catalog at +each syscall made the common path proportional to configured capacity. Treating +every parked G, or every direct-channel waiter, as compensation demand made a +long-lived internal waiter start a replacement for every regular-file syscall; +the 500-operation direct fixture regressed from about 2.3 ms to 12.1 ms. The +retained aggregate counts only independently progressing external obligations. + +The exact parent is `07c6ddf1fc61646cbfdf488a9e10209476d04741`. Go 1.26.5, +the parent LLGo binary and the candidate compile the same `io_workload` source +with independent caches, `-trimpath -ldflags='-s -w'`, LLVM 22.1.8 and +process-start `GOMAXPROCS=1`. Each row is an AB/BA-interleaved 31-process median +with the complete range: + +| 500-operation workload | Go 1.26.5 median [range] | parent median [range] | candidate median [range] | candidate / Go | +| --- | ---: | ---: | ---: | ---: | +| direct `syscall.Seek/Write/Seek/Read` | 1.022 ms [0.941, 1.282] | 5.379 ms [5.004, 7.441] | 2.240 ms [2.044, 2.421] | 2.19x | +| standard `os.File`/`io.ReadFull` chain | 1.021 ms [0.960, 1.803] | 8.772 ms [8.292, 10.518] | 5.556 ms [5.210, 5.888] | 5.44x | +| loopback TCP echo | 9.453 ms [6.704, 19.594] | 20.454 ms [18.939, 24.538] | 20.372 ms [19.224, 23.462] | 2.16x | + +The candidate is 58.4% faster than its parent in the direct syscall fixture and +36.7% faster through the standard file wrappers. TCP changes by less than one +percent and no network gain is claimed; its nonblocking poll/event path does not +use the new direct boundary. The stripped candidate is 7,050,672 bytes versus +1,797,874 bytes for Go (3.92x). + +A strengthened `pipe-block` gate first parks a writer on a real 1 ms timer, then +blocks the sole current M in `syscall.Read`; only a replacement M can service the +timer and write the byte. It passes in about 12 ms for ten iterations. A final +stress run launched 500 direct-file, 200 standard-file, 200 TCP and 100 blocking- +pipe processes without failure or a surviving LLGo process. Runtime core, full +runtime package, 20-repeat native standby, and focused compiler syscall/worker +tests also pass. The next file-performance target is the remaining coroutine +frame chain through `io.ReadFull`, `os.File` and `internal/poll`, not another +file-specific runtime path. diff --git a/doc/llvm-coro-runtime-design.md b/doc/llvm-coro-runtime-design.md index ad82ce4283..6017a17aff 100644 --- a/doc/llvm-coro-runtime-design.md +++ b/doc/llvm-coro-runtime-design.md @@ -3153,3 +3153,51 @@ Native退出实验模式的最低要求是 Tier 2,而不是少量自定义coro 4. 32位atomic、64位fallback、atomic.Pointer barrier、ISR临界区和RawCritical HAL/syscall verifier通过。 满足通用加对应平台门槛后,才能评估将coroutine scheduler设为该平台默认。Native pthread模式的移除是更晚、独立的兼容性决策。 + +## 36. Native syscall 的按需补偿边界 + +`llgo.syscall` 的 coroutine lowering 可以在完整 ProgramIR/callable +certificate 证明后,直接调用统一 native boundary;普通 C/cgo 调用不能因此 +绕过 worker 或 locked-M 路径。这个区分来自编译期语义,不允许在运行期用函数 +地址反查调用类型。 + +native boundary 先把 active resume detach,并释放 managed-execution lease。 +是否立即请求 replacement 只读取这个稳定点上的 O(1) 事实: + +1. P 已有 runnable; +2. active WaitSet 仍持有 timer、poll、manual、worker 或 channel source operation; +3. TaskControl 有 active endpoint; +4. owner-local completion、direct completion inbox、source/registry/scheduler request + 已经发布。 + +P 的 `externalWaitCount` 在 WaitSet activate 时一次增加 +`ParkState.attached`,并在 common detach 时逐 operation 减一。它是外部唤醒 +obligation 的投影,不是第二份 source 状态,也不扫描各 source catalog。 +TaskControl 不依赖 park,因此只保留一个正交的 active endpoint 计数。所有新 +event source 只要复用 common park/detach 协议,就自动进入这项判断。 + +未匹配 direct-channel waiter 不能独立完成,因此不能等同 timer/poll,否则一个 +常驻 runtime waiter 会使每个短 syscall 都启动 M。producer 完成匹配后必须先发布 +typed completion,再发布 exact executor request。完整的跨 route 闭环还要求:若 +目标 route 正处于“已释放 lease、原 M 阻塞、尚无 replacement”的窗口,这个 request +必须能按需激活预留的 replacement。该 follow-up 应复用 +`ExecutionDomainHandoff` 和 active-M directory;不得重新把所有 channel waiter +加入 eager gate,也不得引入第二套 channel 调度器。在此闭环验收前,timer/poll +驱动的 blocking compensation 已验证,但“remote producer 唤醒目标 G,而该 G +反过来解除原 syscall”的 direct-only 环不能标为完成。 + +物理 M 使用 generation 化的异步 standby request/cancel:request 只把 clean M 从 +standby 转入 `DISPATCHING`,不等待 C->Go;原 syscall 快速返回时只能取消仍处于 +`STARTING` 的请求,dispatch 已胜则走 exact handoff return 和 strong join。quota +release 若返回失败必须 fail-stop,因为它可能已经完成 release、只在唤醒其他 +waiter 时失败;这种状态下 restore continuation 会违反一个 P 只有一个 physical +owner 的硬门槛。 + +这项设计的验收 gate 是: + +- 短 regular-file syscall 不创建/dispatch replacement; +- sole-M blocking pipe + timer 必须继续前进; +- TCP nonblocking poll 路径结构和性能不变; +- queued-cancel、dispatch-wins、handoff return、shutdown 各竞态重复通过; +- direct-only 跨 route 依赖在 request-driven replacement 完成前保持显式未完成, + 不得用常驻 waiter eager compensation 掩盖。 diff --git a/internal/build/coro_native_fleet_e2e_test.go b/internal/build/coro_native_fleet_e2e_test.go index cda8dfe5a2..7375b749af 100644 --- a/internal/build/coro_native_fleet_e2e_test.go +++ b/internal/build/coro_native_fleet_e2e_test.go @@ -1783,15 +1783,15 @@ func TestCoroNativeFleetLockedForeignReleasesQuotaBeforeReplacementStarts(t *tes source, "if releaseManaged && !coroTargetReleaseManagedExecutionV1(boundary.driver)", ) - create := strings.Index( + request := strings.Index( source, - "if !coroNativeMStartPhysicalOwnerV1(replacement, slot)", + "queued, started := coroNativeMRequestPhysicalOwnerV1(replacement, slot)", ) - if release < 0 || create < 0 || release >= create { + if release < 0 || request < 0 || release >= request { t.Fatalf( - "locked-thread foreign owner must release its route quota before the replacement pthread can start: release=%d create=%d", + "locked-thread foreign owner must release its route quota before the replacement pthread can be requested: release=%d request=%d", release, - create, + request, ) } } diff --git a/runtime/internal/coro/channel_operation_source.go b/runtime/internal/coro/channel_operation_source.go index 82c9053671..8f1461d3e6 100644 --- a/runtime/internal/coro/channel_operation_source.go +++ b/runtime/internal/coro/channel_operation_source.go @@ -2594,6 +2594,7 @@ func (source *ChannelOperationSource) BeginClose(p *P, id OperationID) ChannelOp // ApplyOne; an exact owner-local direct commit consumes its already-closed // capability without repeating slot lookup, owner proof, or close. func applyClosedChannelOperationSlot( + p *P, slot *channelOperationSlot, id OperationID, record *OperationRecord, @@ -2632,7 +2633,7 @@ func applyClosedChannelOperationSlot( return OperationApplyInvalid } park, ticket := record.link.park, record.link.ticket - if !DetachParkWaitOperation(park, ticket, record, id) { + if !DetachParkWaitOperation(p, park, ticket, record, id) { return OperationApplyInvalid } // No new producer can enter and the admission join above proves no hchan @@ -2663,7 +2664,7 @@ func (source *ChannelOperationSource) ApplyOne(p *P, id OperationID, record *Ope closeResult != ChannelOperationAlreadyQuiesced { return OperationApplyInvalid } - return applyClosedChannelOperationSlot(slot, id, record, disposition) + return applyClosedChannelOperationSlot(p, slot, id, record, disposition) } // ConfirmQuiesced accepts the hchan/backend strong join. The admission word diff --git a/runtime/internal/coro/commit_capable_select_test.go b/runtime/internal/coro/commit_capable_select_test.go index 5de5e2f529..ffc9c5bce9 100644 --- a/runtime/internal/coro/commit_capable_select_test.go +++ b/runtime/internal/coro/commit_capable_select_test.go @@ -200,7 +200,7 @@ func (source *commitSelectFakeSource) finish( !AcknowledgeOperationResolution(&source.records[index], source.ids[index], disposition) { t.Fatalf("acknowledge candidate %d", index) } - if !DetachParkWaitOperation(source.state, source.ticket, &source.records[index], source.ids[index]) { + if !DetachParkWaitOperation(nil, source.state, source.ticket, &source.records[index], source.ids[index]) { t.Fatalf("detach candidate %d", index) } } diff --git a/runtime/internal/coro/executor_resume_handoff.go b/runtime/internal/coro/executor_resume_handoff.go index 5bd4fd6323..4423ec60e6 100644 --- a/runtime/internal/coro/executor_resume_handoff.go +++ b/runtime/internal/coro/executor_resume_handoff.go @@ -195,6 +195,34 @@ func ExecutorResumeHandoffReturnable(driver *ExecutorDriver) bool { return schedule == scheduleIdle || schedule == scheduleRequested } +// ExecutorResumeHandoffCompensationRequired classifies the exact detached +// owner boundary at which a native syscall decides whether another physical M +// must service this route. All timer, poll, manual, worker, and channel source +// operations owned by another task are rooted by the committed park queue at +// this stable boundary. The detached task itself has already proved a +// releasable park state. Task control is the sole externally posted source +// which does not require a park, so its aggregate live-endpoint count is +// checked separately. Durable facts and requests which have already arrived +// retain their independent O(1) gates below. +func ExecutorResumeHandoffCompensationRequired( + handoff *ExecutorResumeHandoff, +) (required, ok bool) { + if handoff == nil || handoff.state != executorResumeHandoffDetached || + handoff.driver == nil || handoff.task == nil || + handoff.action.Kind != ActionResume || handoff.action.Handle == nil || + !ExecutorResumeHandoffReturnable(handoff.driver) { + return false, false + } + driver := handoff.driver + p := driver.p + control := driver.sources.control + return runnableForOSThreadOwner(p) || + p.externalWaitCount != 0 || + control != nil && control.activeCount != 0 || + ownerLocalCompletionPending(driver) || + executorRunSourceRequested(driver), true +} + // ExecutorResumeHandoffContext returns the exact logical task and physical // parent handle for a detached same-M boundary which actually reentered // through a managed callback. It exposes no target function identity and diff --git a/runtime/internal/coro/executor_resume_handoff_test.go b/runtime/internal/coro/executor_resume_handoff_test.go index 7afe5b16e5..7e0de95c4a 100644 --- a/runtime/internal/coro/executor_resume_handoff_test.go +++ b/runtime/internal/coro/executor_resume_handoff_test.go @@ -223,6 +223,100 @@ func TestExecutorResumeHandoffRunsReplacementAndRestoresExactResume(t *testing.T runtime.KeepAlive(peer.frame.memory) } +func TestExecutorResumeHandoffCompensationDemandGate(t *testing.T) { + t.Run("quiescent", func(t *testing.T) { + p := new(P) + driver, _, _ := bindTestExecutorDriver(t, p) + task := newYieldingTestG(t, "foreign-wait-quiescent") + fixture := beginExecutorResumeHandoffFixture(t, p, driver, task) + if required, ok := ExecutorResumeHandoffCompensationRequired(&fixture.handoff); !ok || required { + t.Fatalf("quiescent compensation = (%t, %t), want false, true", required, ok) + } + fixture.restore(t) + }) + + t.Run("runnable", func(t *testing.T) { + p := new(P) + driver, _, _ := bindTestExecutorDriver(t, p) + task := newYieldingTestG(t, "foreign-wait-runnable-owner") + peer := newYieldingTestG(t, "foreign-wait-runnable-peer") + fixture := beginExecutorResumeHandoffFixture(t, p, driver, task) + if !Enqueue(p, peer.g) { + t.Fatal("enqueue compensation peer") + } + if required, ok := ExecutorResumeHandoffCompensationRequired(&fixture.handoff); !ok || !required { + t.Fatalf("runnable compensation = (%t, %t), want true, true", required, ok) + } + fixture.restore(t) + }) + + t.Run("request", func(t *testing.T) { + p := new(P) + driver, registry, executor := bindTestExecutorDriver(t, p) + task := newYieldingTestG(t, "foreign-wait-request") + fixture := beginExecutorResumeHandoffFixture(t, p, driver, task) + if result := registry.Request(executor); result != ExecutorRequestPublished { + t.Fatalf("publish compensation request = %d", result) + } + if required, ok := ExecutorResumeHandoffCompensationRequired(&fixture.handoff); !ok || !required { + t.Fatalf("requested compensation = (%t, %t), want true, true", required, ok) + } + fixture.restore(t) + }) + + t.Run("task-control-endpoint", func(t *testing.T) { + p := new(P) + driver := new(ExecutorDriver) + registry := new(ExecutorRegistry) + control := new(TaskControlSource) + executor := registerTestExecutor(t, registry) + if !BindExecutorSourceCatalog( + driver, + p, + registry, + executor, + ExecutorSourceCatalog{Control: control}, + ) { + t.Fatal("bind task-control compensation executor") + } + task := newYieldingTestG(t, "foreign-wait-control") + if !Enqueue(p, task.g) { + t.Fatal("enqueue task-control compensation owner") + } + step := runnerNextPhysicalAction(t, driver, task, ActionCheckResume) + resume, checked := Checked(p, task.g, step.Action, false) + if !checked || resume.Kind != ActionResume || resume.Handle != task.handle { + t.Fatalf("check task-control compensation resume = (%+v, %t)", resume, checked) + } + takeNormalRunnerDecision(t, task.g) + task.frame.header.SuspendReason = uint16(SuspendNone) + task.frame.header.Lifecycle = uint16(FrameActive) + id, registered := RegisterCurrentExecutorTaskControl(driver, task.g) + if !registered || !EnterOSThreadLock(task.g) { + t.Fatal("register task-control compensation endpoint") + } + fixture := &executorResumeHandoffFixture{ + p: p, driver: driver, task: task, resume: resume, + } + if !DetachExecutorResume( + &fixture.handoff, + driver, + task.g, + ExecutorResumeHandoffLockedForeign, + ) { + t.Fatal("detach task-control compensation owner") + } + if required, ok := ExecutorResumeHandoffCompensationRequired(&fixture.handoff); !ok || !required { + t.Fatalf("task-control compensation = (%t, %t), want true, true", required, ok) + } + fixture.restore(t) + if !BeginCloseCurrentExecutorTaskControl(driver, task.g, id) || + !FinishCloseCurrentExecutorTaskControl(driver, task.g, id) { + t.Fatal("close task-control compensation endpoint") + } + }) +} + func TestExecutorResumeHandoffPreservesInlineAwaitAncestry(t *testing.T) { p := new(P) driver, _, _ := bindTestExecutorDriver(t, p) diff --git a/runtime/internal/coro/manual_operation_source.go b/runtime/internal/coro/manual_operation_source.go index fdd9d87db1..98eaf79eb3 100644 --- a/runtime/internal/coro/manual_operation_source.go +++ b/runtime/internal/coro/manual_operation_source.go @@ -660,7 +660,7 @@ func (source *ManualOperationSource) ApplyOne(p *P, id OperationID, record *Oper return OperationApplyInvalid } park, ticket, wait := slot.record.link.park, slot.record.link.ticket, slot.record.link.wait - detached := wait != nil && DetachParkWaitOperation(park, ticket, &slot.record, id) || + detached := wait != nil && DetachParkWaitOperation(p, park, ticket, &slot.record, id) || wait == nil && DetachParkOperation(park, ticket, &slot.record, id) if !detached { return OperationApplyInvalid diff --git a/runtime/internal/coro/owner_local_completion.go b/runtime/internal/coro/owner_local_completion.go index 0c8500430c..22a5f6c4b5 100644 --- a/runtime/internal/coro/owner_local_completion.go +++ b/runtime/internal/coro/owner_local_completion.go @@ -369,7 +369,8 @@ func completeOwnerLocalDirectChannelInline( !operationCandidateExternallyCommitted(record) || link.park != state || link.wait != wait || link.operation != record || link.ticket != wait.ticket || link.caseID != 1 || link.previous != nil || link.next != nil || - state.head != link || !validReadyQueueHeader(p) || p.readyCount == ^uint32(0) || + state.head != link || p.externalWaitCount == 0 || + !validReadyQueueHeader(p) || p.readyCount == ^uint32(0) || (schedule != scheduleIdle && schedule != scheduleRequested) { return true, false } @@ -441,6 +442,7 @@ func completeOwnerLocalDirectChannelInline( result: ResumeResultChannel, small: small, state: resumePacketMaterialized, } *plan = ResumeCleanupPlan{} + p.externalWaitCount-- *state = ParkState{ ticket: ticket, phase: parkMaterialized, seed: uint32(preferred), outcome: ParkOutcomeCompleted, winnerCase: 1, diff --git a/runtime/internal/coro/park_state_v2.go b/runtime/internal/coro/park_state_v2.go index 88289513c9..8c3699b5a7 100644 --- a/runtime/internal/coro/park_state_v2.go +++ b/runtime/internal/coro/park_state_v2.go @@ -774,7 +774,7 @@ func ParkOperationClaim(record *OperationRecord, id OperationID) ParkClaimResult // DetachParkOperation clears the only physical-source pointer path to the // logical wait before publishing the ready transition. Physical quiescence is // intentionally not required here. -func detachParkOperation(state *ParkState, ticket ParkTicket, record *OperationRecord, id OperationID, fast bool) bool { +func detachParkOperation(p *P, state *ParkState, ticket ParkTicket, record *OperationRecord, id OperationID, fast bool) bool { validState := fast && validActiveParkStateHeader(state, ticket) || !fast && validParkState(state) if !validState || state.phase != parkDetaching || ticket != state.ticket || record == nil || !record.Matches(id) || record.phase != operationActive || record.disposition == OperationDispositionPending || @@ -785,6 +785,11 @@ func detachParkOperation(state *ParkState, ticket ParkTicket, record *OperationR if fast && link.wait == nil { return false } + accounted := fast && link.wait.state == waitSetRecordActive + if accounted && (p == nil || p.externalWaitCount == 0 || + link.wait.g == nil || &link.wait.g.park != state) { + return false + } previous, next := link.previous, link.next if previous == nil { if state.head != link { @@ -807,6 +812,9 @@ func detachParkOperation(state *ParkState, ticket ParkTicket, record *OperationR record.phase = operationDetached record.link = ParkLink{} state.attached-- + if accounted { + p.externalWaitCount-- + } if state.attached == 0 { if state.head != nil { return false @@ -820,14 +828,14 @@ func detachParkOperation(state *ParkState, ticket ParkTicket, record *OperationR } func DetachParkOperation(state *ParkState, ticket ParkTicket, record *OperationRecord, id OperationID) bool { - return detachParkOperation(state, ticket, record, id, false) + return detachParkOperation(nil, state, ticket, record, id, false) } // DetachParkWaitOperation is the O(1) scheduler-integrated detach path. Its // transient ParkLink carries the predecessor, and the complete wait-set was // already audited once by published-epoch resolution. -func DetachParkWaitOperation(state *ParkState, ticket ParkTicket, record *OperationRecord, id OperationID) bool { - return detachParkOperation(state, ticket, record, id, true) +func DetachParkWaitOperation(p *P, state *ParkState, ticket ParkTicket, record *OperationRecord, id OperationID) bool { + return detachParkOperation(p, state, ticket, record, id, true) } func ConsumeParkSet(state *ParkState, ticket ParkTicket) (outcome ParkOutcome, caseID uint32, lease OperationResultLease, ok bool) { diff --git a/runtime/internal/coro/park_state_v2_test.go b/runtime/internal/coro/park_state_v2_test.go index fd19bcae6e..f5f481e5cd 100644 --- a/runtime/internal/coro/park_state_v2_test.go +++ b/runtime/internal/coro/park_state_v2_test.go @@ -314,7 +314,7 @@ func TestDuplicateCaseSealFailureRemainsAbortable(t *testing.T) { for index := range records { discardUnselectedTestResult(t, &records[index], ids[index]) if !AcknowledgeOperationResolution(&records[index], ids[index], OperationDispositionCanceled) || - !DetachParkWaitOperation(&g.park, ticket, &records[index], ids[index]) { + !DetachParkWaitOperation(nil, &g.park, ticket, &records[index], ids[index]) { t.Fatalf("detach duplicate case %d", index) } } diff --git a/runtime/internal/coro/poll_operation_source.go b/runtime/internal/coro/poll_operation_source.go index 2f5e56f26b..caa87127d5 100644 --- a/runtime/internal/coro/poll_operation_source.go +++ b/runtime/internal/coro/poll_operation_source.go @@ -959,7 +959,7 @@ func (source *PollOperationSource) ApplyPollOperationV2One( return OperationApplyInvalid } park, ticket := slot.record.link.park, slot.record.link.ticket - if !DetachParkWaitOperation(park, ticket, &slot.record, id) { + if !DetachParkWaitOperation(p, park, ticket, &slot.record, id) { return OperationApplyInvalid } return OperationApplyDetached diff --git a/runtime/internal/coro/published_epoch_resolution_test.go b/runtime/internal/coro/published_epoch_resolution_test.go index c61489f8f9..f9ae5486de 100644 --- a/runtime/internal/coro/published_epoch_resolution_test.go +++ b/runtime/internal/coro/published_epoch_resolution_test.go @@ -105,7 +105,7 @@ func TestPublishedEpochResolutionHighCardinalityHasExactLinearSteps(t *testing.T } for index := range operations.records { - detachSchedulerParkV2(t, task.g, operations, index) + detachSchedulerParkV2(t, p, task.g, operations, index) } if promoted, ok := PollReady(p); !ok || promoted != 1 || HasWaiting(p) { t.Fatalf("promote bounded high-cardinality task = (%d, %t), waiting=%t", promoted, ok, HasWaiting(p)) @@ -181,7 +181,7 @@ func TestSchedulerOnlyReadyCommitFailsClosedAndRestoresAffectedSnapshot(t *testi } discardUnselectedTestResult(t, &record, id) if !AcknowledgeOperationResolution(&record, id, disposition) || - !DetachParkWaitOperation(&task.g.park, ticket, &record, id) { + !DetachParkWaitOperation(p, &task.g.park, ticket, &record, id) { t.Fatal("detach scheduler-only Ready cleanup") } if promoted, polled := PollReady(p); !polled || promoted != 1 { diff --git a/runtime/internal/coro/run_decision_abi_test.go b/runtime/internal/coro/run_decision_abi_test.go index 92d5deef74..b905a2e057 100644 --- a/runtime/internal/coro/run_decision_abi_test.go +++ b/runtime/internal/coro/run_decision_abi_test.go @@ -90,7 +90,7 @@ func TestTakeRunDecisionWordsPreservesExactTicketAndScalarizesLease(t *testing.T if count, ok := PollReady(p); !ok || count != 0 { t.Fatalf("resolve scalar decision park = (%d, %t)", count, ok) } - detachSchedulerParkV2(t, task.g, operations, 0) + detachSchedulerParkV2(t, p, task.g, operations, 0) if count, ok := PollReady(p); !ok || count != 1 { t.Fatalf("promote scalar decision park = (%d, %t)", count, ok) } diff --git a/runtime/internal/coro/run_slice.go b/runtime/internal/coro/run_slice.go index 47ff63bd15..e39cfbb85b 100644 --- a/runtime/internal/coro/run_slice.go +++ b/runtime/internal/coro/run_slice.go @@ -873,6 +873,7 @@ func ResumedExecutorRun( p.action != action || !p.runDecisionTaken || frame == nil || frame != g.active || frame.handle != action.Handle || frame.parkWait != g.active.parkWait || frame.parkWait.resumeKind != resumeBindingDirectChannel || + !canAccountWaitSetExternal(p, g, frame.parkWait) || !acknowledgeSuspendedGPreempt(g) { return Action{}, false, false } diff --git a/runtime/internal/coro/scalar_result_payload_test.go b/runtime/internal/coro/scalar_result_payload_test.go index c0f2123636..5d6db176ea 100644 --- a/runtime/internal/coro/scalar_result_payload_test.go +++ b/runtime/internal/coro/scalar_result_payload_test.go @@ -169,7 +169,7 @@ func consumeScalarCommitFake( t.Fatalf("candidate %d retained a payload without Owned result", index) } if !AcknowledgeOperationResolution(record, id, disposition) || - !DetachParkWaitOperation(source.state, source.ticket, record, id) || !ConfirmOperationQuiesced(record, id) { + !DetachParkWaitOperation(nil, source.state, source.ticket, record, id) || !ConfirmOperationQuiesced(record, id) { t.Fatalf("finish scalar candidate %d", index) } } @@ -385,7 +385,7 @@ func TestScalarResultCleanupIgnoresInvalidPayloadMetadata(t *testing.T) { t.Fatal("invalid unselected payload blocked loser cleanup") } if !AcknowledgeOperationResolution(&source.records[0], source.ids[0], OperationDispositionCanceled) || - !DetachParkWaitOperation(source.state, source.ticket, &source.records[0], source.ids[0]) || + !DetachParkWaitOperation(nil, source.state, source.ticket, &source.records[0], source.ids[0]) || !ConfirmOperationQuiesced(&source.records[0], source.ids[0]) { t.Fatal("finish invalid unselected scalar payload") } diff --git a/runtime/internal/coro/scheduler.go b/runtime/internal/coro/scheduler.go index 80d897d3ff..e5eac8814b 100644 --- a/runtime/internal/coro/scheduler.go +++ b/runtime/internal/coro/scheduler.go @@ -211,6 +211,14 @@ type P struct { // bounded work sharing independent of a whole-queue length scan while the // head/tail/link audit remains available at lifecycle and debug gates. readyCount uint32 + // externalWaitCount is the number of still-live external wake obligations + // rooted by active WaitSetRecords. Every attached source operation owns one + // unit. The common activate/detach lifecycle maintains it, so individual + // sources need no parallel compensation metadata. Pure logical parks, + // including an unmatched direct channel waiter, deliberately leave it zero: + // only an executing producer can turn that waiter into a durable executor + // request. + externalWaitCount uint32 } const ( @@ -880,7 +888,8 @@ func validSchedulerWaitQueues(p *P) bool { func emptySchedulerWaitQueues(p *P) bool { return p != nil && p.parkWaitHead == nil && p.parkWaitTail == nil && - p.affectedWaitHead == nil && p.affectedWaitTail == nil + p.affectedWaitHead == nil && p.affectedWaitTail == nil && + p.externalWaitCount == 0 } // pollReady is scheduler-thread-only. Parks are reached only through P's @@ -1506,7 +1515,8 @@ func Resumed(p *P, g *G, action Action) (Action, bool) { return Action{Kind: ActionYield}, true } if g.park.phase == parkParked { - if g.queued || g.nextReady != nil || !validParkWaitQueueHeader(p) || !validAffectedWaitQueueHeader(p) { + if g.queued || g.nextReady != nil || !validParkWaitQueueHeader(p) || + !validAffectedWaitQueueHeader(p) || !canAccountWaitSetExternal(p, g, g.active.parkWait) { return Action{}, false } if !acknowledgeSuspendedGPreempt(g) { diff --git a/runtime/internal/coro/scheduler_park_v2_test.go b/runtime/internal/coro/scheduler_park_v2_test.go index 763f2fa146..0b734def29 100644 --- a/runtime/internal/coro/scheduler_park_v2_test.go +++ b/runtime/internal/coro/scheduler_park_v2_test.go @@ -262,7 +262,7 @@ func publishSchedulerParkV2(t *testing.T, p *P, operations *schedulerParkV2Opera } } -func detachSchedulerParkV2(t *testing.T, g *G, operations *schedulerParkV2Operations, index int) { +func detachSchedulerParkV2(t *testing.T, p *P, g *G, operations *schedulerParkV2Operations, index int) { t.Helper() disposition, ok := OperationDispositionOf(&operations.records[index], operations.ids[index]) if !ok { @@ -272,7 +272,7 @@ func detachSchedulerParkV2(t *testing.T, g *G, operations *schedulerParkV2Operat if !AcknowledgeOperationResolution(&operations.records[index], operations.ids[index], disposition) { t.Fatalf("acknowledge scheduler park candidate %d", index) } - if !DetachParkWaitOperation(&g.park, operations.ticket, &operations.records[index], operations.ids[index]) { + if !DetachParkWaitOperation(p, &g.park, operations.ticket, &operations.records[index], operations.ids[index]) { t.Fatalf("detach scheduler park candidate %d", index) } } @@ -348,11 +348,11 @@ func TestSchedulerParkSetEarlyCompletionDetachGateAndRunDecision(t *testing.T) { if disposition, ok := OperationDispositionOf(&operations.records[1], operations.ids[1]); !ok || disposition != OperationDispositionLost { t.Fatalf("early loser disposition = (%d, %t)", disposition, ok) } - detachSchedulerParkV2(t, task.g, operations, 0) + detachSchedulerParkV2(t, p, task.g, operations, 0) if count, ok := PollReady(p); !ok || count != 0 || !HasWaiting(p) || ParkReady(&task.g.park, operations.ticket) { t.Fatalf("poll partial detach = (%d, %t), waiting=%t ready=%t", count, ok, HasWaiting(p), ParkReady(&task.g.park, operations.ticket)) } - detachSchedulerParkV2(t, task.g, operations, 1) + detachSchedulerParkV2(t, p, task.g, operations, 1) if !ParkReady(&task.g.park, operations.ticket) { t.Fatal("final source detach did not publish ParkReady") } @@ -456,7 +456,7 @@ func TestSchedulerRecordAwareWaitSetHighCardinalityUsesLocalDetach(t *testing.T) t.Fatal("distant candidate corruption escaped complete audit") } detached := make([]bool, candidateCount) - detachSchedulerParkV2(t, task.g, operations, headIndex) + detachSchedulerParkV2(t, p, task.g, operations, headIndex) detached[headIndex] = true distant.link.ticket = savedTicket if !validParkState(&task.g.park) { @@ -470,7 +470,7 @@ func TestSchedulerRecordAwareWaitSetHighCardinalityUsesLocalDetach(t *testing.T) middleIndex++ } for _, index := range []int{distantIndex, middleIndex} { - detachSchedulerParkV2(t, task.g, operations, index) + detachSchedulerParkV2(t, p, task.g, operations, index) detached[index] = true } detachedCount := 3 @@ -478,7 +478,7 @@ func TestSchedulerRecordAwareWaitSetHighCardinalityUsesLocalDetach(t *testing.T) if detached[index] { continue } - detachSchedulerParkV2(t, task.g, operations, index) + detachSchedulerParkV2(t, p, task.g, operations, index) detachedCount++ } if detachedCount != candidateCount || task.g.park.attached != 0 || !ParkReady(&task.g.park, operations.ticket) { @@ -519,7 +519,7 @@ func TestSchedulerParkSetReadyTaskCancelSuppressesCaseAndKeepsLease(t *testing.T if count, ok := PollReady(p); !ok || count != 0 || task.g.park.phase != parkDetaching { t.Fatalf("resolve late-cancel winner = (%d, %t), phase=%d", count, ok, task.g.park.phase) } - detachSchedulerParkV2(t, task.g, operations, 0) + detachSchedulerParkV2(t, p, task.g, operations, 0) if count, ok := PollReady(p); !ok || count != 1 || !task.g.queued || task.g.park.phase != parkReady { t.Fatalf("promote late-cancel winner = (%d, %t), queued=%t phase=%d", count, ok, task.g.queued, task.g.park.phase) } @@ -563,7 +563,7 @@ func TestSchedulerTaskCancelAfterDeliveredParkIsObservedAtNextResumeGate(t *test if count, ok := PollReady(p); !ok || count != 0 || task.g.park.phase != parkDetaching { t.Fatalf("resolve post-delivery park = (%d, %t), phase=%d", count, ok, task.g.park.phase) } - detachSchedulerParkV2(t, task.g, operations, 0) + detachSchedulerParkV2(t, p, task.g, operations, 0) if count, ok := PollReady(p); !ok || count != 1 { t.Fatalf("promote post-delivery park = (%d, %t)", count, ok) } @@ -809,7 +809,7 @@ func TestSchedulerParkPreparationAbortDetachesInlineWithoutParkingG(t *testing.T task.g.pending.kind != pendingNone || task.g.state != GRunning || HasWaiting(p) { t.Fatalf("abort producer-visible preparation: phase=%d pending=%d state=%d waiting=%t", task.g.park.phase, task.g.pending.kind, task.g.state, HasWaiting(p)) } - detachSchedulerParkV2(t, task.g, operations, 0) + detachSchedulerParkV2(t, p, task.g, operations, 0) if !ParkReady(&task.g.park, operations.ticket) { t.Fatal("preparation abort did not finish detach barrier") } diff --git a/runtime/internal/coro/task_cancel_test.go b/runtime/internal/coro/task_cancel_test.go index e8ddabb49e..3a442b5418 100644 --- a/runtime/internal/coro/task_cancel_test.go +++ b/runtime/internal/coro/task_cancel_test.go @@ -43,12 +43,14 @@ func attachWaitingTaskCancelFixture(p *P, g *G) { g.active = frame p.parkWaitHead = record p.parkWaitTail = record + p.externalWaitCount = g.park.attached } func detachWaitingTaskCancelFixture(p *P, g *G) { record := g.active.parkWait p.parkWaitHead = nil p.parkWaitTail = nil + p.externalWaitCount = 0 if p.affectedWaitHead == record { p.affectedWaitHead = nil p.affectedWaitTail = nil diff --git a/runtime/internal/coro/task_control_source.go b/runtime/internal/coro/task_control_source.go index 5a66d281f1..974eabf8cd 100644 --- a/runtime/internal/coro/task_control_source.go +++ b/runtime/internal/coro/task_control_source.go @@ -55,6 +55,11 @@ type TaskControlSource struct { routedProducerSource slots [TaskControlSourceCapacity]taskControlSlot scanLimit uint32 + // activeCount is the exact number of endpoints which can still admit an + // external Post. It is owner-only; producer requests retain the existing + // atomic pending word. The native blocking-syscall gate reads it only while + // this source's owner P is detached at a stable boundary. + activeCount uint32 } func taskControlScanLimit(source *TaskControlSource) (uint32, bool) { @@ -81,8 +86,9 @@ func taskControlReusableSlot(slot *taskControlSlot) bool { } func validTaskControlOwner(source *TaskControlSource, p *P) bool { - _, scanOK := taskControlScanLimit(source) - return scanOK && validRoutedProducerSource(&source.routedProducerSource, p) + limit, scanOK := taskControlScanLimit(source) + return scanOK && source.activeCount <= limit && + validRoutedProducerSource(&source.routedProducerSource, p) } // registeredTaskControlDelivery proves that one exact endpoint still pins its @@ -139,6 +145,7 @@ func RegisterTaskControl(source *TaskControlSource, p *P, task *G) (OperationID, if !activateProducerSourceSlot(&slot.producerSourceSlot, generation) { return OperationID{}, false } + source.activeCount++ return id, true } return OperationID{}, false @@ -319,11 +326,13 @@ func ConfirmTaskControlQuiesced(source *TaskControlSource, p *P, id OperationID) slot, ok := taskControlSlotFor(source, id) if !ok || !validTaskControlOwner(source, p) || preemptLoad(&slot.generation) != id.Generation || preemptLoad(&slot.state) != uint32(producerSourceClosing) || !producerSourceSlotQuiesced(&slot.producerSourceSlot) || - preemptLoad(&slot.request) != uint32(TaskCancelNone) || slot.task == nil || slot.task.taskControlLeases == 0 { + preemptLoad(&slot.request) != uint32(TaskCancelNone) || slot.task == nil || + slot.task.taskControlLeases == 0 || source.activeCount == 0 { return false } slot.task.taskControlLeases-- slot.task = nil + source.activeCount-- return markProducerSourceQuiesced(&slot.producerSourceSlot) } @@ -364,12 +373,14 @@ func validTaskControlTerminalSlot(source *TaskControlSource, index int, state pr } func taskControlTerminalLeaseCountsValid(source *TaskControlSource) bool { + activeCount := uint32(0) for index := range source.slots { slot := &source.slots[index] state := producerSourceLifecycle(preemptLoad(&slot.state)) if state != producerSourceActive && state != producerSourceClosing { continue } + activeCount++ needed := uint8(1) for prior := 0; prior < index; prior++ { other := &source.slots[prior] @@ -385,7 +396,7 @@ func taskControlTerminalLeaseCountsValid(source *TaskControlSource) bool { return false } } - return true + return activeCount == source.activeCount } // taskControlSourceCanBeginTerminalClose permits live task endpoints while @@ -489,6 +500,7 @@ func finishTaskControlSourceTerminalClose(source *TaskControlSource, p *P) bool } slot.task.taskControlLeases-- slot.task = nil + source.activeCount-- if !markProducerSourceQuiesced(&slot.producerSourceSlot) { return false } @@ -504,7 +516,8 @@ func finishTaskControlSourceTerminalClose(source *TaskControlSource, p *P) bool } func taskControlSourceEmpty(source *TaskControlSource, p *P) bool { - if source == nil || !routedProducerHeaderEmpty(&source.routedProducerSource, p) { + if source == nil || source.activeCount != 0 || + !routedProducerHeaderEmpty(&source.routedProducerSource, p) { return false } for index := range source.slots { @@ -523,6 +536,7 @@ func BindTaskControlSourceAtRoute(source *TaskControlSource, p *P, route RouteID return false } source.scanLimit = 0 + source.activeCount = 0 return true } @@ -539,6 +553,7 @@ func UnbindTaskControlSource(source *TaskControlSource, p *P) bool { return false } source.scanLimit = 0 + source.activeCount = 0 return true } diff --git a/runtime/internal/coro/timer_registration.go b/runtime/internal/coro/timer_registration.go index bb0b6d679f..1401455ea5 100644 --- a/runtime/internal/coro/timer_registration.go +++ b/runtime/internal/coro/timer_registration.go @@ -731,7 +731,7 @@ func (table *TimerRegistrationTable) ApplyTimerV2One(p *P, id OperationID, recor // checks above repeat the timer-local ownership proof. Failure after the two // monotonic acknowledgements is therefore fail-stop corruption, not a // Deferred retry: replaying an already-applied disposition would be unsafe. - if !DetachParkWaitOperation(park, ticket, &slot.record, id) { + if !DetachParkWaitOperation(p, park, ticket, &slot.record, id) { return OperationApplyInvalid } return OperationApplyDetached diff --git a/runtime/internal/coro/wait_set_record.go b/runtime/internal/coro/wait_set_record.go index 493ac9c267..1538ff4298 100644 --- a/runtime/internal/coro/wait_set_record.go +++ b/runtime/internal/coro/wait_set_record.go @@ -102,7 +102,8 @@ func validCommittedWaitSetRecord(record *WaitSetRecord, g *G, frame *Frame) bool func validParkWaitQueueHeader(p *P) bool { return p != nil && (p.parkWaitHead == nil) == (p.parkWaitTail == nil) && - (p.parkWaitHead == nil || p.parkWaitHead.activePrev == nil && p.parkWaitTail.activeNext == nil) + (p.parkWaitHead == nil || p.parkWaitHead.activePrev == nil && p.parkWaitTail.activeNext == nil) && + (p.parkWaitHead != nil || p.externalWaitCount == 0) } func validAffectedWaitQueueHeader(p *P) bool { @@ -155,11 +156,16 @@ func validActiveParkStateHeader(state *ParkState, ticket ParkTicket) bool { // ParkState headers. It never walks candidate ParkLinks and is the predicate // used by fact publication, cancellation, affected pop, and promotion. func validActiveWaitSetRecordFast(p *P, record *WaitSetRecord) bool { - if p == nil || record == nil || record.state != waitSetRecordActive || !validParkTicket(record.ticket) || + if record == nil { + return false + } + external, externalOK := waitSetExternalCount(record.g, record) + if p == nil || record.state != waitSetRecordActive || !validParkTicket(record.ticket) || record.g == nil || !ValidG(record.g) || record.g.state != GWaiting || !record.g.waiting || record.g.queued || record.g.nextReady != nil || record.g.runP != nil || record.g.transferState != runnableTransferGIdle || record.g.active == nil || - record.g.active.parkWait != record || !validActiveParkStateHeader(&record.g.park, record.ticket) || + record.g.active.parkWait != record || !externalOK || external > p.externalWaitCount || + !validActiveParkStateHeader(&record.g.park, record.ticket) || !validTrustedWaitSetResumeBinding(record) { return false } @@ -196,13 +202,19 @@ func validParkWaitQueue(p *P) bool { } } var tail *WaitSetRecord + var externalWaitCount uint32 for record := p.parkWaitHead; record != nil; record = record.activeNext { if !validActiveWaitSetRecord(p, record) { return false } + count, countOK := waitSetExternalCount(record.g, record) + if !countOK || count > ^uint32(0)-externalWaitCount { + return false + } + externalWaitCount += count tail = record } - if tail != p.parkWaitTail { + if tail != p.parkWaitTail || externalWaitCount != p.externalWaitCount { return false } for slow, fast := p.affectedWaitHead, p.affectedWaitHead; fast != nil && fast.workNext != nil; { @@ -300,19 +312,34 @@ func activateWaitSetRecord(p *P, g *G, record *WaitSetRecord) bool { if !validParkWaitQueueHeader(p) || !validAffectedWaitQueueHeader(p) || !validCommittedWaitSetRecord(record, g, g.active) || g.state != GWaiting || g.waiting || g.queued || g.nextReady != nil || g.runP != nil || - !validParkState(&g.park) || g.park.phase != parkParked { + !validParkState(&g.park) || g.park.phase != parkParked || + !canAccountWaitSetExternal(p, g, record) { return false } activateWaitSetRecordUnchecked(p, g, record) return true } +func waitSetExternalCount(g *G, record *WaitSetRecord) (uint32, bool) { + if g == nil || record == nil { + return 0, false + } + return g.park.attached, true +} + +func canAccountWaitSetExternal(p *P, g *G, record *WaitSetRecord) bool { + count, ok := waitSetExternalCount(g, record) + return ok && p != nil && count <= ^uint32(0)-p.externalWaitCount +} + // activateWaitSetRecordUnchecked is the no-fail queue write half. Resumed may // call it directly only when dispatchPending has just performed the complete // direct-channel ParkState/record/cleanup audit in the same scheduler-owner // activation; no callback or producer can mutate these owner-only queues in // between. Other entry points retain activateWaitSetRecord's full validation. func activateWaitSetRecordUnchecked(p *P, g *G, record *WaitSetRecord) { + count, _ := waitSetExternalCount(g, record) + p.externalWaitCount += count record.activePrev = p.parkWaitTail record.state = waitSetRecordActive g.waiting = true diff --git a/runtime/internal/coro/worker_operation_source.go b/runtime/internal/coro/worker_operation_source.go index 5f5dbf2f7f..7cdfce719f 100644 --- a/runtime/internal/coro/worker_operation_source.go +++ b/runtime/internal/coro/worker_operation_source.go @@ -816,7 +816,7 @@ func (source *WorkerOperationSource) ApplyOne(p *P, id OperationID, record *Oper return OperationApplyInvalid } park, ticket, wait := slot.record.link.park, slot.record.link.ticket, slot.record.link.wait - detached := wait != nil && DetachParkWaitOperation(park, ticket, &slot.record, id) || + detached := wait != nil && DetachParkWaitOperation(p, park, ticket, &slot.record, id) || wait == nil && DetachParkOperation(park, ticket, &slot.record, id) if !detached { return OperationApplyInvalid diff --git a/runtime/internal/corofleet/_owner/owner.c b/runtime/internal/corofleet/_owner/owner.c index 8aabb74f81..823eb2d8ba 100644 --- a/runtime/internal/corofleet/_owner/owner.c +++ b/runtime/internal/corofleet/_owner/owner.c @@ -46,6 +46,7 @@ enum { enum llgo_coro_fleet_owner_state_v1 { LLGO_CORO_FLEET_OWNER_UNUSED_V1 = 0, LLGO_CORO_FLEET_OWNER_STARTING_V1, + LLGO_CORO_FLEET_OWNER_DISPATCHING_V1, LLGO_CORO_FLEET_OWNER_RUNNING_V1, LLGO_CORO_FLEET_OWNER_RETURNED_V1, LLGO_CORO_FLEET_OWNER_PARKING_V1, @@ -317,9 +318,20 @@ static void *llgo_coro_fleet_owner_run_v1( (void)pthread_mutex_unlock(&factory->mutex); return 0; } + while (record->state == LLGO_CORO_FLEET_OWNER_STARTING_V1 && + record->thread == (pthread_t)0) { + if (pthread_cond_wait(&factory->changed, &factory->mutex) != 0) { + record->state = LLGO_CORO_FLEET_OWNER_FAILED_V1; + factory->state = LLGO_CORO_FLEET_FACTORY_FAILED_V1; + (void)pthread_mutex_unlock(&factory->mutex); + return (void *)(uintptr_t)1; + } + } if (record->state != LLGO_CORO_FLEET_OWNER_STARTING_V1 || + record->thread == (pthread_t)0 || + pthread_equal(record->thread, pthread_self()) == 0 || record->slot == 0 || record->token == 0 || - record->acknowledged != 0 || record->published != 0) { + record->acknowledged != 0 || record->published > 1) { record->state = LLGO_CORO_FLEET_OWNER_FAILED_V1; factory->state = LLGO_CORO_FLEET_FACTORY_FAILED_V1; (void)pthread_cond_broadcast(&factory->changed); @@ -327,6 +339,13 @@ static void *llgo_coro_fleet_owner_run_v1( return (void *)(uintptr_t)1; } uint32_t slot = record->slot; + record->state = LLGO_CORO_FLEET_OWNER_DISPATCHING_V1; + if (pthread_cond_broadcast(&factory->changed) != 0) { + record->state = LLGO_CORO_FLEET_OWNER_FAILED_V1; + factory->state = LLGO_CORO_FLEET_FACTORY_FAILED_V1; + (void)pthread_mutex_unlock(&factory->mutex); + return (void *)(uintptr_t)1; + } if (pthread_mutex_unlock(&factory->mutex) != 0) { return (void *)(uintptr_t)1; } @@ -704,6 +723,113 @@ int __llgo_coro_fleet_owner_try_reuse_v1( return 0; } +int __llgo_coro_fleet_owner_request_reuse_v1( + pthread_t *thread, uint32_t *token, uint32_t slot) { + struct llgo_coro_fleet_factory_v1 *factory = + &llgo_coro_fleet_factory_v1; + if (thread == 0 || token == 0 || slot == 0 || + slot > LLGO_CORO_FLEET_OWNER_SLOT_CAPACITY_V1 || + pthread_mutex_lock(&factory->mutex) != 0) { + return -1; + } + *thread = (pthread_t)0; + *token = 0; + struct llgo_coro_fleet_owner_record_v1 *existing = 0; + if (factory->state == LLGO_CORO_FLEET_FACTORY_UNUSED_V1 || + factory->state == LLGO_CORO_FLEET_FACTORY_STOPPING_V1 || + factory->state == LLGO_CORO_FLEET_FACTORY_FAILED_V1 || + llgo_coro_fleet_find_record_for_slot_locked_v1( + slot, &existing) != 0 || existing != 0) { + (void)pthread_mutex_unlock(&factory->mutex); + return -1; + } + if (factory->standby_head == 0) { + if (factory->standby_count != 0) { + factory->state = LLGO_CORO_FLEET_FACTORY_FAILED_V1; + (void)pthread_mutex_unlock(&factory->mutex); + return -1; + } + (void)pthread_mutex_unlock(&factory->mutex); + return 1; + } + struct llgo_coro_fleet_owner_record_v1 *record = + factory->standby_head; + if (factory->standby_count == 0 || + record->state != LLGO_CORO_FLEET_OWNER_STANDBY_V1 || + record->thread == (pthread_t)0 || record->slot != 0 || + record->token == 0 || record->acknowledged != 0 || + record->published != 0 || record->joining != 0) { + factory->state = LLGO_CORO_FLEET_FACTORY_FAILED_V1; + (void)pthread_mutex_unlock(&factory->mutex); + return -1; + } + factory->standby_head = record->standby_next; + factory->standby_count--; + record->standby_next = 0; + record->slot = slot; + record->published = 1; + record->state = LLGO_CORO_FLEET_OWNER_STARTING_V1; + *thread = record->thread; + *token = record->token; + if (pthread_cond_broadcast(&factory->changed) != 0 || + pthread_mutex_unlock(&factory->mutex) != 0) { + return -1; + } + return 0; +} + +int __llgo_coro_fleet_owner_cancel_reuse_v1( + pthread_t thread, uint32_t token, uint32_t slot) { + struct llgo_coro_fleet_factory_v1 *factory = + &llgo_coro_fleet_factory_v1; + if (thread == (pthread_t)0 || token == 0 || slot == 0 || + pthread_mutex_lock(&factory->mutex) != 0) { + return -1; + } + struct llgo_coro_fleet_owner_record_v1 *record = 0; + struct llgo_coro_fleet_owner_record_v1 *slot_record = 0; + if (llgo_coro_fleet_find_record_for_token_locked_v1( + token, &record) != 0 || + llgo_coro_fleet_find_record_for_slot_locked_v1( + slot, &slot_record) != 0 || + record == 0 || slot_record != record || + record->thread == (pthread_t)0 || + pthread_equal(record->thread, thread) == 0 || + record->slot != slot || record->token != token || + record->published != 1 || record->joining != 0) { + (void)pthread_mutex_unlock(&factory->mutex); + return -1; + } + if (record->state != LLGO_CORO_FLEET_OWNER_STARTING_V1) { + int dispatched = + (record->state == LLGO_CORO_FLEET_OWNER_DISPATCHING_V1 && + record->acknowledged == 0) || + ((record->state == LLGO_CORO_FLEET_OWNER_RUNNING_V1 || + record->state == LLGO_CORO_FLEET_OWNER_RETURNED_V1) && + record->acknowledged == 1); + (void)pthread_mutex_unlock(&factory->mutex); + return dispatched ? 1 : -1; + } + if (record->acknowledged != 0 || factory->standby_count >= + LLGO_CORO_FLEET_OWNER_STANDBY_CAPACITY_V1) { + factory->state = LLGO_CORO_FLEET_FACTORY_FAILED_V1; + (void)pthread_mutex_unlock(&factory->mutex); + return -1; + } + llgo_coro_fleet_clear_slot_locked_v1(record); + record->slot = 0; + record->published = 0; + record->state = LLGO_CORO_FLEET_OWNER_STANDBY_V1; + record->standby_next = factory->standby_head; + factory->standby_head = record; + factory->standby_count++; + if (pthread_cond_broadcast(&factory->changed) != 0 || + pthread_mutex_unlock(&factory->mutex) != 0) { + return -1; + } + return 0; +} + int __llgo_coro_fleet_owner_ready_v1(uint32_t slot) { struct llgo_coro_fleet_factory_v1 *factory = &llgo_coro_fleet_factory_v1; @@ -718,7 +844,7 @@ int __llgo_coro_fleet_owner_ready_v1(uint32_t slot) { return -1; } while (record != 0 && record->thread == (pthread_t)0 && - record->state == LLGO_CORO_FLEET_OWNER_STARTING_V1 && + record->state == LLGO_CORO_FLEET_OWNER_DISPATCHING_V1 && factory->state != LLGO_CORO_FLEET_FACTORY_FAILED_V1) { if (pthread_cond_wait(&factory->changed, &factory->mutex) != 0) { factory->state = LLGO_CORO_FLEET_FACTORY_FAILED_V1; @@ -728,9 +854,9 @@ int __llgo_coro_fleet_owner_ready_v1(uint32_t slot) { } if (record == 0 || record->thread == (pthread_t)0 || pthread_equal(record->thread, pthread_self()) == 0 || - record->state != LLGO_CORO_FLEET_OWNER_STARTING_V1 || + record->state != LLGO_CORO_FLEET_OWNER_DISPATCHING_V1 || record->slot != slot || record->acknowledged != 0 || - record->published != 0) { + record->published > 1) { (void)pthread_mutex_unlock(&factory->mutex); return -1; } @@ -827,7 +953,8 @@ int __llgo_coro_fleet_owner_release_v1( return -1; } while (record != 0 && - record->state == LLGO_CORO_FLEET_OWNER_RUNNING_V1) { + (record->state == LLGO_CORO_FLEET_OWNER_DISPATCHING_V1 || + record->state == LLGO_CORO_FLEET_OWNER_RUNNING_V1)) { if (pthread_cond_wait(&factory->changed, &factory->mutex) != 0) { factory->state = LLGO_CORO_FLEET_FACTORY_FAILED_V1; (void)pthread_mutex_unlock(&factory->mutex); diff --git a/runtime/internal/corofleet/_owner/owner.h b/runtime/internal/corofleet/_owner/owner.h index 85e512ba2e..224215cbe7 100644 --- a/runtime/internal/corofleet/_owner/owner.h +++ b/runtime/internal/corofleet/_owner/owner.h @@ -26,6 +26,10 @@ int __llgo_coro_fleet_owner_create_v3( pthread_t *thread, uint32_t *token, uint32_t slot); int __llgo_coro_fleet_owner_try_reuse_v1( pthread_t *thread, uint32_t *token, uint32_t slot); +int __llgo_coro_fleet_owner_request_reuse_v1( + pthread_t *thread, uint32_t *token, uint32_t slot); +int __llgo_coro_fleet_owner_cancel_reuse_v1( + pthread_t thread, uint32_t token, uint32_t slot); int __llgo_coro_fleet_owner_ready_v1(uint32_t slot); int __llgo_coro_fleet_owner_join_v1(pthread_t thread, uint32_t token); int __llgo_coro_fleet_owner_release_v1( diff --git a/runtime/internal/corofleet/call_llgo.go b/runtime/internal/corofleet/call_llgo.go index 076dd09dbf..90c48cee61 100644 --- a/runtime/internal/corofleet/call_llgo.go +++ b/runtime/internal/corofleet/call_llgo.go @@ -61,6 +61,22 @@ func CreateOwner(thread *pthread.Thread, token *uint32, slot uint32) c.Int //go:linkname TryReuseOwner C.__llgo_coro_fleet_owner_try_reuse_v1 func TryReuseOwner(thread *pthread.Thread, token *uint32, slot uint32) c.Int +// RequestReuseOwner assigns a standby pthread without waiting for it to enter +// Go or claim the execution-domain handoff. Zero means queued, one means the +// cache was empty, and every other value is an invariant failure. The exact +// thread/token/slot request must subsequently be canceled or released. +// +//go:linkname RequestReuseOwner C.__llgo_coro_fleet_owner_request_reuse_v1 +func RequestReuseOwner(thread *pthread.Thread, token *uint32, slot uint32) c.Int + +// CancelReuseOwner withdraws an exact queued RequestReuseOwner. Zero means the +// standby pthread had not begun dispatch and is already back in the cache; one +// means dispatch won the race and the caller must complete the ordinary owner +// release protocol. Every other value is an invariant failure. +// +//go:linkname CancelReuseOwner C.__llgo_coro_fleet_owner_cancel_reuse_v1 +func CancelReuseOwner(thread pthread.Thread, token, slot uint32) c.Int + // OwnerReady completes CreateOwner only after the new raw owner has claimed // its stable scalar directory slot and execution route. // diff --git a/runtime/internal/corofleet/native_factory_c_test.go b/runtime/internal/corofleet/native_factory_c_test.go index 236529bc5a..3bb118cdff 100644 --- a/runtime/internal/corofleet/native_factory_c_test.go +++ b/runtime/internal/corofleet/native_factory_c_test.go @@ -45,7 +45,7 @@ enum { requester_count = 16, }; -static _Atomic uint32_t seen[requester_count + 13]; +static _Atomic uint32_t seen[requester_count + 15]; static _Atomic uint32_t inherited_taint; static _Atomic uint32_t standby_mode; static _Atomic uint32_t terminal_mode; @@ -54,7 +54,7 @@ static uint32_t terminal_token; uint32_t __llgo_coro_native_fleet_owner_v2(uint32_t slot) { sigset_t current; - if (slot == 0 || slot > requester_count + 12 || + if (slot == 0 || slot > requester_count + 14 || pthread_sigmask(SIG_SETMASK, NULL, ¤t) != 0 || __llgo_coro_fleet_owner_ready_v1(slot) != 0) { return 0; @@ -165,6 +165,61 @@ int main(void) { return 24; } + const uint32_t cancel_slot = requester_count + 13; + int canceled = 0; + for (uint32_t attempt = 0; attempt < 1024 && !canceled; attempt++) { + pthread_t requested = (pthread_t)0; + uint32_t requested_token = 0; + uint32_t before = atomic_load_explicit( + &seen[cancel_slot], memory_order_relaxed); + if (__llgo_coro_fleet_owner_request_reuse_v1( + &requested, &requested_token, cancel_slot) != 0 || + requested == (pthread_t)0 || requested_token == 0) { + return 37; + } + int cancel_result = __llgo_coro_fleet_owner_cancel_reuse_v1( + requested, requested_token, cancel_slot); + if (cancel_result == 0) { + if (atomic_load_explicit( + &seen[cancel_slot], memory_order_relaxed) != before) { + return 38; + } + canceled = 1; + } else if (cancel_result == 1) { + while (atomic_load_explicit( + &seen[cancel_slot], memory_order_relaxed) == before) { + (void)sched_yield(); + } + if (__llgo_coro_fleet_owner_release_v1( + requested, requested_token, cancel_slot) != 0) { + return 39; + } + } else { + return 40; + } + } + if (!canceled) { + return 41; + } + + const uint32_t dispatch_slot = requester_count + 14; + pthread_t requested = (pthread_t)0; + uint32_t requested_token = 0; + if (__llgo_coro_fleet_owner_request_reuse_v1( + &requested, &requested_token, dispatch_slot) != 0) { + return 42; + } + while (atomic_load_explicit( + &seen[dispatch_slot], memory_order_relaxed) == 0) { + (void)sched_yield(); + } + if (__llgo_coro_fleet_owner_cancel_reuse_v1( + requested, requested_token, dispatch_slot) != 1 || + __llgo_coro_fleet_owner_release_v1( + requested, requested_token, dispatch_slot) != 0) { + return 43; + } + pthread_t overflow_owners[8]; uint32_t overflow_tokens[8]; for (uint32_t index = 0; index < 8; index++) { diff --git a/runtime/internal/runtime/coro_native_m_owner_llgo.go b/runtime/internal/runtime/coro_native_m_owner_llgo.go index 09499a753e..90d8f9df96 100644 --- a/runtime/internal/runtime/coro_native_m_owner_llgo.go +++ b/runtime/internal/runtime/coro_native_m_owner_llgo.go @@ -221,6 +221,31 @@ func coroNativeMStartPhysicalOwnerV1( return false } +// coroNativeMRequestPhysicalOwnerV1 publishes an already-cached clean M +// without waiting for its C-to-Go dispatch or scheduler-domain claim. queued +// distinguishes that cancelable path from the one-time synchronous creation +// fallback which seeds the standby cache. +func coroNativeMRequestPhysicalOwnerV1( + owner *coroNativeMOwnerV1, + slot uint32, +) (queued, ok bool) { + if owner == nil || slot == 0 || owner.thread != nil || owner.token != 0 { + return false, false + } + switch corofleet.RequestReuseOwner(&owner.thread, &owner.token, slot) { + case 0: + return true, owner.thread != nil && owner.token != 0 + case 1: + owner.thread = nil + owner.token = 0 + return false, coroNativeMStartPhysicalOwnerV1(owner, slot) + default: + owner.thread = nil + owner.token = 0 + return false, false + } +} + func coroNativeMJoinPhysicalOwnerV1(owner *coroNativeMOwnerV1) bool { if owner == nil || owner.thread == nil || owner.token == 0 || corofleet.JoinOwner(owner.thread, owner.token) != 0 { @@ -658,7 +683,7 @@ func coroNativeMClearReplacementStorageV1(owner *coroNativeMOwnerV1) { func coroNativeMRecycleReplacementV1(slot uint32) bool { owner, ok := coroNativeMOwnerForSlotV1(slot) - if !ok || coroNativeMOwnerLifecycleLoadV1(owner) != coroNativeMOwnerReturnedV1 || + if !ok || owner.thread == nil || owner.self == nil || owner.token == 0 || !owner.handle.Valid() || owner.handle.Route > coroNativeFleetDomainCapacityV1 || @@ -668,6 +693,13 @@ func coroNativeMRecycleReplacementV1(slot uint32) bool { owner.resume.Detached() || !owner.handoff.Idle() { return false } + switch coroNativeMOwnerLifecycleLoadV1(owner) { + case coroNativeMOwnerReplacementActiveV1, + coroNativeMOwnerSuccessorActiveV1, + coroNativeMOwnerReturnedV1: + default: + return false + } root, rootOK := coroNativeMOwnerForSlotV1(owner.lineageRootSlot) parent, parentOK := coroNativeMOwnerForSlotV1(owner.parentSlot) if !rootOK || !parentOK || root.lineageRootSlot != owner.lineageRootSlot || @@ -676,12 +708,7 @@ func coroNativeMRecycleReplacementV1(slot uint32) bool { parent.handle != owner.handle || !parent.handoff.Returned(owner.baton) || coroNativeAtomicLoadV1( &coroNativeMDirectoryV1State.active[owner.handle.Route-1], - ) != owner.parentSlot || - !coroNativeMOwnerLifecycleCASV1( - owner, - coroNativeMOwnerReturnedV1, - coroNativeMOwnerPreparingV1, - ) { + ) != owner.parentSlot { return false } released := corofleet.ReleaseOwner( @@ -693,6 +720,22 @@ func coroNativeMRecycleReplacementV1(slot uint32) bool { coroNativeAtomicStoreV1(&owner.lifecycle, uint32(coroNativeMOwnerFailedV1)) return false } + if coroNativeMOwnerLifecycleLoadV1(owner) != coroNativeMOwnerReturnedV1 || + root.lineageRootSlot != owner.lineageRootSlot || + root.baton != owner.baton || root.parentSlot != owner.parentSlot || + coroNativeAtomicLoadV1(&root.lineageSlot) != slot || + !parent.handoff.Returned(owner.baton) || + coroNativeAtomicLoadV1( + &coroNativeMDirectoryV1State.active[owner.handle.Route-1], + ) != owner.parentSlot || + !coroNativeMOwnerLifecycleCASV1( + owner, + coroNativeMOwnerReturnedV1, + coroNativeMOwnerPreparingV1, + ) { + coroNativeAtomicStoreV1(&owner.lifecycle, uint32(coroNativeMOwnerFailedV1)) + return false + } if released == 1 && !coroTargetReleasePhysicalThreadV1() { coroNativeAtomicStoreV1(&owner.lifecycle, uint32(coroNativeMOwnerFailedV1)) return false @@ -702,17 +745,18 @@ func coroNativeMRecycleReplacementV1(slot uint32) bool { } // coroNativeMWaitAndRecycleOSThreadSuspendV1 is the original M's blocking -// rendezvous for an ordinary locked Yield/Park handoff. ReleaseOwner waits on -// corofleet's existing condition variable while the exact child is still in -// Go; there is no scheduler busy loop. This staged protocol admits only the -// original replacement record, not a retirement successor lineage. +// rendezvous for an ordinary locked Yield/Park handoff or for an asynchronously +// dispatched replacement whose handoff was revoked before Claim. ReleaseOwner +// waits on corofleet's existing condition variable while the exact child is +// still in Go; there is no scheduler busy loop. This staged protocol admits +// only the original replacement record, not a retirement successor lineage. func coroNativeMWaitAndRecycleOSThreadSuspendV1( slot uint32, owner, parent *coroNativeMOwnerV1, ) bool { resolved, ownerOK := coroNativeMOwnerForSlotV1(slot) if !ownerOK || resolved != owner || owner == nil || parent == nil || - owner.thread == nil || owner.self == nil || owner.token == 0 || + owner.thread == nil || owner.token == 0 || !owner.handle.Valid() || owner.handle.Route > coroNativeFleetDomainCapacityV1 || !owner.baton.Valid() || owner.parentSlot == 0 || @@ -725,7 +769,9 @@ func coroNativeMWaitAndRecycleOSThreadSuspendV1( return false } switch coroNativeMOwnerLifecycleLoadV1(owner) { - case coroNativeMOwnerReplacementActiveV1, coroNativeMOwnerReturnedV1: + case coroNativeMOwnerReplacementPublishedV1, + coroNativeMOwnerReplacementActiveV1, + coroNativeMOwnerReturnedV1: default: return false } @@ -741,7 +787,8 @@ func coroNativeMWaitAndRecycleOSThreadSuspendV1( coroNativeAtomicStoreV1(&owner.lifecycle, uint32(coroNativeMOwnerFailedV1)) return false } - if coroNativeMOwnerLifecycleLoadV1(owner) != coroNativeMOwnerReturnedV1 || + if owner.self == nil || + coroNativeMOwnerLifecycleLoadV1(owner) != coroNativeMOwnerReturnedV1 || owner.lineageRootSlot != slot || coroNativeAtomicLoadV1(&owner.lineageSlot) != slot || !parent.handoff.Returned(owner.baton) || @@ -798,18 +845,37 @@ func coroNativeMClaimReplacementV1( } released, releasedOK := parent.handoff.Released() if !releasedOK { - if !parent.handoff.Returned(owner.baton) { + if !parent.handoff.Returned(owner.baton) || + !coroNativeMOwnerLifecycleCASV1( + owner, + coroNativeMOwnerReplacementPublishedV1, + coroNativeMOwnerReturnedV1, + ) { return nil, nil, nil, false, false } - coroNativeAtomicStoreV1(&owner.lifecycle, uint32(coroNativeMOwnerReturnedV1)) return owner, parent, domain, false, true } - if released != owner.baton || !parent.handoff.Claim(owner.baton) || - !coroNativeAtomicCASV1( - &coroNativeMDirectoryV1State.active[owner.handle.Route-1], - owner.parentSlot, - slot, - ) || !coroNativeMOwnerLifecycleCASV1( + if released != owner.baton { + return nil, nil, nil, false, false + } + if !parent.handoff.Claim(owner.baton) { + // Released was an exact snapshot, not a lease. RequestReturn may win + // before Claim's CAS; that is the normal asynchronous-cancel outcome. + if parent.handoff.Returned(owner.baton) && + coroNativeMOwnerLifecycleCASV1( + owner, + coroNativeMOwnerReplacementPublishedV1, + coroNativeMOwnerReturnedV1, + ) { + return owner, parent, domain, false, true + } + return nil, nil, nil, false, false + } + if !coroNativeAtomicCASV1( + &coroNativeMDirectoryV1State.active[owner.handle.Route-1], + owner.parentSlot, + slot, + ) || !coroNativeMOwnerLifecycleCASV1( owner, coroNativeMOwnerReplacementPublishedV1, coroNativeMOwnerReplacementActiveV1, @@ -867,8 +933,14 @@ func coroNativeMReplacementLineageOwnerV1( owner, ownerOK := coroNativeMOwnerForSlotV1(slot) if !ownerOK || owner.handle != root.handle || owner.baton != baton || owner.parentSlot != root.parentSlot || - owner.lineageRootSlot != rootSlot || - coroNativeMOwnerLifecycleLoadV1(owner) != coroNativeMOwnerReturnedV1 { + owner.lineageRootSlot != rootSlot { + return 0, nil, false + } + switch coroNativeMOwnerLifecycleLoadV1(owner) { + case coroNativeMOwnerReplacementActiveV1, + coroNativeMOwnerSuccessorActiveV1, + coroNativeMOwnerReturnedV1: + default: return 0, nil, false } return slot, owner, true diff --git a/runtime/internal/runtime/coro_native_replacement_owner_llgo.go b/runtime/internal/runtime/coro_native_replacement_owner_llgo.go index 36b971db0f..e7a4a10ae4 100644 --- a/runtime/internal/runtime/coro_native_replacement_owner_llgo.go +++ b/runtime/internal/runtime/coro_native_replacement_owner_llgo.go @@ -440,16 +440,17 @@ func coroNativeMRunReplacementOwnerV1(slot uint32) bool { "native replacement M claim failed", ) } - if !claimed { - return coroNativeFleetPhysicalOwnerFailV1( - "native replacement M claim was revoked before startup", - ) - } if corofleet.OwnerReady(slot) != 0 { return coroNativeFleetPhysicalOwnerFailV1( "native replacement M startup acknowledgement failed", ) } + if !claimed { + // RequestReturn won before this clean M reached Claim. OwnerReady still + // acknowledges the physical dispatch so ReleaseOwner can return the + // exact pthread to standby; no scheduler domain was touched. + return true + } return coroNativeMRunClaimedReplacementOwnerV1( slot, owner, diff --git a/runtime/internal/runtime/coro_os_thread_foreign_llgo.go b/runtime/internal/runtime/coro_os_thread_foreign_llgo.go index a86762401a..5d94b69059 100644 --- a/runtime/internal/runtime/coro_os_thread_foreign_llgo.go +++ b/runtime/internal/runtime/coro_os_thread_foreign_llgo.go @@ -41,12 +41,14 @@ type coroNativeForeignBoundaryV1 struct { domain *coroNativeFleetDomainV1 replacement *coroNativeMOwnerV1 - parentSlot uint32 - replacementSlot uint32 - ownerEpoch uint32 - baton coro.ExecutionDomainHandoffHandle - callbackAcquired bool - active bool + parentSlot uint32 + replacementSlot uint32 + ownerEpoch uint32 + baton coro.ExecutionDomainHandoffHandle + replacementQueued bool + replacementSkipped bool + callbackAcquired bool + active bool } var ( @@ -79,7 +81,8 @@ func (boundary *coroNativeForeignBoundaryV1) startReplacementV1( if boundary == nil || !boundary.active || boundary.driver == nil || boundary.parent == nil || boundary.domain == nil || boundary.replacement != nil || boundary.replacementSlot != 0 || - boundary.baton.Valid() { + boundary.baton.Valid() || boundary.replacementQueued || + boundary.replacementSkipped { return false } baton, begun := boundary.parent.handoff.Begin(boundary.ownerEpoch) @@ -99,12 +102,15 @@ func (boundary *coroNativeForeignBoundaryV1) startReplacementV1( return false } if releaseManaged && !coroTargetReleaseManagedExecutionV1(boundary.driver) { - _ = boundary.parent.handoff.RequestReturn(baton) - _ = boundary.parent.handoff.Complete(baton) - _ = coroNativeMReleaseUnstartedReplacementV1(slot) + // Release may have already dropped the quota before a required waiter + // doorbell failed. Its boolean result therefore cannot authorize restoring + // the detached resume or releasing the handoff as though the lease were + // still held. Fail closed instead of creating two physical owners for one P. + coroRuntimeAbort("native direct foreign execution quota release failed") return false } - if !coroNativeMStartPhysicalOwnerV1(replacement, slot) { + queued, started := coroNativeMRequestPhysicalOwnerV1(replacement, slot) + if !started { replacement.thread = nil replacement.token = 0 rollback := boundary.parent.handoff.RequestReturn(baton) == @@ -121,18 +127,21 @@ func (boundary *coroNativeForeignBoundaryV1) startReplacementV1( boundary.replacement = replacement boundary.replacementSlot = slot boundary.baton = baton + boundary.replacementQueued = queued return true } func (boundary *coroNativeForeignBoundaryV1) beginV1( task *coro.G, mode coro.ExecutorResumeHandoffMode, + lazyCompensation bool, ) bool { if boundary == nil || boundary.active || boundary.driver != nil || boundary.task != nil || boundary.parent != nil || boundary.domain != nil || boundary.replacement != nil || boundary.parentSlot != 0 || boundary.replacementSlot != 0 || boundary.ownerEpoch != 0 || boundary.baton.Valid() || + boundary.replacementQueued || boundary.replacementSkipped || boundary.callbackAcquired { return false } @@ -150,6 +159,32 @@ func (boundary *coroNativeForeignBoundaryV1) beginV1( boundary.parentSlot = parentSlot boundary.ownerEpoch = ownerEpoch boundary.active = true + if lazyCompensation { + required, demandOK := + coro.ExecutorResumeHandoffCompensationRequired(&boundary.resume) + if demandOK && !required { + if !coroTargetReleaseManagedExecutionV1(boundary.driver) { + // See startReplacementV1: a false result does not prove that the + // quota release itself failed before publication. + coroRuntimeAbort("native direct lazy execution quota release failed") + return false + } + boundary.replacementSkipped = true + return true + } + if !demandOK || !required { + restored := coro.RestoreExecutorResume(&boundary.resume) + boundary.driver = nil + boundary.task = nil + boundary.parent = nil + boundary.domain = nil + boundary.parentSlot = 0 + boundary.ownerEpoch = 0 + boundary.active = false + _ = restored + return false + } + } if boundary.startReplacementV1(true) { return true } @@ -169,22 +204,99 @@ func (boundary *coroNativeForeignBoundaryV1) reclaimReplacementV1() bool { if boundary == nil || !boundary.active || boundary.driver == nil || boundary.parent == nil || boundary.domain == nil || boundary.replacement == nil || boundary.replacementSlot == 0 || - !boundary.baton.Valid() { + !boundary.baton.Valid() || boundary.replacementSkipped { return false } + if boundary.replacementQueued { + released, stillQueued := boundary.parent.handoff.Released() + if stillQueued && released != boundary.baton { + coroRuntimeAbort("native direct queued replacement handoff mismatch") + } + if stillQueued { + switch corofleet.CancelReuseOwner( + boundary.replacement.thread, + boundary.replacement.token, + boundary.replacementSlot, + ) { + case 0: + boundary.replacement.thread = nil + boundary.replacement.token = 0 + withdrawn := boundary.parent.handoff.RequestReturn(boundary.baton) == + coro.ExecutionDomainHandoffReturnUnclaimed && + boundary.parent.handoff.Complete(boundary.baton) && + coroNativeMReleaseUnstartedReplacementV1(boundary.replacementSlot) + if !withdrawn { + coroRuntimeAbort("native direct queued replacement withdrawal failed") + } + boundary.replacement = nil + boundary.replacementSlot = 0 + boundary.baton = coro.ExecutionDomainHandoffHandle{} + boundary.replacementQueued = false + return true + case 1: + // Dispatch won between Released and cancellation. The handoff race + // below decides whether it claimed or only acknowledges return. + default: + // Claim and an immediate retirement may consume both the Released + // phase and the original C token between our snapshot and cancel. + // Only a still-live exact Released publication makes the failed + // cancellation an invariant error. + current, releasedNow := boundary.parent.handoff.Released() + if releasedNow && current == boundary.baton { + coroRuntimeAbort("native direct replacement cancel state invalid") + } + } + } + // Once Claim has consumed Released, the C record may already have + // retired into a clean successor and its original token is no longer a + // cancel capability. The handoff generation is then the sole authority. + boundary.replacementQueued = false + } returnResult := boundary.parent.handoff.RequestReturn(boundary.baton) + if returnResult == coro.ExecutionDomainHandoffReturnUnclaimed { + if !coroNativeMWaitAndRecycleOSThreadSuspendV1( + boundary.replacementSlot, + boundary.replacement, + boundary.parent, + ) { + coroRuntimeAbort("native direct revoked replacement recycle failed") + } + if !boundary.parent.handoff.Complete(boundary.baton) { + coroRuntimeAbort("native direct revoked handoff completion failed") + } + boundary.replacement = nil + boundary.replacementSlot = 0 + boundary.baton = coro.ExecutionDomainHandoffHandle{} + boundary.replacementQueued = false + return true + } + returnRequested := returnResult == coro.ExecutionDomainHandoffReturnClaimed + alreadyReturned := false + if returnResult == coro.ExecutionDomainHandoffReturnInvalid { + returnRequested = boundary.parent.handoff.ReturnRequested(boundary.baton) + alreadyReturned = boundary.parent.handoff.Returned(boundary.baton) + } request := coro.ExecutorRequestInvalid - if returnResult == coro.ExecutionDomainHandoffReturnClaimed { + if returnRequested { // The return baton is the durable fact. The executor request is its // preemption transport: a replacement may currently be inside an // unbounded managed resume and cannot observe a doorbell until it first // reaches the compiler safepoint gate. request = coroNativeFleetV1State.fleet.RequestExecutor(boundary.domain.handle) } - ringOK := returnResult == coro.ExecutionDomainHandoffReturnClaimed && + ringOK := returnRequested && coro.ExecutorRequestAccepted(request) && boundary.domain.doorbell.Ring() - for ringOK && !boundary.parent.handoff.Returned(boundary.baton) { + if !returnRequested && !alreadyReturned { + coroRuntimeAbort("native direct replacement return state invalid") + } + if returnRequested && !coro.ExecutorRequestAccepted(request) { + coroRuntimeAbort("native direct replacement preemption request failed") + } + if returnRequested && !ringOK { + coroRuntimeAbort("native direct replacement doorbell failed") + } + for returnRequested && ringOK && !boundary.parent.handoff.Returned(boundary.baton) { if corofleet.Yield() != 0 { ringOK = false } @@ -196,18 +308,25 @@ func (boundary *coroNativeForeignBoundaryV1) reclaimReplacementV1() bool { boundary.baton, ) returned = returned && returnedOwner.thread != nil && - coroNativeMOwnerLifecycleLoadV1(returnedOwner) == coroNativeMOwnerReturnedV1 && coroNativeAtomicLoadV1( &coroNativeMDirectoryV1State.active[boundary.domain.handle.Route-1], ) == boundary.parentSlot - if !ringOK || !returned || - !coroNativeMRecycleReplacementV1(returnedSlot) || - !boundary.parent.handoff.Complete(boundary.baton) { - return false + if returnRequested && !ringOK { + coroRuntimeAbort("native direct replacement return wait failed") + } + if !returned { + coroRuntimeAbort("native direct replacement lineage return failed") + } + if !coroNativeMRecycleReplacementV1(returnedSlot) { + coroRuntimeAbort("native direct claimed replacement recycle failed") + } + if !boundary.parent.handoff.Complete(boundary.baton) { + coroRuntimeAbort("native direct claimed handoff completion failed") } boundary.replacement = nil boundary.replacementSlot = 0 boundary.baton = coro.ExecutionDomainHandoffHandle{} + boundary.replacementQueued = false return true } @@ -217,12 +336,19 @@ func (boundary *coroNativeForeignBoundaryV1) restartReplacementV1() bool { } func (boundary *coroNativeForeignBoundaryV1) finishV1() bool { - if boundary == nil || boundary.callbackAcquired || - !boundary.reclaimReplacementV1() || - !coroTargetReenterManagedExecutionV1(boundary.driver) || - !coro.RestoreExecutorResume(&boundary.resume) { + if boundary == nil || boundary.callbackAcquired { return false } + if !boundary.replacementSkipped && !boundary.reclaimReplacementV1() { + coroRuntimeAbort("native direct foreign replacement reclaim failed") + } + boundary.replacementSkipped = false + if !coroTargetReenterManagedExecutionV1(boundary.driver) { + coroRuntimeAbort("native direct foreign execution quota reentry failed") + } + if !coro.RestoreExecutorResume(&boundary.resume) { + coroRuntimeAbort("native direct foreign resume restore failed") + } boundary.driver = nil boundary.task = nil boundary.parent = nil @@ -290,7 +416,8 @@ func coroNativeForeignReentryRunV1( ) coro.CompletionSnapshot { if boundary == nil || !boundary.active || !boundary.callbackAcquired || boundary.replacement != nil || boundary.replacementSlot != 0 || - boundary.baton.Valid() || child == nil { + boundary.baton.Valid() || boundary.replacementQueued || + boundary.replacementSkipped || child == nil { coroRuntimeAbort("invalid synchronous foreign callback child") } var record coro.ForeignReentryRecord @@ -396,7 +523,7 @@ func __llgo_coro_same_m_foreign_call_v1( coroRuntimeAbort("invalid same-M foreign call") } var boundary coroNativeForeignBoundaryV1 - if !boundary.beginV1(task, coro.ExecutorResumeHandoffSameMForeign) { + if !boundary.beginV1(task, coro.ExecutorResumeHandoffSameMForeign, false) { coroRuntimeAbort("same-M foreign call cannot detach active resume") } previous, installed := coroNativeForeignBoundarySetTLSV1(&boundary) @@ -420,51 +547,39 @@ func __llgo_coro_same_m_foreign_call_v1( } } -// __llgo_coro_os_thread_foreign_call_v1 is the sole same-M blocking foreign -// boundary. The compiler selects it dynamically only while the current G owns -// this P/M island through LockOSThread. All ordinary calls continue through -// the shared any-thread worker pool. This owner detaches the active resume, -// reserves one scalar-slot replacement M, releases its managed-execution -// P lease before creating the replacement thread, and strongly rejoins that -// replacement before restoring the resume. Releasing first is mandatory: -// execution-quota ownership belongs to the route, so a replacement which -// starts while its parent still holds that route is a fail-closed double -// acquire rather than ordinary quota contention. On return, the parent first -// strongly joins the replacement and then reacquires the same P lease before -// restoring the detached LLVM resume, so the enclosing run slice can safely -// continue to retain its logical lease state. -// -//export __llgo_coro_os_thread_foreign_call_v1 -func __llgo_coro_os_thread_foreign_call_v1( - g unsafe.Pointer, +func coroNativeForeignWordCallV1( + task *coro.G, + mode coro.ExecutorResumeHandoffMode, + lazyCompensation bool, function, traceTarget uintptr, argc uint32, a0, a1, a2, a3, a4, a5, a6, a7, a8 uintptr, r1, r2, errno *uintptr, ) uint32 { - task := (*coro.G)(g) if function == 0 || traceTarget == 0 || argc > coroworker.MaxArgs || r1 == nil || r2 == nil || errno == nil || r1 == r2 || r1 == errno || r2 == errno || - !coro.CurrentOSThreadLocked(task) { - coroRuntimeAbort("invalid locked-thread foreign call") + (mode != coro.ExecutorResumeHandoffLockedForeign && + mode != coro.ExecutorResumeHandoffSameMForeign) || + lazyCompensation && mode != coro.ExecutorResumeHandoffSameMForeign { + coroRuntimeAbort("invalid native direct foreign call") } var boundary coroNativeForeignBoundaryV1 - if !boundary.beginV1(task, coro.ExecutorResumeHandoffLockedForeign) { - coroRuntimeAbort("locked-thread foreign call cannot detach active resume") + if !boundary.beginV1(task, mode, lazyCompensation) { + coroRuntimeAbort("native direct foreign call cannot detach active resume") } args := [coroworker.MaxArgs]uintptr{a0, a1, a2, a3, a4, a5, a6, a7, a8} var result coroworker.Result callOK := coroworker.Call(function, traceTarget, argc, &args, &result) if !boundary.finishV1() { - coroRuntimeAbort("locked-thread foreign call cannot reacquire managed execution") + coroRuntimeAbort("native direct foreign call cannot reacquire managed execution") } if !callOK { - coroRuntimeAbort("locked-thread foreign call failed") + coroRuntimeAbort("native direct foreign call failed") } if result.Fault != coroworker.FaultNone { if !StoreCoroWorkerFaultPCs(task, result.FaultPC, result.FaultTarget) { - coroRuntimeAbort("locked-thread foreign fault has no traceback identity") + coroRuntimeAbort("native direct foreign fault has no traceback identity") } switch result.Fault { case coroworker.FaultMemory: @@ -472,9 +587,61 @@ func __llgo_coro_os_thread_foreign_call_v1( case coroworker.FaultDivide: return coroWorkerResumeFaultDivideV1 default: - coroRuntimeAbort("locked-thread foreign call returned unknown fault") + coroRuntimeAbort("native direct foreign call returned unknown fault") } } *r1, *r2, *errno = result.R1, result.R2, result.Errno return coroWorkerResumeSuccessV1 } + +// __llgo_coro_native_syscall_call_v1 is the native entersyscall/exitsyscall +// boundary for compiler-certified llgo.syscall calls. The current M performs +// the syscall directly after releasing its execution domain. The detached +// scheduler boundary requests a cached clean M only when independently +// progressing route work already requires compensation; a quick return can +// still cancel that request before dispatch, while a blocking syscall lets a +// dispatch winner claim and service the route. +// +//export __llgo_coro_native_syscall_call_v1 +func __llgo_coro_native_syscall_call_v1( + g unsafe.Pointer, + function, traceTarget uintptr, + argc uint32, + a0, a1, a2, a3, a4, a5, a6, a7, a8 uintptr, + r1, r2, errno *uintptr, +) uint32 { + return coroNativeForeignWordCallV1( + (*coro.G)(g), + coro.ExecutorResumeHandoffSameMForeign, + true, + function, traceTarget, argc, + a0, a1, a2, a3, a4, a5, a6, a7, a8, + r1, r2, errno, + ) +} + +// __llgo_coro_os_thread_foreign_call_v1 is the LockOSThread form of the same +// direct blocking boundary. The dynamic guard preserves the G-to-M contract; +// its compensation and exact return protocol are shared with native syscalls. +// +//export __llgo_coro_os_thread_foreign_call_v1 +func __llgo_coro_os_thread_foreign_call_v1( + g unsafe.Pointer, + function, traceTarget uintptr, + argc uint32, + a0, a1, a2, a3, a4, a5, a6, a7, a8 uintptr, + r1, r2, errno *uintptr, +) uint32 { + task := (*coro.G)(g) + if !coro.CurrentOSThreadLocked(task) { + coroRuntimeAbort("invalid locked-thread foreign call") + } + return coroNativeForeignWordCallV1( + task, + coro.ExecutorResumeHandoffLockedForeign, + false, + function, traceTarget, argc, + a0, a1, a2, a3, a4, a5, a6, a7, a8, + r1, r2, errno, + ) +} diff --git a/runtime/poll_worker_source_test.go b/runtime/poll_worker_source_test.go index 86d88b0f7b..5d1eef9cc6 100644 --- a/runtime/poll_worker_source_test.go +++ b/runtime/poll_worker_source_test.go @@ -536,7 +536,7 @@ func TestRuntimeCoroWorkerBlockingCallHasOnlyGuardedSameMEntrance(t *testing.T) entrance := readRuntimePollFile(t, runtimeCoroOSThreadForeignSource) for _, required := range []string{ - "sole same-M blocking foreign", + "native entersyscall/exitsyscall", "!coro.CurrentOSThreadLocked(task)", "type coroNativeForeignBoundaryV1 struct", "coroNativeForeignBoundaryTLSV1 tls.StaticHandle[*coroNativeForeignBoundaryV1]", @@ -548,8 +548,13 @@ func TestRuntimeCoroWorkerBlockingCallHasOnlyGuardedSameMEntrance(t *testing.T) "boundary.parent.handoff.Begin(boundary.ownerEpoch)", "coroNativeMAllocateReplacementV1(", "coroTargetReleaseManagedExecutionV1(boundary.driver)", - "coroNativeMStartPhysicalOwnerV1(replacement, slot)", - "boundary.beginV1(task, coro.ExecutorResumeHandoffLockedForeign)", + "coroNativeMRequestPhysicalOwnerV1(replacement, slot)", + "corofleet.CancelReuseOwner(", + "if !boundary.beginV1(task, mode, lazyCompensation)", + "coro.ExecutorResumeHandoffCompensationRequired(&boundary.resume)", + "//export __llgo_coro_native_syscall_call_v1", + "coro.ExecutorResumeHandoffSameMForeign", + "coro.ExecutorResumeHandoffLockedForeign", "callOK := coroworker.Call(function, traceTarget, argc, &args, &result)", "boundary.parent.handoff.RequestReturn(boundary.baton)", "coroNativeMReplacementLineageOwnerV1(", @@ -564,7 +569,7 @@ func TestRuntimeCoroWorkerBlockingCallHasOnlyGuardedSameMEntrance(t *testing.T) "//export __llgo_coro_foreign_reentry_run_v1", "//export __llgo_coro_foreign_reentry_failure_v1", "//export __llgo_coro_same_m_foreign_call_v1", - "boundary.beginV1(task, coro.ExecutorResumeHandoffSameMForeign)", + "boundary.beginV1(task, coro.ExecutorResumeHandoffSameMForeign, false)", "callOK := coroworker.Call(thunk, 0, 1, &args, &result)", } { if !strings.Contains(entrance, required) { @@ -594,17 +599,28 @@ func TestRuntimeCoroWorkerBlockingCallHasOnlyGuardedSameMEntrance(t *testing.T) detach := strings.Index(entrance, "coro.DetachExecutorResume(") start := strings.Index(entrance, "if boundary.startReplacementV1(true)") leave := strings.Index(entrance, "coroTargetReleaseManagedExecutionV1(boundary.driver)") - create := strings.Index(entrance, "coroNativeMStartPhysicalOwnerV1(replacement, slot)") - begin := strings.LastIndex(entrance, "boundary.beginV1(task, coro.ExecutorResumeHandoffLockedForeign)") - call := strings.Index(entrance, "callOK := coroworker.Call(function, traceTarget, argc, &args, &result)") + create := strings.Index(entrance, "coroNativeMRequestPhysicalOwnerV1(replacement, slot)") + helper := strings.Index(entrance, "func coroNativeForeignWordCallV1(") + begin, call := -1, -1 + if helper >= 0 { + begin = strings.Index(entrance[helper:], "if !boundary.beginV1(task, mode, lazyCompensation)") + call = strings.Index(entrance[helper:], "callOK := coroworker.Call(function, traceTarget, argc, &args, &result)") + if begin >= 0 { + begin += helper + } + if call >= 0 { + call += helper + } + } finish := strings.LastIndex(entrance, "boundary.finishV1()") + cancel := strings.Index(entrance, "corofleet.CancelReuseOwner(") request := strings.LastIndex(entrance, "boundary.parent.handoff.RequestReturn(boundary.baton)") recycle := strings.Index(entrance, "coroNativeMRecycleReplacementV1(returnedSlot)") reenter := strings.LastIndex(entrance, "coroTargetReenterManagedExecutionV1(boundary.driver)") restore := strings.LastIndex(entrance, "coro.RestoreExecutorResume(&boundary.resume)") - if detach < 0 || start <= detach || create <= leave || + if detach < 0 || start <= detach || create <= leave || cancel <= create || request < 0 || recycle <= request || reenter <= recycle || restore <= reenter || - begin < 0 || call <= begin || finish <= call { + helper < 0 || begin < helper || call <= begin || finish <= call { t.Errorf("%s does not bracket same-M C with detach/release/create/return/recycle/restore", runtimeCoroOSThreadForeignSource) } quota := readRuntimePollFile(t, "internal/runtime/coro_execution_quota_native_llgo.go")