diff --git a/cl/coro_call_site_plan.go b/cl/coro_call_site_plan.go index 67fe494871..5bb9af7ca9 100644 --- a/cl/coro_call_site_plan.go +++ b/cl/coro_call_site_plan.go @@ -35,6 +35,103 @@ func isCoroPythonIntrinsicOpcode(opcode int) bool { } } +type coroCallArgumentLifetimeUniverse interface { + canonicalAlias(*ssa.Function) *ssa.Function + sortedUseOwners(*ssa.Function) []*preparedEmissionPackage + functionABIContext(*ssa.Function, *preparedEmissionPackage) (*context, error) + freezeCoroCallableShape(*ssa.Function) (coroCallableFrozenShape, error) +} + +func classifyCoroCallArgumentsBorrowedUntilCompletion( + universe coroCallArgumentLifetimeUniverse, + ctx *context, + caller *ssa.Function, + call ssa.CallInstruction, +) (bool, error) { + direct, ok := call.(*ssa.Call) + if !ok || direct.Common() == nil || direct.Common().IsInvoke() || + direct.Common().Method != nil || ctx == nil || ctx.prog == nil || caller == nil { + return false, nil + } + callee := universe.canonicalAlias(direct.Common().StaticCallee()) + if callee == nil || callee.Pkg == nil || caller.Pkg == nil || + callee.Pkg != caller.Pkg || callee.Pkg != ctx.goPkg { + return false, nil + } + // A source stub may remain visible to x/tools SSA even though patch + // selection or reachability omitted its physical declaration. Such a call + // has no frozen ABI owner and therefore cannot receive an occurrence-level + // lifetime certificate; it must not make ProgramIR construction fail either. + if len(universe.sortedUseOwners(callee)) == 0 { + return false, nil + } + shape, err := universe.freezeCoroCallableShape(callee) + if err != nil { + return false, fmt.Errorf("freeze C argument lifetime for %q: %w", callee.Name(), err) + } + if shape.kind != cFunc { + return false, nil + } + parsed, present, err := coroCallableContractCertificateFor(callee) + if err != nil { + return false, fmt.Errorf("freeze C argument lifetime for %q: %w", callee.Name(), err) + } + contract := defaultCoroForeignDeclarationContract().Contract + if present { + if parsed.Scope != coroCallableContractScopeDeclaration { + return false, nil + } + contract = parsed.Contract + } + switch contract.Memory { + case coro.MemoryByValue, coro.MemoryBorrowUntilReturn: + return true, nil + case coro.MemoryBorrowUntilComplete: + return contract.Progress == coro.ProgressExecutorSafe || + contract.Progress == coro.ProgressMayBlock, nil + default: + return false, nil + } +} + +func coroCallArgumentsBorrowedUntilCompletion( + universe coroCallArgumentLifetimeUniverse, + call *ssa.Call, +) (bool, error) { + if universe == nil || call == nil || call.Parent() == nil { + return false, nil + } + caller := universe.canonicalAlias(call.Parent()) + if caller == nil || caller != call.Parent() { + return false, nil + } + owners := universe.sortedUseOwners(caller) + if len(owners) == 0 { + return false, nil + } + borrowed, classified := false, false + for _, owner := range owners { + ctx, err := universe.functionABIContext(caller, owner) + if err != nil { + return false, err + } + candidate, err := classifyCoroCallArgumentsBorrowedUntilCompletion(universe, ctx, caller, call) + if err != nil { + return false, err + } + if classified && candidate != borrowed { + return false, fmt.Errorf( + "call %q has owner-dependent C argument lifetime", call.String(), + ) + } + borrowed, classified = candidate, true + } + if !classified { + return false, nil + } + return borrowed, nil +} + // freezeCallSites is the final pre-SSAPlan ProgramIR builder stage. Runtime // helper closure, patch redirects, physical identities, and worker // certificates must already be immutable. The stage validates raw SSA exactly @@ -392,6 +489,11 @@ func (ir *coroProgramIR) freezeCallSites(u *EmissionUniverse) error { case intrinsic && classifyErr == nil && semantics.ElidesManagedCall() && plan.StaticSpawnTarget == nil: plan.Elision = CoroCallElidedIntrinsic } + if classifyErr == nil { + direct, _ := call.(*ssa.Call) + plan.ArgumentsBorrowedUntilCompletion, classifyErr = + coroCallArgumentsBorrowedUntilCompletion(u, direct) + } if plan.ElidesCall() && intrinsic && classifyErr == nil && workerCertified { if workerCertificate.ID == "" { classifyErr = fmt.Errorf("freeze intrinsic elision certificate: certified call has an empty identity") diff --git a/cl/coro_frame_retention_test.go b/cl/coro_frame_retention_test.go index d400991925..dd8a092e47 100644 --- a/cl/coro_frame_retention_test.go +++ b/cl/coro_frame_retention_test.go @@ -82,6 +82,26 @@ func Root(limit uint32) bool { } ` +const coroForeignBorrowedAllocationFixture = `package foo + +import "unsafe" + +type result struct { + value uintptr +} + +//go:linkname call C.coro_borrowed_allocation_fixture +func call(resultAddress uintptr) bool + +func Root() uintptr { + var value result + if !call(uintptr(unsafe.Pointer(&value))) { + return 0 + } + return value.value +} +` + func TestCoroBorrowedAllocationUsesPhysicalLocalStorage(t *testing.T) { prog, ssaPkg, files, universe, plan := prepareCoroPreemptTestPlan( t, @@ -154,6 +174,100 @@ func TestCoroBorrowedAllocationUsesPhysicalLocalStorage(t *testing.T) { } } +func TestCoroForeignBorrowContractUsesPhysicalLocalStorage(t *testing.T) { + prog, ssaPkg, _, universe, plan := prepareCoroPreemptTestPlan( + t, + coroForeignBorrowedAllocationFixture, + []coroRootFactoryTestRoot{{name: "Root", demand: coro.AsyncDemand}}, + nil, + -1, + ) + defer prog.Dispose() + root := ssaPkg.Func("Root") + var foreignCall *ssa.Call + for _, block := range root.Blocks { + for _, instruction := range block.Instrs { + call, ok := instruction.(*ssa.Call) + if ok && call.Common() != nil && call.Common().StaticCallee() != nil && + call.Common().StaticCallee().Name() == "call" { + foreignCall = call + } + } + } + if foreignCall == nil { + t.Fatal("foreign-borrow fixture has no direct C call") + } + site, frozen, err := universe.CoroCallSitePlan(foreignCall) + if err != nil || !frozen || !site.ArgumentsBorrowedUntilCompletion { + t.Fatalf("foreign-borrow call SitePlan = %+v, frozen=%t, err=%v, samePkg=%t", + site, frozen, err, foreignCall.Common().StaticCallee().Pkg == ssaPkg) + } + audit, err := newCoroPhysicalPureSSAAudit(universe, plan, root, "") + if err != nil { + t.Fatal(err) + } + proof := audit.currentFrameRetentionProof() + allocations := coroFrameRetentionHeapAllocs(root) + if len(allocations) != 1 || len(proof.borrowedAllocations) != 1 || + len(proof.managedHeapAllocations) != 0 { + t.Fatalf("foreign-borrow fixture SSA/borrowed/managed allocations = %d/%d/%d, want 1/1/0", + len(allocations), len(proof.borrowedAllocations), len(proof.managedHeapAllocations)) + } + borrow, retained := proof.borrowedAllocations[allocations[0]] + if !retained || borrow.FunctionsVisited == 0 { + t.Fatalf("foreign declaration did not retain an exact borrow proof: %+v", borrow) + } + if reason := audit.validateAlloc(allocations[0]); reason != "" { + t.Fatalf("foreign borrowed allocation rejected: %s", reason) + } +} + +func TestCoroForeignRetainedContractRejectsBorrowedStorage(t *testing.T) { + source := strings.Replace( + coroForeignBorrowedAllocationFixture, + "//go:linkname call C.coro_borrowed_allocation_fixture", + "//llgo:coro contract foreign.v1 scope=declaration progress=may-block affinity=any-thread reentry=none memory=retained\n//go:linkname call C.coro_borrowed_allocation_fixture", + 1, + ) + prog, ssaPkg, _, universe, plan := prepareCoroPreemptTestPlan( + t, + source, + []coroRootFactoryTestRoot{{name: "Root", demand: coro.AsyncDemand}}, + nil, + -1, + ) + defer prog.Dispose() + root := ssaPkg.Func("Root") + var foreignCall *ssa.Call + for _, block := range root.Blocks { + for _, instruction := range block.Instrs { + call, ok := instruction.(*ssa.Call) + if ok && call.Common() != nil && call.Common().StaticCallee() != nil && + call.Common().StaticCallee().Name() == "call" { + foreignCall = call + } + } + } + if foreignCall == nil { + t.Fatal("foreign-retained fixture has no direct C call") + } + site, frozen, err := universe.CoroCallSitePlan(foreignCall) + if err != nil || !frozen || site.ArgumentsBorrowedUntilCompletion { + t.Fatalf("foreign-retained call SitePlan = %+v, frozen=%t, err=%v; want conservative lifetime", + site, frozen, err) + } + audit, err := newCoroPhysicalPureSSAAudit(universe, plan, root, "") + if err != nil { + t.Fatal(err) + } + proof := audit.currentFrameRetentionProof() + allocations := coroFrameRetentionHeapAllocs(root) + if len(allocations) != 1 || len(proof.borrowedAllocations) != 0 { + t.Fatalf("foreign-retained fixture SSA/borrowed/managed allocations = %d/%d/%d, want one SSA allocation and no borrow proof", + len(allocations), len(proof.borrowedAllocations), len(proof.managedHeapAllocations)) + } +} + func TestCoroGenericParkStateRetentionIsSourceIndependent(t *testing.T) { for _, symbol := range []string{"__llgo_coro_fixture_prepare", "__llgo_coro_another_source_prepare"} { t.Run(symbol, func(t *testing.T) { diff --git a/cl/emission_runtime_helpers.go b/cl/emission_runtime_helpers.go index ed64275af3..ba067832db 100644 --- a/cl/emission_runtime_helpers.go +++ b/cl/emission_runtime_helpers.go @@ -104,17 +104,20 @@ func planCoroPlainAllocation(ctx *context, allocation *ssa.Alloc) coroPlainAlloc } // coroBorrowedAllocationUniverse is the complete read-only authority needed by -// the interprocedural lifetime proof. Keeping this as a two-method view avoids +// the interprocedural lifetime proof. Keeping this as a narrow view avoids // turning a local allocation classifier into another whole-program plan owner. type coroBorrowedAllocationUniverse interface { + coroCallArgumentLifetimeUniverse Resolve(*ssa.Function) (*ssa.Function, bool) FunctionBackground(*ssa.Function) (llssa.Background, bool, error) } // proveCoroBorrowedAllocation admits only bodies which the frozen emission -// universe will actually compile as managed Go. Source bodies attached to C, -// Python, or LLVM intrinsic declarations are type-checking stubs rather than -// memory-semantics evidence and must never justify local storage. +// universe will actually compile as managed Go, or exact call-site plans which +// prove that a foreign call stops borrowing an argument no later than its +// logical completion. Source bodies attached to C, Python, or LLVM intrinsic +// declarations remain type-checking stubs and never constitute memory- +// semantics evidence themselves. func proveCoroBorrowedAllocation( universe coroBorrowedAllocationUniverse, allocation *ssa.Alloc, @@ -123,6 +126,10 @@ func proveCoroBorrowedAllocation( return coro.SSABorrowedAllocationProof{Allocation: allocation}, false } return coro.ProveSSABorrowedAllocationWithConfig(allocation, coro.SSABorrowedAllocationConfig{ + BorrowedCallArgument: func(call *ssa.Call, _ int) bool { + borrowed, err := coroCallArgumentsBorrowedUntilCompletion(universe, call) + return err == nil && borrowed + }, ResolveCalleeBody: func(function *ssa.Function) (*ssa.Function, bool) { canonical, resolved := universe.Resolve(function) if !resolved || canonical == nil || len(canonical.Blocks) == 0 { diff --git a/cl/emission_universe.go b/cl/emission_universe.go index f4a1cd289f..2f3900d97f 100644 --- a/cl/emission_universe.go +++ b/cl/emission_universe.go @@ -546,6 +546,13 @@ type CoroCallSitePlan struct { // construction and is the only authority consumed by analysis, physical // planning, lowering facts, and codegen. ControlOperation CoroControlOperation + // ArgumentsBorrowedUntilCompletion is an occurrence-level lifetime fact for + // one exact ordinary call to a source-visible C declaration in the caller's + // package. It permits caller-owned storage to live in the native/coroutine + // frame only when the frozen callable contract ends every argument borrow no + // later than this logical call completes. Dynamic, deferred, spawned, + // cross-package, async-completion, and retained-memory calls keep this false. + ArgumentsBorrowedUntilCompletion bool // RawPlainSynchronousIntrinsic proves that this exact direct llgo.syscall // occurrence has the uintptr word ABI accepted by ordinary synchronous // intrinsic lowering. It authorizes only a separately proven raw/plain diff --git a/doc/coro-performance-baseline.md b/doc/coro-performance-baseline.md index 95b4830a68..990ceaa9f7 100644 --- a/doc/coro-performance-baseline.md +++ b/doc/coro-performance-baseline.md @@ -1888,3 +1888,78 @@ file I/O, direct file syscalls, a sole-M blocking pipe whose writer first parks on a real timer, and loopback TCP. Runtime-core tests cover allocation-elided and dynamic destruction, panic/recover compatibility, slow yield, nesting, and the depth-bound scheduler fallback. + +### Escape-free scalar synchronous native-call checkpoint + +The 2026-08-15 follow-up starts at merge +`b48265391044cf8e20f160e550f1abfcf4fa8b98`; the measured implementation is +`9f0470cd1`. Profiling the standard-file fixture showed that every +compiler-certified native syscall created two +collector allocations before entering the kernel: one for the nine-word +argument record and one for the six-word result record. Their addresses crossed +the Go-to-C ABI in `coroworker.Call`, so LLGo's conservative escape decision was +correct; wrapping the pointers in a runtime `noescape` helper did not change the +SSA allocation class. + +The retained implementation replaces that Go-visible pointer ABI with a local +runtime declaration of `__llgo_coro_worker_call_words_v2`. Its nine arguments +and the address of one caller-owned result record cross as scalar words. The C +wrapper owns the argument array and calls the old pointer-taking leaf entirely +inside the native worker island; no Go declaration can select that leaf. + +Scalar encoding alone is not an escape proof: x/tools SSA still conservatively +marks the result record as heap storage because its address reaches a bodyless +C declaration. The compiler now freezes an occurrence-level argument-lifetime +fact and feeds it into the generic interprocedural borrowed-allocation proof. +Only an ordinary static call to a same-package, owner-backed C declaration can +qualify, and its callable contract must end the borrow by return or synchronous +completion. Retained-memory, asynchronous-completion, cross-package, dynamic, +deferred, spawned, ownerless, and unresolved calls fail closed. Unit tests cover +the positive call and an explicit `memory=retained` rejection; no new source +annotation is needed for this runtime leaf. + +A seven-word C struct return was also tested and rejected before retention: +separately compiled Mach-O objects disagreed about the hidden `sret` ABI and +faulted despite a full-LTO fixture appearing to work. Keeping the result address +as an explicit `uintptr` gives the cross-object boundary an all-scalar ABI and +the independent-link E2Es guard it. Fault attribution, native handoff, lazy +compensation, and result/fault interpretation are otherwise unchanged. + +The parent and candidate were built from the same `io_workload` source with +independent caches, full LTO, stripped output, Go 1.26.5, and LLVM 22.1.8 on +Darwin arm64. Nine AB/BA-interleaved 5,000-operation standard-file runs produced +these hardware-counter medians: + +| Metric | fused parent | scalar-ABI candidate | delta | +| --- | ---: | ---: | ---: | +| retired instructions | 1,409,175,860 | 1,256,015,357 | -10.87% | +| cycles | 471,775,229 | 226,316,522 | -52.03% | +| stripped file bytes | 7,014,704 | 7,014,768 | +64 (+0.001%) | +| Mach-O `__text` bytes | 3,137,408 | 3,137,244 | -164 (-0.005%) | + +These are fresh-cache rebuilds of the exact parent and candidate; earlier +temporary binaries were discarded after their runtime archives were found not +to be comparable. Retired-instruction ranges were +1,276,319,938--1,417,813,058 for the parent and +1,252,119,179--1,257,990,658 for the candidate. Candidate cycles were tightly +grouped at 221,952,862--229,185,984, while parent cycles ranged from +257,958,534 to 515,229,856 as its allocation and collection work varied. The +cycle reduction is therefore a strong same-host observation, not a portable +latency claim. + +An independent 15-process AB/BA wall-time gate measured the standard-file +fixture at 42.658 ms [41.355, 44.624] versus 53.021 ms [44.064, 53.907] for +the parent (-19.54%) and 7.740 ms [7.644, 8.172] for Go (5.51x Go). Direct +`syscall` file I/O improved from 21.684 ms [21.102, 22.030] to 18.890 ms +[18.223, 19.222] (-12.88%, 2.44x Go). Loopback TCP remained within noise at +19.741 ms versus 19.729 ms for the parent because it already uses the +nonblocking poll/event path; Go measured 9.561 ms in the same run. + +Disassembly of `coroNativeForeignWordCallV1` changes two runtime GC allocation +calls into one stack result area plus `__llgo_coro_worker_call_words_v2`. The +reentry-capable same-M path changes three allocations to one; its remaining +104-byte boundary is temporarily published in TLS for C-to-Go callbacks and is +not on the file/syscall path. Small standard-file, direct-syscall, sole-M +blocking-pipe/timer, and loopback-TCP execution gates pass with +`GOMAXPROCS=1`, as do separately linked same-M callback, locked-thread +compensation, deferred replacement, and worker-capability E2Es. diff --git a/internal/build/coro_native_target_plan_test.go b/internal/build/coro_native_target_plan_test.go index c7cdb6c899..2bde50bf9a 100644 --- a/internal/build/coro_native_target_plan_test.go +++ b/internal/build/coro_native_target_plan_test.go @@ -633,7 +633,7 @@ func TestRealNativeCoroTargetIsTrustedPlainSchedulerIsland(t *testing.T) { "__llgo_coro_worker_queue_init_v1": false, "__llgo_coro_worker_queue_stop_v1": false, "__llgo_coro_worker_queue_destroy_after_join_v1": false, - "__llgo_coro_worker_call_v1": false, + "__llgo_coro_worker_call_words_v2": false, "__llgo_coro_worker_queue_reserve_v2": false, "__llgo_coro_worker_queue_cancel_reservation_v2": false, "__llgo_coro_worker_queue_submit_reserved_v4": false, diff --git a/internal/coro/ssa_borrowed_allocation.go b/internal/coro/ssa_borrowed_allocation.go index f594d4a7fb..886a861209 100644 --- a/internal/coro/ssa_borrowed_allocation.go +++ b/internal/coro/ssa_borrowed_allocation.go @@ -43,8 +43,12 @@ type SSABorrowedAllocationProof struct { // intrinsic, foreign symbol, or other physical implementation must reject that // body (or resolve it to an exact managed-Go definition); otherwise an inert // declaration stub can incorrectly prove that an argument is not retained. +// BorrowedCallArgument is the separate extension point for a frontend-owned, +// exact call certificate which proves that one argument is not retained beyond +// the logical call. Dynamic calls never reach it. type SSABorrowedAllocationConfig struct { - ResolveCalleeBody func(*ssa.Function) (*ssa.Function, bool) + ResolveCalleeBody func(*ssa.Function) (*ssa.Function, bool) + BorrowedCallArgument func(*ssa.Call, int) bool } type ssaBorrowParameterKey struct { @@ -280,19 +284,25 @@ func (analyzer *ssaBorrowedAllocationAnalyzer) proveCallArgument(call *ssa.Call, } return true } - if analyzer.config.ResolveCalleeBody != nil { - var resolved bool - callee, resolved = analyzer.config.ResolveCalleeBody(callee) - if !resolved || callee == nil { - return false - } - } found := false + resolvedBody := false for index, argument := range common.Args { if argument != value { continue } found = true + if analyzer.config.BorrowedCallArgument != nil && + analyzer.config.BorrowedCallArgument(call, index) { + continue + } + if analyzer.config.ResolveCalleeBody != nil && !resolvedBody { + var resolved bool + callee, resolved = analyzer.config.ResolveCalleeBody(callee) + if !resolved || callee == nil { + return false + } + resolvedBody = true + } if !analyzer.proveParameter(callee, index) { return false } diff --git a/internal/coro/ssa_borrowed_allocation_test.go b/internal/coro/ssa_borrowed_allocation_test.go index 66dc3cc1a2..317edf4e1e 100644 --- a/internal/coro/ssa_borrowed_allocation_test.go +++ b/internal/coro/ssa_borrowed_allocation_test.go @@ -28,6 +28,8 @@ import ( func TestSSABorrowedAllocationProof(t *testing.T) { _, pkg := buildCoroTestSSA(t, "borrowed_allocation.go", `package coroid +import "unsafe" + type transaction struct { self *transaction endpoint *int @@ -84,6 +86,13 @@ func safeRecursive() bool { return value.phase == 0 } +func externalWord(uintptr) +func safeExternalWord() bool { + var value transaction + externalWord(uintptr(unsafe.Pointer(&value))) + return value.phase == 0 +} + func recurseA(value *transaction, escape bool) { recurseB(value, escape) } func recurseB(value *transaction, escape bool) { if escape { escaped = value; return } @@ -112,6 +121,25 @@ func escapeRecursive() { var value transaction; recurseA(&value, false) } if proof, ok := ProveSSABorrowedAllocation(recursiveAlloc); !ok || proof.ParametersProven == 0 { t.Fatalf("safe recursive borrow proof = %+v, present=%t", proof, ok) } + external := packageFunction(t, pkg, "safeExternalWord") + externalAlloc := exactHeapAllocation(t, external) + if proof, ok := ProveSSABorrowedAllocation(externalAlloc); ok { + t.Fatalf("bodyless external call received an implicit borrow proof: %+v", proof) + } + proof, ok = ProveSSABorrowedAllocationWithConfig(externalAlloc, SSABorrowedAllocationConfig{ + BorrowedCallArgument: func(call *ssa.Call, index int) bool { + callee := call.Common().StaticCallee() + return callee != nil && callee.Name() == "externalWord" && index == 0 + }, + }) + if !ok || proof.Allocation != externalAlloc { + t.Fatalf("certified external borrow proof = %+v, present=%t", proof, ok) + } + if proof, ok := ProveSSABorrowedAllocationWithConfig(externalAlloc, SSABorrowedAllocationConfig{ + BorrowedCallArgument: func(*ssa.Call, int) bool { return false }, + }); ok { + t.Fatalf("rejected external argument received a borrow proof: %+v", proof) + } for _, name := range []string{ "escapeGlobal", "escapeReturn", "escapeGo", "escapeDefer", "escapeDynamic", "escapeRecursive", diff --git a/runtime/internal/coroworker/_worker/worker.c b/runtime/internal/coroworker/_worker/worker.c index f9d972ae31..dadc5d9fa3 100644 --- a/runtime/internal/coroworker/_worker/worker.c +++ b/runtime/internal/coroworker/_worker/worker.c @@ -789,7 +789,7 @@ static bool llgo_coro_worker_prepare_fault_signals_v1(void) { return true; } -bool __llgo_coro_worker_call_v1( +static bool llgo_coro_worker_call_array_v1( uintptr_t function, uintptr_t trace_target, uint32_t argc, @@ -902,6 +902,31 @@ bool __llgo_coro_worker_call_v1( return true; } +bool __llgo_coro_worker_call_words_v2( + uintptr_t function, + uintptr_t trace_target, + uint32_t argc, + uintptr_t a0, + uintptr_t a1, + uintptr_t a2, + uintptr_t a3, + uintptr_t a4, + uintptr_t a5, + uintptr_t a6, + uintptr_t a7, + uintptr_t a8, + uintptr_t result_address) { + const uintptr_t args[LLGO_CORO_WORKER_MAX_ARGS_V1] = { + a0, a1, a2, a3, a4, a5, a6, a7, a8, + }; + return llgo_coro_worker_call_array_v1( + function, + trace_target, + argc, + args, + (struct llgo_coro_worker_result_v1 *)result_address); +} + /* * This is the complete blocking worker island. Neither the queue wait nor the * uintptr-shaped foreign call can enter a managed LLVM coroutine. Only the @@ -921,7 +946,7 @@ static void *llgo_coro_worker_main_v1(void *unused) { } struct llgo_coro_worker_result_v1 result; - if (!__llgo_coro_worker_call_v1( + if (!llgo_coro_worker_call_array_v1( job.function, job.trace_target, job.argc, job.args, &result) || __llgo_coro_native_worker_complete_v1( job.source_slot, diff --git a/runtime/internal/coroworker/_worker/worker.h b/runtime/internal/coroworker/_worker/worker.h index d499227132..486866bf5c 100644 --- a/runtime/internal/coroworker/_worker/worker.h +++ b/runtime/internal/coroworker/_worker/worker.h @@ -88,11 +88,19 @@ uint32_t __llgo_coro_worker_queue_wait_take_v1( bool __llgo_coro_worker_queue_stop_v1(uint32_t worker_count); bool __llgo_coro_worker_queue_destroy_after_join_v1(void); -bool __llgo_coro_worker_call_v1( +bool __llgo_coro_worker_call_words_v2( uintptr_t function, uintptr_t trace_target, uint32_t argc, - const uintptr_t args[LLGO_CORO_WORKER_MAX_ARGS_V1], - struct llgo_coro_worker_result_v1 *result); + uintptr_t a0, + uintptr_t a1, + uintptr_t a2, + uintptr_t a3, + uintptr_t a4, + uintptr_t a5, + uintptr_t a6, + uintptr_t a7, + uintptr_t a8, + uintptr_t result_address); #endif diff --git a/runtime/internal/coroworker/call_llgo.go b/runtime/internal/coroworker/call_llgo.go index 0e80af87ae..b19136a5e9 100644 --- a/runtime/internal/coroworker/call_llgo.go +++ b/runtime/internal/coroworker/call_llgo.go @@ -107,13 +107,3 @@ func QueueStop(workerCount uint32) bool // //go:linkname QueueDestroyAfterJoin C.__llgo_coro_worker_queue_destroy_after_join_v1 func QueueDestroyAfterJoin() bool - -// Call executes one exact uintptr-shaped foreign thunk synchronously on the -// calling native thread. traceTarget is the compiler-known source C entry used -// only for fault attribution; zero disables the hardware-fault landing pad for -// a reentry-capable boundary which cannot be abandoned by siglongjmp. It is -// reserved for the runtime's dynamically proved LockOSThread path; ordinary -// potentially blocking calls use the bounded worker queue above. -// -//go:linkname Call C.__llgo_coro_worker_call_v1 -func Call(function, traceTarget uintptr, argc uint32, args *[MaxArgs]uintptr, result *Result) bool diff --git a/runtime/internal/runtime/coro_os_thread_foreign_llgo.go b/runtime/internal/runtime/coro_os_thread_foreign_llgo.go index 9fe72850ba..d7d46b5298 100644 --- a/runtime/internal/runtime/coro_os_thread_foreign_llgo.go +++ b/runtime/internal/runtime/coro_os_thread_foreign_llgo.go @@ -56,6 +56,23 @@ var ( coroNativeForeignBoundaryTLSReadyV1 bool ) +// coroWorkerCallWordsV2 executes one exact uintptr-shaped foreign thunk +// synchronously on the calling native thread. The C wrapper owns its scratch +// argument record. resultAddress is the scalar encoding of a caller-owned +// coroworker.Result which remains live for this exact call; the compiler's +// frozen foreign borrow contract keeps that scratch result in the native or +// coroutine frame instead of the collector heap. traceTarget is used only for +// fault attribution; zero disables the hardware-fault landing pad for a +// reentry-capable boundary which cannot be abandoned by siglongjmp. +// +//go:linkname coroWorkerCallWordsV2 C.__llgo_coro_worker_call_words_v2 +func coroWorkerCallWordsV2( + function, traceTarget uintptr, + argc uint32, + a0, a1, a2, a3, a4, a5, a6, a7, a8 uintptr, + resultAddress uintptr, +) bool + // coroNativeForeignBoundaryTLSStartV1 makes the process-global pthread key an // explicit part of native-fleet startup. Runtime-only archives and other // section-garbage-collected library links are not required to retain or invoke @@ -644,12 +661,15 @@ func __llgo_coro_same_m_foreign_call_v1( if !installed { coroRuntimeAbort("same-M foreign call cannot publish callback context") } - args := [coroworker.MaxArgs]uintptr{record} - var result coroworker.Result // Managed reentry cannot be abandoned by a signal longjmp. The zero trace // target deliberately disables the worker fault landing pad for this exact // boundary; reentry faults retain the process signal disposition. - callOK := coroworker.Call(thunk, 0, 1, &args, &result) + var result coroworker.Result + callOK := coroWorkerCallWordsV2( + thunk, 0, 1, + record, 0, 0, 0, 0, 0, 0, 0, 0, + uintptr(unsafe.Pointer(&result)), + ) if !coroNativeForeignBoundaryRestoreTLSV1(&boundary, previous) { coroRuntimeAbort("same-M foreign call cannot restore callback context") } @@ -682,9 +702,12 @@ func coroNativeForeignWordCallV1( if !boundary.beginV1(task, mode, lazyCompensation) { coroRuntimeAbort("native direct foreign call cannot detach active resume") } - args := [coroworker.MaxArgs]uintptr{a0, a1, a2, a3, a4, a5, a6, a7, a8} var result coroworker.Result - callOK := coroworker.Call(function, traceTarget, argc, &args, &result) + callOK := coroWorkerCallWordsV2( + function, traceTarget, argc, + a0, a1, a2, a3, a4, a5, a6, a7, a8, + uintptr(unsafe.Pointer(&result)), + ) if !boundary.finishV1() { coroRuntimeAbort("native direct foreign call cannot reacquire managed execution") } diff --git a/runtime/poll_worker_source_test.go b/runtime/poll_worker_source_test.go index 3a182059a1..253afb7c7c 100644 --- a/runtime/poll_worker_source_test.go +++ b/runtime/poll_worker_source_test.go @@ -517,25 +517,26 @@ func TestRuntimeCoroWorkerKeepsPthreadCreationCertificateOwnerScoped(t *testing. } } -func TestRuntimeCoroWorkerBlockingCallHasOnlyGuardedSameMEntrance(t *testing.T) { +func TestRuntimeCoroWorkerBlockingCallHasOnlyScalarScratchSameMEntrance(t *testing.T) { declaration := readRuntimePollFile(t, runtimeCoroWorkerCallSource) - requireRuntimeAnnotationFreeCDeclarations(t, runtimeCoroWorkerCallSource, "Call") if strings.Contains(declaration, "func QueueWaitTake(") { t.Errorf("%s exposes the blocking worker consumer loop to managed Go", runtimeCoroWorkerCallSource) } - for _, required := range []string{ - "reserved for the runtime's dynamically proved", - "LockOSThread path", - "//go:linkname Call C.__llgo_coro_worker_call_v1", - "func Call(function, traceTarget uintptr, argc uint32, args *[MaxArgs]uintptr, result *Result) bool", - } { - if !strings.Contains(declaration, required) { - t.Errorf("%s lacks guarded same-M call contract %q", runtimeCoroWorkerCallSource, required) - } + if strings.Contains(declaration, "func Call(") || + strings.Contains(declaration, "func CallWords(") || + strings.Contains(declaration, "C.__llgo_coro_worker_call_v1") || + strings.Contains(declaration, "C.__llgo_coro_worker_call_words_v2") { + t.Errorf("%s exposes a synchronous C worker leaf outside its exact runtime owner", runtimeCoroWorkerCallSource) } entrance := readRuntimePollFile(t, runtimeCoroOSThreadForeignSource) + requireRuntimeAnnotationFreeCDeclarations(t, runtimeCoroOSThreadForeignSource, "coroWorkerCallWordsV2") for _, required := range []string{ + "C wrapper owns its scratch", + "frozen foreign borrow contract", + "//go:linkname coroWorkerCallWordsV2 C.__llgo_coro_worker_call_words_v2", + "func coroWorkerCallWordsV2(", + "resultAddress uintptr,", "native entersyscall/exitsyscall", "!coro.CurrentOSThreadLocked(task)", "type coroNativeForeignBoundaryV1 struct", @@ -555,7 +556,7 @@ func TestRuntimeCoroWorkerBlockingCallHasOnlyGuardedSameMEntrance(t *testing.T) "//export __llgo_coro_native_syscall_call_v1", "coro.ExecutorResumeHandoffSameMForeign", "coro.ExecutorResumeHandoffLockedForeign", - "callOK := coroworker.Call(function, traceTarget, argc, &args, &result)", + "callOK := coroWorkerCallWordsV2(", "boundary.parent.handoff.RequestReturn(boundary.baton)", "coroNativeMReplacementLineageOwnerV1(", "coroNativeMRecycleReplacementV1(returnedSlot)", @@ -570,7 +571,7 @@ func TestRuntimeCoroWorkerBlockingCallHasOnlyGuardedSameMEntrance(t *testing.T) "//export __llgo_coro_foreign_reentry_failure_v1", "//export __llgo_coro_same_m_foreign_call_v1", "boundary.beginV1(task, coro.ExecutorResumeHandoffSameMForeign, false)", - "callOK := coroworker.Call(thunk, 0, 1, &args, &result)", + "if !callOK", } { if !strings.Contains(entrance, required) { t.Errorf("%s lacks locked-thread call guard %q", runtimeCoroOSThreadForeignSource, required) @@ -622,7 +623,7 @@ func TestRuntimeCoroWorkerBlockingCallHasOnlyGuardedSameMEntrance(t *testing.T) begin, call := -1, -1 if helper >= 0 { begin = strings.Index(entrance[helper:], "if !boundary.beginV1(task, mode, lazyCompensation)") - call = strings.Index(entrance[helper:], "callOK := coroworker.Call(function, traceTarget, argc, &args, &result)") + call = strings.Index(entrance[helper:], "callOK := coroWorkerCallWordsV2(") if begin >= 0 { begin += helper } @@ -651,6 +652,11 @@ func TestRuntimeCoroWorkerBlockingCallHasOnlyGuardedSameMEntrance(t *testing.T) helper < 0 || begin < helper || call <= begin || helperFinish <= call { t.Errorf("%s does not bracket same-M C with detach/release/create/return/recycle/restore", runtimeCoroOSThreadForeignSource) } + if strings.Count(entrance, "callOK := coroWorkerCallWordsV2(") != 2 || + strings.Contains(entrance, "coroworker.Call(") || + strings.Contains(entrance, "coroworker.CallWords(") { + t.Errorf("%s does not use the scalar Go-to-C ABI for both synchronous native calls", runtimeCoroOSThreadForeignSource) + } quota := readRuntimePollFile(t, "internal/runtime/coro_execution_quota_native_llgo.go") for _, required := range []string{ "func coroTargetReleaseManagedExecutionV1(driver *coro.ExecutorDriver) bool", @@ -666,13 +672,19 @@ func TestRuntimeCoroWorkerBlockingCallHasOnlyGuardedSameMEntrance(t *testing.T) for _, required := range []string{ "This is the complete blocking worker island", "__llgo_coro_worker_queue_wait_take_v1(&job)", - "__llgo_coro_worker_call_v1(", + "static bool llgo_coro_worker_call_array_v1(", + "return llgo_coro_worker_call_array_v1(", + "if (!llgo_coro_worker_call_array_v1(", + "__llgo_coro_worker_call_words_v2(", "__llgo_coro_native_worker_complete_v1(", } { if !strings.Contains(cSource, required) { t.Errorf("%s lacks fixed native-stack worker step %q", runtimeCoroWorkerCSource, required) } } + if strings.Contains(cSource, "bool __llgo_coro_worker_call_v1(") { + t.Errorf("%s still exports the superseded pointer-taking worker-call ABI", runtimeCoroWorkerCSource) + } } func TestRuntimeNativeSyscallDeferredReplacementUsesStableRequestGate(t *testing.T) { diff --git a/runtime/syscall_worker_source_test.go b/runtime/syscall_worker_source_test.go index 68c7d06cdb..ae273f600b 100644 --- a/runtime/syscall_worker_source_test.go +++ b/runtime/syscall_worker_source_test.go @@ -646,12 +646,20 @@ struct llgo_coro_worker_result_v1 { uintptr_t fault_target; }; -bool __llgo_coro_worker_call_v1( +bool __llgo_coro_worker_call_words_v2( uintptr_t function, uintptr_t trace_target, uint32_t argc, - const uintptr_t args[9], - struct llgo_coro_worker_result_v1 *result); + uintptr_t a0, + uintptr_t a1, + uintptr_t a2, + uintptr_t a3, + uintptr_t a4, + uintptr_t a5, + uintptr_t a6, + uintptr_t a7, + uintptr_t a8, + uintptr_t result_address); uint32_t __llgo_coro_native_worker_complete_v1( uint32_t source_slot, @@ -701,14 +709,24 @@ static uintptr_t fault_memory(uintptr_t address) { return *(volatile uintptr_t *)address; } +static bool call_words( + uintptr_t function, + uintptr_t trace_target, + uint32_t argc, + struct llgo_coro_worker_result_v1 *result) { + return __llgo_coro_worker_call_words_v2( + function, trace_target, argc, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + (uintptr_t)(void *)result); +} + static int call_and_check( uintptr_t function, uintptr_t want_r1, uintptr_t want_errno, int base) { - const uintptr_t args[9] = {0}; struct llgo_coro_worker_result_v1 result = {0}; - if (!__llgo_coro_worker_call_v1(function, function, 1, args, &result)) { + if (!call_words(function, function, 1, &result)) { return base; } if (result.r1 != want_r1) { @@ -756,11 +774,18 @@ int main(void) { if (status != 0) { return status; } + struct llgo_coro_worker_result_v1 invalid = {0}; + if (call_words( + (uintptr_t)(void *)&success_with_errno, + (uintptr_t)(void *)&success_with_errno, + 10, + &invalid)) { + return 49; + } for (int attempt = 0; attempt < 2; ++attempt) { - const uintptr_t args[9] = {0}; struct llgo_coro_worker_result_v1 result = {0}; uintptr_t function = (uintptr_t)(void *)&fault_memory; - if (!__llgo_coro_worker_call_v1(function, function, 1, args, &result)) { + if (!call_words(function, function, 1, &result)) { return 50 + attempt * 10; } if (result.fault != 1 || result.fault_target != function) {