diff --git a/.github/actions/setup-deps/action.yml b/.github/actions/setup-deps/action.yml index 35ee24ff7c..da6907f274 100644 --- a/.github/actions/setup-deps/action.yml +++ b/.github/actions/setup-deps/action.yml @@ -4,7 +4,7 @@ inputs: llvm-version: description: "LLVM version to install" required: true - default: "19" + default: "22" install-llvm: description: "Whether to install LLVM" required: false @@ -26,7 +26,7 @@ runs: llvm_formula="llvm@${{inputs.llvm-version}}" lld_formula="lld@${{inputs.llvm-version}}" - # GitHub macOS runners may pre-link another LLVM/LLD version (for example llvm@18), + # GitHub macOS runners may pre-link another LLVM/LLD version, # which makes lld@ fail during Homebrew's automatic link step. while IFS= read -r formula; do case "${formula}" in diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 5c3e16613b..81cec9a56d 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -47,7 +47,7 @@ jobs: - name: Install dependencies uses: ./.github/actions/setup-deps with: - llvm-version: 19 + llvm-version: 22 - name: Set up Go uses: ./.github/actions/setup-go diff --git a/.github/workflows/build-cache.yml b/.github/workflows/build-cache.yml index 24556747c4..7d9b0513dc 100644 --- a/.github/workflows/build-cache.yml +++ b/.github/workflows/build-cache.yml @@ -20,7 +20,7 @@ jobs: strategy: matrix: os: [macos-latest, ubuntu-latest] - llvm: [19] + llvm: [22] runs-on: ${{matrix.os}} steps: - uses: actions/checkout@v7 diff --git a/.github/workflows/coroutine.yml b/.github/workflows/coroutine.yml index 08a932048f..0aa8bac8fd 100644 --- a/.github/workflows/coroutine.yml +++ b/.github/workflows/coroutine.yml @@ -22,22 +22,19 @@ jobs: fail-fast: false matrix: include: - - { llvm: 19, go: "1.26.5", tags: "llvm19", lane: "compat", check: "test (19, 1.26.5, llvm19)" } - - { llvm: 20, go: "1.26.5", tags: "llvm20", lane: "compat", check: "test (20, 1.26.5, llvm20)" } - - { llvm: 21, go: "1.26.5", tags: "llvm21", lane: "compat", check: "test (21, 1.26.5, llvm21)" } - - { llvm: 22, go: "1.26.5", tags: "llvm22", lane: "compat", check: "test (22, 1.26.5, llvm22)" } - - { llvm: 19, go: "1.26.5", tags: "llvm19", lane: "integration", check: "test integration (19, 1.26.5, llvm19)" } - - { llvm: 19, go: "1.26.5", tags: "llvm19", lane: "integration-build", check: "test production build plan (19, 1.26.5, llvm19)" } - - { llvm: 19, go: "1.26.5", tags: "llvm19", lane: "native-e2e", check: "test native linked E2E (19, 1.26.5, llvm19)" } - - { llvm: 19, go: "1.26.5", tags: "llvm19", lane: "fleet-e2e", check: "test native fleet E2E (19, 1.26.5, llvm19)" } - - { llvm: 19, go: "1.26.5", tags: "llvm19", lane: "stdlib-time", fixtures: "time timer sync", check: "test stdlib time, timer and sync (19, 1.26.5, llvm19)" } - - { llvm: 19, go: "1.26.5", tags: "llvm19", lane: "stdlib-file", fixtures: "file syscall-file syscall-pipe", check: "test stdlib file and syscall (19, 1.26.5, llvm19)" } - - { llvm: 19, go: "1.26.5", tags: "llvm19", lane: "stdlib-tcp", fixtures: "tcp", check: "test stdlib TCP (19, 1.26.5, llvm19)" } - - { llvm: 19, go: "1.26.5", tags: "llvm19", lane: "target-core", check: "test target adapters (19, 1.26.5, llvm19)" } - - { llvm: 19, go: "1.26.5", tags: "llvm19", lane: "target-js-wasm", check: "test JS/WASM command reactor (llvm19)" } - - { llvm: 19, go: "1.26.5", tags: "llvm19", lane: "target-wasi-command", check: "test WASI command reactor (llvm19)" } - - { llvm: 19, go: "1.26.5", tags: "llvm19", lane: "target-wasm", wasm_target: "wasip2", check: "test WASM host probes (wasip2, llvm19)" } - - { llvm: 19, go: "1.26.5", tags: "llvm19", lane: "target-wasm", wasm_target: "wasm-unknown", check: "test WASM host probes (wasm-unknown, llvm19)" } + - { llvm: 22, go: "1.26.5", lane: "compat", check: "test (LLVM 22, Go 1.26.5)" } + - { llvm: 22, go: "1.26.5", lane: "integration", check: "test integration (LLVM 22, Go 1.26.5)" } + - { llvm: 22, go: "1.26.5", lane: "integration-build", check: "test production build plan (LLVM 22, Go 1.26.5)" } + - { llvm: 22, go: "1.26.5", lane: "native-e2e", check: "test native linked E2E (LLVM 22, Go 1.26.5)" } + - { llvm: 22, go: "1.26.5", lane: "fleet-e2e", check: "test native fleet E2E (LLVM 22, Go 1.26.5)" } + - { llvm: 22, go: "1.26.5", lane: "stdlib-time", fixtures: "time timer sync", check: "test stdlib time, timer and sync (LLVM 22, Go 1.26.5)" } + - { llvm: 22, go: "1.26.5", lane: "stdlib-file", fixtures: "file syscall-file syscall-pipe", check: "test stdlib file and syscall (LLVM 22, Go 1.26.5)" } + - { llvm: 22, go: "1.26.5", lane: "stdlib-tcp", fixtures: "tcp", check: "test stdlib TCP (LLVM 22, Go 1.26.5)" } + - { llvm: 22, go: "1.26.5", lane: "target-core", check: "test target adapters (LLVM 22, Go 1.26.5)" } + - { llvm: 22, go: "1.26.5", lane: "target-js-wasm", check: "test JS/WASM command reactor (LLVM 22)" } + - { llvm: 22, go: "1.26.5", lane: "target-wasi-command", check: "test WASI command reactor (LLVM 22)" } + - { llvm: 22, go: "1.26.5", lane: "target-wasm", wasm_target: "wasip2", check: "test WASM host probes (wasip2, LLVM 22)" } + - { llvm: 22, go: "1.26.5", lane: "target-wasm", wasm_target: "wasm-unknown", check: "test WASM host probes (wasm-unknown, LLVM 22)" } steps: - uses: actions/checkout@v7 @@ -164,6 +161,7 @@ jobs: # generation reuse, cancellation, and single-owner posting without # silently restoring the old channel spin gate dependency. go test -race -shuffle=on -tags=coro_sema_owner_test \ + ./internal/runtime/coro_current_task_route_test_adapter.go \ ./internal/runtime/coro_keyed_registry_atomic_host.go \ ./internal/runtime/coro_resume_materialize.go \ ./internal/runtime/coro_operation_capacity.go \ @@ -174,6 +172,7 @@ jobs: GOOS=js GOARCH=wasm CGO_ENABLED=0 go test \ -tags=coro_sema_owner_test \ -exec="$(go env GOROOT)/lib/wasm/go_js_wasm_exec" \ + ./internal/runtime/coro_current_task_route_test_adapter.go \ ./internal/runtime/coro_keyed_registry_atomic_host.go \ ./internal/runtime/coro_resume_materialize.go \ ./internal/runtime/coro_operation_capacity.go \ @@ -373,7 +372,7 @@ jobs: - name: Verify production time.Sleep coroutine plan if: matrix.lane == 'compat' - run: go test -tags='${{ matrix.tags }}' -v ./internal/build -run '^TestCoroNativeTimeSleepProductionPlanAndCodegen$' -timeout=10m -count=1 + run: go test -v ./internal/build -run '^TestCoroNativeTimeSleepProductionPlanAndCodegen$' -timeout=10m -count=1 - name: Test coroutine compiler integration if: matrix.lane == 'integration' @@ -383,33 +382,33 @@ jobs: - name: Test structured LLVM coroutine builder if: matrix.lane == 'compat' - run: go test -tags='${{ matrix.tags }}' -v ./ssa -run '^TestCoro' -count=1 + run: go test -v ./ssa -run '^TestCoro' -count=1 - name: Test canonical coroutine plan digest and cache identity if: matrix.lane == 'compat' run: | - go test -tags='${{ matrix.tags }}' ./internal/coro -run '^TestCoroPlanDigest' -count=1 - go test -tags='${{ matrix.tags }}' ./internal/build -run '^Test(BuildCoroPlanInstallsArchiveDigest|CoroutinePlanInputsAffectFingerprint|CoroEntryResolutionUsesPlanMatchedPackageCache|CoroPlanDigestMetadataUsesEffectiveLLVMTarget|CoroPhysicalABICacheRegistrationPreservesCollectedFuncInfo)$' -count=1 - go test -tags='${{ matrix.tags }}' ./cl -run '^Test(CompilationCoroABIIdentityValidation|CoroEntryResolutionCacheRegistrationWithDigest|CoroPhysicalABICacheRegistrationPreservesPhysicalMetadata)$' -count=1 + go test ./internal/coro -run '^TestCoroPlanDigest' -count=1 + go test ./internal/build -run '^Test(BuildCoroPlanInstallsArchiveDigest|CoroutinePlanInputsAffectFingerprint|CoroEntryResolutionUsesPlanMatchedPackageCache|CoroPhysicalABICacheRegistrationPreservesCollectedFuncInfo)$' -count=1 + go test ./cl -run '^Test(CompilationCoroABIIdentityValidation|CoroEntryResolutionCacheRegistrationWithDigest|CoroPhysicalABICacheRegistrationPreservesPhysicalMetadata)$' -count=1 - name: Test coroutine physical ABI and function dispatch lowering if: matrix.lane == 'compat' # Run every compiler test whose name is part of the coroutine contract; # in particular this covers pure SSA aggregates/PHI and caller-frame # park lowering on native64 and wasm32 before and after CoroSplit. - run: go test -tags='${{ matrix.tags }}' -v ./cl -run '^Test(Coro|EmissionUniverse(ActiveABIMethodTablesUseFrozenWrapperSymbols|ABIMethodDemandReferencesAreExactRecursiveAndOwnerScoped))' -count=1 + run: go test -v ./cl -run '^Test(Coro|EmissionUniverse(ActiveABIMethodTablesUseFrozenWrapperSymbols|ABIMethodDemandReferencesAreExactRecursiveAndOwnerScoped))' -count=1 - name: Test coroutine TLS function dispatch proof if: matrix.lane == 'compat' - run: go test -tags='${{ matrix.tags }}' -v ./internal/build -run '^TestCoroTLS' -count=1 + run: go test -v ./internal/build -run '^TestCoroTLS' -count=1 - name: Test coroutine registry and control integration if: matrix.lane == 'compat' - run: go test -tags='${{ matrix.tags }}' -v ./internal/build -run '^Test(CollectLinkedCoroRootAnchors|ActiveCoroABIVersions|BuildCoroPlanErrors|CoroProgramManifest.*|CoroProgramBootstrap.*|SelectCoroProgramBootstrap.*|GenMainModule.*Coro.*)$' -count=1 + run: go test -v ./internal/build -run '^Test(CollectLinkedCoroRootAnchors|ActiveCoroABIVersions|BuildCoroPlanErrors|CoroProgramManifest.*|CoroProgramBootstrap.*|SelectCoroProgramBootstrap.*|GenMainModule.*Coro.*)$' -count=1 - name: Test LLVM 22 tool configuration - if: matrix.lane == 'compat' && matrix.llvm == 22 - run: go test -tags=llvm22 ./xtool/env/llvm ./internal/xtool/llvm + if: matrix.lane == 'compat' + run: go test ./xtool/env/llvm ./internal/xtool/llvm - name: Test resolved LLVM target configuration if: matrix.lane == 'target-core' diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index ae51ed85ae..5961801a21 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -122,7 +122,7 @@ jobs: set -e set -x source doc/_readme/scripts/install_ubuntu.sh - echo "PATH=/usr/lib/llvm-19/bin:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/llvm-22/bin:$PATH" >> $GITHUB_ENV - name: Install llgo with tools run: | diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 0e6c46a374..1b35da079e 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -22,7 +22,7 @@ jobs: os: - macos-latest - ubuntu-latest - llvm: [19] + llvm: [22] runs-on: ${{matrix.os}} steps: - uses: actions/checkout@v7 @@ -109,7 +109,7 @@ jobs: - name: Install dependencies uses: ./.github/actions/setup-deps with: - llvm-version: 19 + llvm-version: 22 - name: Set up Go uses: ./.github/actions/setup-go @@ -128,7 +128,7 @@ jobs: run: | set -euo pipefail - llvm_config="$(command -v llvm-config || command -v llvm-config-19)" + llvm_config="$(command -v llvm-config || command -v llvm-config-22)" cmake -S ltoplugin -B ltoplugin/build \ -DLLVM_DIR="$("${llvm_config}" --cmakedir)" \ -DCMAKE_BUILD_TYPE=Release @@ -190,7 +190,7 @@ jobs: - name: Install dependencies uses: ./.github/actions/setup-deps with: - llvm-version: 19 + llvm-version: 22 - name: Set up Go uses: ./.github/actions/setup-go diff --git a/.github/workflows/goroot.yml b/.github/workflows/goroot.yml index a69f4394ac..8b52907a96 100644 --- a/.github/workflows/goroot.yml +++ b/.github/workflows/goroot.yml @@ -43,7 +43,7 @@ jobs: - name: Install dependencies uses: ./.github/actions/setup-deps with: - llvm-version: 19 + llvm-version: 22 - name: Set up Go uses: ./.github/actions/setup-go diff --git a/.github/workflows/llgo.yml b/.github/workflows/llgo.yml index e306d77ec6..a5011fb2fb 100644 --- a/.github/workflows/llgo.yml +++ b/.github/workflows/llgo.yml @@ -28,19 +28,19 @@ jobs: # 35-minute demo job. include: - os: ubuntu-latest - llvm: 19 + llvm: 22 go: "1.24.2" lane: compatibility - os: ubuntu-latest - llvm: 19 + llvm: 22 go: "1.26.5" lane: primary - os: macos-latest - llvm: 19 + llvm: 22 go: "1.24.2" lane: compatibility - os: macos-latest - llvm: 19 + llvm: 22 go: "1.26.5" lane: primary runs-on: ${{matrix.os}} @@ -147,7 +147,7 @@ jobs: os: - macos-latest - ubuntu-latest - llvm: [19] + llvm: [22] go: ["1.24.2", "1.26.5"] # In-command package parallelism lets Ubuntu use two shards while # retaining headroom for the serial std build-mode checks. @@ -244,17 +244,17 @@ jobs: matrix: include: - os: ubuntu-latest - llvm: 19 + llvm: 22 go: "1.24.2" lane: compatibility - os: ubuntu-latest - llvm: 19 + llvm: 22 go: "1.26.5" lane: primary # Keep the Go 1.26 user-module compatibility matrix on both host # platforms; release artifact smoke tests alone only cover go 1.26. - os: macos-latest - llvm: 19 + llvm: 22 go: "1.26.5" lane: primary runs-on: ${{matrix.os}} @@ -314,7 +314,7 @@ jobs: # Ubuntu. Native Darwin coverage remains in the primary LLGo lanes and # release artifact smoke tests. os: [ubuntu-latest] - llvm: [19] + llvm: [22] runs-on: ${{matrix.os}} steps: - uses: actions/checkout@v7 @@ -367,7 +367,7 @@ jobs: - name: Install dependencies uses: ./.github/actions/setup-deps with: - llvm-version: 19 + llvm-version: 22 - name: Set up Emscripten uses: emscripten-core/setup-emsdk@v15 diff --git a/.github/workflows/model-demo.yml b/.github/workflows/model-demo.yml index 89af920fcc..7c6d8f6be1 100644 --- a/.github/workflows/model-demo.yml +++ b/.github/workflows/model-demo.yml @@ -19,7 +19,7 @@ jobs: - name: Install dependencies uses: ./.github/actions/setup-deps with: - llvm-version: 19 + llvm-version: 22 - name: Set up Go uses: ./.github/actions/setup-go diff --git a/.github/workflows/targets.yml b/.github/workflows/targets.yml index f64d6af40b..18f7466ccc 100644 --- a/.github/workflows/targets.yml +++ b/.github/workflows/targets.yml @@ -21,7 +21,7 @@ jobs: matrix: os: - ubuntu-latest - llvm: [19] + llvm: [22] runs-on: ${{matrix.os}} steps: - uses: actions/checkout@v7 diff --git a/README.md b/README.md index a8185aec21..ae6c4876d9 100644 --- a/README.md +++ b/README.md @@ -341,15 +341,20 @@ Here are the Go packages that can be imported correctly: ## Dependencies - [Go 1.21+](https://go.dev) -- [LLVM 18](https://llvm.org) -- [Clang 18](https://clang.llvm.org) -- [LLD 18](https://lld.llvm.org) +- [LLVM 22](https://llvm.org) +- [Clang 22](https://clang.llvm.org) +- [LLD 22](https://lld.llvm.org) - [pkg-config 0.29+](https://www.freedesktop.org/wiki/Software/pkg-config/) - [bdwgc/libgc 8.0+](https://www.hboehm.info/gc/) - [OpenSSL 3.0+](https://www.openssl.org/) - [zlib 1.2+](https://www.zlib.net) - [Python 3.12+](https://www.python.org) (optional, for [github.com/goplus/lib/py](https://pkg.go.dev/github.com/goplus/lib/py)) +LLGo's compiler, Go binding, coroutine lowering, and supported CI baseline are +LLVM 22 only. Version-pinned Espressif artifacts and generated ABI fixtures in +the target support tree are external vendor inputs, not an alternate supported +host LLVM/CoroSplit version. + ## How to install Follow these steps to generate the `llgo` command (its usage is the same as the `go` command): @@ -360,9 +365,9 @@ Follow these steps to generate the `llgo` command (its usage is the same as the ```sh brew update -brew install llvm@19 lld@19 bdw-gc openssl cjson libffi libuv pkg-config +brew install llvm@22 lld@22 bdw-gc openssl cjson libffi libuv pkg-config brew install python@3.12 # optional -brew link --overwrite llvm@19 lld@19 libffi +brew link --overwrite llvm@22 lld@22 libffi # curl https://raw.githubusercontent.com/xgo-dev/llgo/refs/heads/main/install.sh | bash ./install.sh ``` @@ -374,10 +379,10 @@ brew link --overwrite llvm@19 lld@19 libffi ```sh -echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-19 main" | sudo tee /etc/apt/sources.list.d/llvm.list +echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-22 main" | sudo tee /etc/apt/sources.list.d/llvm.list wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - sudo apt-get update -sudo apt-get install -y llvm-19-dev clang-19 libclang-19-dev lld-19 libunwind-19-dev libc++-19-dev pkg-config libgc-dev libssl-dev zlib1g-dev libffi-dev libcjson-dev libsqlite3-dev libuv1-dev +sudo apt-get install -y llvm-22-dev clang-22 libclang-22-dev lld-22 libunwind-22-dev libc++-22-dev pkg-config libgc-dev libssl-dev zlib1g-dev libffi-dev libcjson-dev libsqlite3-dev libuv1-dev sudo apt-get install -y python3.12-dev # optional #curl https://raw.githubusercontent.com/xgo-dev/llgo/refs/heads/main/install.sh | bash ./install.sh @@ -386,10 +391,10 @@ sudo apt-get install -y python3.12-dev # optional #### Alpine Linux ```sh -apk add go llvm19-dev clang19-dev lld19 pkgconf gc-dev libunwind-dev openssl-dev zlib-dev +apk add go llvm22-dev clang22-dev lld22 pkgconf gc-dev libunwind-dev openssl-dev zlib-dev apk add python3-dev # optional apk add g++ # build only -export LLVM_CONFIG=/usr/lib/llvm19/bin/llvm-config +export LLVM_CONFIG=/usr/lib/llvm22/bin/llvm-config export CGO_CPPFLAGS="$($LLVM_CONFIG --cppflags)" export CGO_CXXFLAGS=-std=c++17 export CGO_LDFLAGS="$($LLVM_CONFIG --ldflags) $($LLVM_CONFIG --libs all)" diff --git a/benchmark/coro_core/README.md b/benchmark/coro_core/README.md index 33cc6eef2b..61339eb4c2 100644 --- a/benchmark/coro_core/README.md +++ b/benchmark/coro_core/README.md @@ -20,6 +20,15 @@ runtime costs: - `handoff`: unbuffered channel scheduling throughput; - `timers`: concurrent standard-library timer registration and wakeup. +`testdata/pure_idle`, `testdata/pure_compute`, and `testdata/pure_handoff` have +no imports and no output. They are closed-world negative fixtures for the +optional worker executor. The first isolates fixed startup cost; the second +measures a billion direct, non-inlined integer calls plus loop safepoints; the +third performs 100,000 request/ack round trips through two unbuffered channels +without pulling standard-library, timer, poller, or worker costs into the +artifact. Success is established by exit status and compiler/runtime +capability gates, not by parsing stdout. + `testdata/io_workload` is separate so importing `os`, `io`, and `net` does not pollute the core artifact. Its modes are: @@ -28,6 +37,13 @@ pollute the core artifact. Its modes are: - `tcp`: one persistent loopback TCP connection, with 4 KiB request/echo round trips between two goroutines. +`testdata/preempt_timer` is a bounded progress gate: one goroutine sleeps on a +standard-library timer while the sole runnable goroutine executes a pure +compute loop. The timer must wake by compiler safepoint preemption before the +100-million-iteration guard is exhausted. This catches executor-service +optimizations which accidentally rely only on runnable peers or callbacks and +therefore starve elapsed timer/poll sources. + The final output field is workload wall time in nanoseconds, measured inside the process after argument parsing. It excludes process startup; an external resource tool should still measure peak RSS for `park`. @@ -55,8 +71,12 @@ the host Go tool as a lightweight source check: ```sh go test \ + ./benchmark/coro_core/testdata/pure_idle \ + ./benchmark/coro_core/testdata/pure_compute \ + ./benchmark/coro_core/testdata/pure_handoff \ ./benchmark/coro_core/testdata/workload \ ./benchmark/coro_core/testdata/io_workload \ + ./benchmark/coro_core/testdata/preempt_timer \ ./benchmark/coro_core/testdata/wasm ``` diff --git a/benchmark/coro_core/testdata/preempt_timer/main.go b/benchmark/coro_core/testdata/preempt_timer/main.go new file mode 100644 index 0000000000..b4d957c76e --- /dev/null +++ b/benchmark/coro_core/testdata/preempt_timer/main.go @@ -0,0 +1,51 @@ +//go:build !baremetal + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This bounded program deliberately uses only ordinary Go APIs so Go gc and +// the coroutine backend execute the same progress contract. +package main + +import ( + "time" +) + +const spinLimit = 100_000_000 + +//go:noinline +func spinStep(value, salt uint32) uint32 { + return (value^(salt+0x1f123bb5))*1_664_525 + 1_013_904_223 +} + +func main() { + fired := make(chan struct{}, 1) + go func() { + time.Sleep(2 * time.Millisecond) + fired <- struct{}{} + }() + + value := uint32(1) + for iteration := 0; iteration != spinLimit; iteration++ { + value = spinStep(value, uint32(iteration)) + } + select { + case <-fired: + default: + panic("timer did not preempt the sole runnable compute loop") + } + println("ok preempt-timer", spinLimit, value) +} diff --git a/benchmark/coro_core/testdata/pure_compute/main.go b/benchmark/coro_core/testdata/pure_compute/main.go new file mode 100644 index 0000000000..d8ae7aeaf9 --- /dev/null +++ b/benchmark/coro_core/testdata/pure_compute/main.go @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This fixture deliberately has no imports, I/O, goroutines, or dynamic calls. +// It measures the static-call and safepoint path without pulling a potentially +// blocking C function into the closed program. +package main + +const iterations = 1_000_000_000 + +var result int + +//go:noinline +func arithmeticStep(value, salt int) int { + value ^= salt + 0x1f123bb5 + return (value*1_664_525 + 1_013_904_223) & 0x7fff_ffff +} + +func main() { + value := 1 + for index := range iterations { + value = arithmeticStep(value, index) + } + result = value +} diff --git a/benchmark/coro_core/testdata/pure_handoff/main.go b/benchmark/coro_core/testdata/pure_handoff/main.go new file mode 100644 index 0000000000..3c936601f2 --- /dev/null +++ b/benchmark/coro_core/testdata/pure_handoff/main.go @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This fixture deliberately has no imports or output. It isolates one child +// goroutine and two unbuffered channels from standard-library, timer, poller, +// and worker-executor costs. Each iteration is one request/ack round trip. +package main + +const handoffs = 100_000 + +var result int + +func main() { + values := make(chan int) + acks := make(chan int) + done := make(chan struct{}) + go func() { + for range handoffs { + acks <- <-values + 1 + } + close(done) + }() + + checksum := 0 + for value := range handoffs { + values <- value + checksum ^= <-acks + } + <-done + result = checksum +} diff --git a/xtool/env/llvm/llvm_config_darwin_amd64_llvm14.go b/benchmark/coro_core/testdata/pure_idle/main.go similarity index 64% rename from xtool/env/llvm/llvm_config_darwin_amd64_llvm14.go rename to benchmark/coro_core/testdata/pure_idle/main.go index 326a42b8ca..5d64ed9f22 100644 --- a/xtool/env/llvm/llvm_config_darwin_amd64_llvm14.go +++ b/benchmark/coro_core/testdata/pure_idle/main.go @@ -1,7 +1,5 @@ -//go:build !byollvm && darwin && amd64 && llvm14 - /* - * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +14,9 @@ * limitations under the License. */ -package llvm +// This fixture deliberately imports and observes nothing. It isolates process +// and scheduler startup without making a foreign write part of the closed +// program, so the compiler must not request a worker executor. +package main -const ldLLVMConfigBin = "/usr/local/opt/llvm@14/bin/llvm-config" +func main() {} diff --git a/chore/gentests/gentests.go b/chore/gentests/gentests.go index c306925bf3..2aee92f038 100644 --- a/chore/gentests/gentests.go +++ b/chore/gentests/gentests.go @@ -32,7 +32,7 @@ import ( ) func main() { - llvm.SetupPath() + llvm.SetupPathOrExit() dir, _, err := mod.FindGoMod(".") check(err) diff --git a/chore/litgen/litgen.go b/chore/litgen/litgen.go index 34d51fe437..87a724d8cf 100644 --- a/chore/litgen/litgen.go +++ b/chore/litgen/litgen.go @@ -27,7 +27,7 @@ import ( ) func main() { - llvm.SetupPath() + llvm.SetupPathOrExit() flag.Usage = func() { fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s [...]\n", filepath.Base(os.Args[0])) flag.PrintDefaults() diff --git a/chore/llgen/llgen.go b/chore/llgen/llgen.go index f4050d3128..20df689db2 100644 --- a/chore/llgen/llgen.go +++ b/chore/llgen/llgen.go @@ -31,7 +31,7 @@ var ( ) func main() { - llvm.SetupPath() + llvm.SetupPathOrExit() flag.Parse() if len(flag.Args()) != 1 { fmt.Fprintln(os.Stderr, "Usage: llgen [flags] ") diff --git a/chore/nmdump/nmdump.go b/chore/nmdump/nmdump.go index 25affe487d..76482cefa3 100644 --- a/chore/nmdump/nmdump.go +++ b/chore/nmdump/nmdump.go @@ -31,6 +31,7 @@ func main() { return } + llvm.SetupPathOrExit() nm := llvm.New("").Nm() var flags []string diff --git a/chore/nmindex/nmindex.go b/chore/nmindex/nmindex.go index 4cddb63024..8aad290498 100644 --- a/chore/nmindex/nmindex.go +++ b/chore/nmindex/nmindex.go @@ -54,6 +54,7 @@ The commands are: } func makeIndex() { + llvm.SetupPathOrExit() env := llvm.New("") idxDir := indexDir() os.MkdirAll(idxDir, 0755) diff --git a/cl/_testgo/allocinloop/in.go b/cl/_testgo/allocinloop/in.go index 8559658c78..cf2727b79f 100644 --- a/cl/_testgo/allocinloop/in.go +++ b/cl/_testgo/allocinloop/in.go @@ -12,13 +12,16 @@ func Foo(s string) int { // A long-running loop is a stackless coroutine and must contain scheduler // polling without turning its per-iteration scalar work into heap allocation. // CHECK-LABEL: define ptr @"main.Test$coro"(ptr %0, ptr %1){{.*}} { -// CHECK: call i1 @__llgo_coro_preempt_poll_v1(ptr %0) -// CHECK: call i64 @main.Foo(%"{{.*}}String" { ptr @{{[0-9]+}}, i64 5 }) -// CHECK: call ptr @"{{.*}}PrintInt$coro" -// CHECK: call void @__llgo_coro_await_prepare_v3 -// CHECK: icmp slt i64 {{.*}}, 10000000 -// CHECK: call ptr @"{{.*}}PrintByte$coro"({{.*}}i8 10) -// CHECK: call void @__llgo_coro_await_prepare_v3 +// LLVM may lay out the loop body, poll slow path, and exit blocks in any CFG +// order. Keep these as one unordered function-local cohort rather than +// freezing a backend block order. +// CHECK-DAG: call i64 @main.Foo(%"{{.*}}String" { ptr @{{[0-9]+}}, i64 5 }) +// CHECK-DAG: call i1 @__llgo_coro_preempt_poll_v1(ptr %0) +// CHECK-DAG: call ptr @"{{.*}}PrintInt$coro" +// CHECK-DAG: call void @__llgo_coro_await_prepare_v3 +// CHECK-DAG: icmp slt i64 {{.*}}, 10000000 +// CHECK-DAG: call ptr @"{{.*}}PrintByte$coro"({{.*}}i8 10) +// CHECK-DAG: call void @__llgo_coro_await_prepare_v3 // CHECK-NOT: call ptr @"{{.*}}AllocZ" func Test() { j := 0 diff --git a/cl/_testgo/cgofull/cgofull.go b/cl/_testgo/cgofull/cgofull.go index 2134e407fa..744d94f221 100644 --- a/cl/_testgo/cgofull/cgofull.go +++ b/cl/_testgo/cgofull/cgofull.go @@ -177,7 +177,7 @@ func runPy() { // This is a structural cgo/coro boundary test. Exact worker thunk hashes and // LLVM block numbering are deliberately not part of the contract. // CHECK-LABEL: define ptr @"main.Bar$coro"( -// CHECK: call void @__llgo_coro_os_thread_foreign_call_v1( +// CHECK: call i32 @__llgo_coro_os_thread_foreign_call_v1( // CHECK: call void @__llgo_coro_worker_park_v1( // CHECK: call i8 @llvm.coro.suspend( // CHECK: call i32 @__llgo_coro_worker_resume_v1( diff --git a/cl/_testgo/cgopython/cgopython.go b/cl/_testgo/cgopython/cgopython.go index 1e8bce5c65..fa97dd10a9 100644 --- a/cl/_testgo/cgopython/cgopython.go +++ b/cl/_testgo/cgopython/cgopython.go @@ -29,7 +29,7 @@ import "runtime" // CHECK: call token @llvm.coro.id // CHECK: call ptr @"runtime.LockOSThread$coro" // CHECK: call i1 @__llgo_coro_os_thread_locked_v1 -// CHECK: call void @__llgo_coro_os_thread_foreign_call_v1 +// CHECK: call i32 @__llgo_coro_os_thread_foreign_call_v1 // CHECK: call void @__llgo_coro_worker_park_v1 // CHECK: call i32 @__llgo_coro_worker_resume_v1 diff --git a/cl/_testgo/chan/in.go b/cl/_testgo/chan/in.go index 58b40a076f..1ab6110759 100644 --- a/cl/_testgo/chan/in.go +++ b/cl/_testgo/chan/in.go @@ -37,4 +37,4 @@ func main() { // CHECK: CoroChanTrySend // CHECK-LABEL: define ptr @"main.main$2$coro"( // CHECK: call i8 @llvm.coro.suspend( -// CHECK: CoroChanTryClose +// CHECK: CoroChanTryCloseTask diff --git a/cl/_testgo/goroutinebuiltin/in.go b/cl/_testgo/goroutinebuiltin/in.go index 8a9c5226a7..83f8fa1ff7 100644 --- a/cl/_testgo/goroutinebuiltin/in.go +++ b/cl/_testgo/goroutinebuiltin/in.go @@ -11,7 +11,7 @@ package main // CHECK-NOT: NewProc // CHECK-NOT: _llgo_routine // CHECK-LABEL: define ptr @"main.close$wrapper$llgo$builtin-spawn$v1${{.*}}$coro"( -// CHECK: call i32 @"{{.*}}CoroChanTryClose"( +// CHECK: call i32 @"{{.*}}CoroChanTryCloseTask"( // CHECK: call i8 @llvm.coro.suspend( // CHECK-NOT: NewProc // CHECK-NOT: _llgo_routine diff --git a/cl/_testlibc/argv/in.go b/cl/_testlibc/argv/in.go index c2b163c9d0..a8543022ae 100644 --- a/cl/_testlibc/argv/in.go +++ b/cl/_testlibc/argv/in.go @@ -9,7 +9,7 @@ import ( func main() { // CHECK-DAG: load i32, ptr @__llgo_argc, align 4 // CHECK-DAG: load ptr, ptr @__llgo_argv, align 8 - // CHECK: call void @__llgo_coro_os_thread_foreign_call_v1 + // CHECK: call i32 @__llgo_coro_os_thread_foreign_call_v1 // CHECK: call void @__llgo_coro_worker_park_v1 // CHECK: call i32 (ptr, ...) @printf(ptr {{%.*}}, ptr {{%.*}}) for i := c.Int(0); i < c.Argc; i++ { diff --git a/cl/_testlibc/atomic/in.go b/cl/_testlibc/atomic/in.go index 70cc91fa04..78ce26ff1a 100644 --- a/cl/_testlibc/atomic/in.go +++ b/cl/_testlibc/atomic/in.go @@ -13,7 +13,7 @@ func main() { // CHECK: store atomic i64 100, ptr [[ADDR:%[0-9]+]] seq_cst, align 8 atomic.Store(&v, 100) // CHECK: load atomic i64, ptr [[ADDR]] seq_cst, align 8 - // CHECK: call void @__llgo_coro_os_thread_foreign_call_v1 + // CHECK: call i32 @__llgo_coro_os_thread_foreign_call_v1 c.Printf(c.Str("store: %ld\n"), atomic.Load(&v)) // CHECK: atomicrmw add ptr [[ADDR]], i64 1 seq_cst, align 8 ret := atomic.Add(&v, 1) diff --git a/cl/_testlibc/sqlite/in.go b/cl/_testlibc/sqlite/in.go index dae2bcfc50..e5e770f23d 100644 --- a/cl/_testlibc/sqlite/in.go +++ b/cl/_testlibc/sqlite/in.go @@ -9,7 +9,7 @@ import ( // CHECK-LABEL: define ptr @"main.check$coro"( func check(err sqlite.Errno) { if err != sqlite.OK { - // CHECK: call void @__llgo_coro_os_thread_foreign_call_v1 + // CHECK: call i32 @__llgo_coro_os_thread_foreign_call_v1 c.Printf(c.Str("==> Error: (%d) %s\n"), err, err.Errstr()) c.Exit(1) } @@ -22,7 +22,7 @@ func main() { db, err := sqlite.OpenV2(c.Str(":memory:"), sqlite.OpenReadWrite|sqlite.OpenMemory, nil) check(err) - // CHECK: call void @__llgo_coro_os_thread_foreign_call_v1 + // CHECK: call i32 @__llgo_coro_os_thread_foreign_call_v1 db.Close() } diff --git a/cl/cgo_test.go b/cl/cgo_test.go index e5de5b9f91..d0a80b4185 100644 --- a/cl/cgo_test.go +++ b/cl/cgo_test.go @@ -5,16 +5,19 @@ package cl import ( "go/ast" + "go/build" "go/constant" "go/importer" "go/parser" "go/token" "go/types" + "path/filepath" "runtime" "strings" "testing" "github.com/goplus/gogen/packages" + llpackages "github.com/goplus/llgo/internal/packages" llssa "github.com/goplus/llgo/ssa" "github.com/xgo-dev/llvm" gossa "golang.org/x/tools/go/ssa" @@ -211,6 +214,144 @@ func findStaticCall(t *testing.T, fn *gossa.Function, name string) *gossa.Call { return nil } +func TestCgoGeneratedTouchIntrinsicRequiresExactDeclarationAndGuard(t *testing.T) { + ssaPkg, fset, files := buildGoSSAPkg(t, ` +package foo + +import _ "unsafe" + +//go:linkname _Cgo_always_false runtime.cgoAlwaysFalse +var _Cgo_always_false bool + +//go:linkname _Cgo_use runtime.cgoUse +func _Cgo_use(any) + +//go:linkname _Cgo_keepalive runtime.cgoKeepAlive +func _Cgo_keepalive(any) + +func Guarded(value any) { + if _Cgo_always_false { + _Cgo_use(value) + } +} + +func GuardedKeepAlive(value any) { + if _Cgo_always_false { + _Cgo_keepalive(value) + } +} + +func Unguarded(value any) { _Cgo_use(value) } +`) + prog := newLLSSAProg(t) + defer prog.Dispose() + if err := ParsePkgSyntax(prog, fset, ssaPkg.Pkg, files); err != nil { + t.Fatal(err) + } + ctx := &context{prog: prog, goTyps: ssaPkg.Pkg} + for _, test := range []struct { + declaration string + wantName string + opcode int + caller string + }{ + {declaration: "_Cgo_use", wantName: "cgoUse", opcode: llgoCgoUse, caller: "Guarded"}, + {declaration: "_Cgo_keepalive", wantName: "cgoKeepAlive", opcode: llgoCgoKeepAlive, caller: "GuardedKeepAlive"}, + } { + declaration := ssaPkg.Func(test.declaration) + _, name, kind := ctx.funcName(declaration) + if name != test.wantName || kind != llgoInstr || llgoInstrs[name] != test.opcode { + t.Fatalf("%s classification = (%q, %d/%d), want (%q, llgoInstr/%d)", + test.declaration, name, kind, llgoInstrs[name], test.wantName, test.opcode) + } + call := findStaticCall(t, ssaPkg.Func(test.caller), test.declaration) + if err := verifyCoroGeneratedTouchCall(prog, test.opcode, call); err != nil { + t.Fatalf("verify %s generated guard: %v", test.declaration, err) + } + } + unguarded := findStaticCall(t, ssaPkg.Func("Unguarded"), "_Cgo_use") + if err := verifyCoroGeneratedTouchCall(prog, llgoCgoUse, unguarded); err == nil || + !strings.Contains(err.Error(), "false-global guard") { + t.Fatalf("unguarded cgo touch error = %v", err) + } +} + +func TestCgoGeneratedTouchMetadataFromCmdCgo(t *testing.T) { + if !build.Default.CgoEnabled { + t.Skip("cmd/cgo metadata requires cgo") + } + fset := token.NewFileSet() + prog := newLLSSAProg(t) + defer prog.Dispose() + dedup := llpackages.NewDeduper() + var preloadErr error + dedup.SetPreload(func(pkg *types.Package, files []*ast.File) { + if preloadErr != nil { + return + } + preloadErr = ParsePkgSyntax(prog, fset, pkg, files) + }) + loaded, err := llpackages.LoadEx(dedup, nil, &llpackages.Config{ + Mode: llpackages.NeedName | llpackages.NeedFiles | + llpackages.NeedCompiledGoFiles | llpackages.NeedImports | + llpackages.NeedDeps | llpackages.NeedTypes | + llpackages.NeedTypesSizes | llpackages.NeedSyntax | + llpackages.NeedTypesInfo, + Dir: filepath.Join("_testgo", "cgobasic"), + Fset: fset, + }, ".") + if err != nil { + t.Fatal(err) + } + if preloadErr != nil { + t.Fatal(preloadErr) + } + if len(loaded) != 1 || loaded[0].Types == nil { + t.Fatalf("loaded packages = %d, want one typed package", len(loaded)) + } + fullName := llssa.FullName(loaded[0].Types, "_Cgo_use") + if linkname, ok := prog.Linkname(fullName); !ok || linkname != "runtime.cgoUse" { + files := make([]string, len(loaded[0].CompiledGoFiles)) + for index, file := range loaded[0].CompiledGoFiles { + files[index] = filepath.Base(file) + } + t.Fatalf("linkname %q = (%q, %v), want runtime.cgoUse; compiled files: %v", + fullName, linkname, ok, files) + } + goProg, ssaPackages := ssautil.AllPackages(loaded, + gossa.SanityCheckFunctions|gossa.InstantiateGenerics) + goProg.Build() + if len(ssaPackages) != 1 || ssaPackages[0] == nil { + t.Fatalf("SSA packages = %d, want one package", len(ssaPackages)) + } + declaration := ssaPackages[0].Func("_Cgo_use") + ctx := &context{prog: prog, goTyps: loaded[0].Types} + _, name, kind := ctx.funcName(declaration) + if name != "cgoUse" || kind != llgoInstr { + t.Fatalf("real cmd/cgo _Cgo_use classification = (%q, %d), want (cgoUse, llgoInstr)", + name, kind) + } + for _, member := range ssaPackages[0].Members { + function, ok := member.(*gossa.Function) + if !ok || !strings.HasPrefix(function.Name(), "_Cfunc_") { + continue + } + for _, block := range function.Blocks { + for _, instruction := range block.Instrs { + call, ok := instruction.(*gossa.Call) + if !ok || call.Call.StaticCallee() != declaration { + continue + } + if err := verifyCoroGeneratedTouchCall(prog, llgoCgoUse, call); err != nil { + t.Fatalf("verify real cmd/cgo touch in %s: %v", function.Name(), err) + } + return + } + } + } + t.Fatal("real cmd/cgo output contains no _Cgo_use call") +} + func TestCgoCgocall_InitArgsFromParams(t *testing.T) { ssaPkg, _, _ := buildGoSSAPkg(t, ` package foo @@ -348,6 +489,11 @@ func TestCgoC2Return_ErrnoNeedsConvert(t *testing.T) { prog := newLLSSAProg(t) pkg := prog.NewPackage("foo", "foo") errType := types.Universe.Lookup("error").Type() + syscallPkg, err := importer.Default().Import("syscall") + if err != nil { + t.Fatal(err) + } + errnoType := syscallPkg.Scope().Lookup("Errno").Type() sig := types.NewSignatureType(nil, nil, nil, nil, types.NewTuple( types.NewVar(0, nil, "", types.Typ[types.Int]), @@ -358,7 +504,7 @@ func TestCgoC2Return_ErrnoNeedsConvert(t *testing.T) { b := fn.MakeBody(1) ctx := &context{prog: prog, pkg: pkg, fn: fn} - ctx.cgoErrnoTy = types.Typ[types.Int32] // avoid needing goProg for lookup + ctx.cgoErrnoTy = errnoType // avoid needing goProg while retaining the real error implementation ctx.cgoErrno = b.Const(constant.MakeInt64(1), ctx.type_(types.Typ[types.Int64], llssa.InGo)) ret := b.Const(constant.MakeInt64(7), ctx.type_(types.Typ[types.Int], llssa.InGo)) diff --git a/cl/compile.go b/cl/compile.go index e349d34259..0bdc2122ca 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -209,6 +209,7 @@ type context struct { vargs map[*ssa.Alloc][]llssa.Expr // varargs funcs map[*ssa.Function]llssa.Function rawPlainFuncs map[*ssa.Function]llssa.Function + outcomePlainFuncs map[*ssa.Function]llssa.Function linkOnceFns map[*ssa.Function]none stackDefers map[*ssa.Function]bool anonDefers map[*ssa.Function]bool @@ -226,6 +227,11 @@ type context struct { coroEmission *coroPhysicalEmissionSession coroPlainSite *coroSiteEmissionObserver rawPlainBody bool // compiling the legacy ABI variant of a managed function + // physicalReachable is the owner-scoped constant-CFG projection used by + // every separately emitted managed body. A full coroutine already carries + // it in coroEmissionPlan; ordinary/outcome-independent bodies use this copy + // so helper inventory, analysis, and codegen agree about dead blocks. + physicalReachable map[*ssa.BasicBlock]bool coroOwnerBodySymbols map[string]none // preservePatchedNamed keeps an alternate package's named type intact // while constructing source-level ABI certificates. Ordinary codegen @@ -678,9 +684,31 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun // declaration behind. p.compileFuncDeclVariant(pkg, entry.function, true) } + if entry.planned && entry.plan.Emission == coro.EmitCoroutine && + entry.plan.HasStaticOutcome() { + p.compileOutcomePlainFunction(entry.function) + } return fn, py, kind } +func (p *context) compileOutcomePlainFunction(v *ssa.Function) (goFn llssa.Function, pyFn llssa.PyObjRef, kind int) { + if v == nil || p.compilation == nil || p.immutablePlan() == nil || p.immutableEmissionUniverse() == nil { + panic("outcome-plain function resolution requires an exact function, emission universe, and coroutine plan") + } + canonical, ok := p.immutableEmissionUniverse().Resolve(v) + if !ok || canonical == nil { + panic(fmt.Errorf("outcome-plain function resolution: function %q is absent from the prepared emission universe", v.Name())) + } + entry := p.mustOutcomePlainFunctionSymbol(canonical) + if entry.ftype != goFunc { + return p.funcOfEntry(entry) + } + if p.ownsFunctionEmission(canonical) { + return p.compileFuncDeclVariantEntry(p.pkg, entry, false) + } + return p.funcOfEntry(entry) +} + // compileFuncDeclVariant materializes either the managed primary or the exact // legacy Go-ABI body requested by RawPlainEntry. The SSA CFG is shared, but the // latter deliberately runs through ordinary native-stack lowering: no @@ -826,7 +854,9 @@ func (p *context) compileFuncDeclVariantEntry(pkg llssa.Package, entry plannedFu if noInlineDirective || runtimeStackNoInline || pcLineNoInline { fn.DisableTailCalls() } - if rawPlain { + if entry.outcomePlainTwin { + p.outcomePlainFuncs[f] = fn + } else if rawPlain { p.rawPlainFuncs[f] = fn } else { p.funcs[f] = fn @@ -845,7 +875,7 @@ func (p *context) compileFuncDeclVariantEntry(pkg llssa.Package, entry plannedFu pkg.EmitFuncInfo(fn.Name(), funcInfoDisplayName(goName), pos.Filename, pos.Line, pos.Column) } var childInits []func() - if !rawPlain && len(f.AnonFuncs) > 0 { + if !rawPlain && !entry.outcomePlainTwin && len(f.AnonFuncs) > 0 { parentInits := p.inits p.inits = nil for _, af := range f.AnonFuncs { @@ -877,16 +907,19 @@ func (p *context) compileFuncDeclVariantEntry(pkg llssa.Package, entry plannedFu dbgSymsEnabled := p.frontendOptions().DebugSymbols && (f == nil || f.Origin() == nil) p.inits = append(p.inits, func() { oldFn, oldGoFn, oldMethodNilDerefChecks, oldPatchOriginalInitIf, oldUnevaluatedSSA, oldRawPlainBody := p.fn, p.goFn, p.methodNilDerefChecks, p.patchOriginalInitIf, p.unevaluatedSSA, p.rawPlainBody + oldPhysicalReachable := p.physicalReachable oldLocalityFunction := p.locality.function p.fn = fn p.goFn = f p.patchOriginalInitIf = patchOriginalInitIf p.rawPlainBody = rawPlain + p.physicalReachable = coroPhysicalConstantReachableBlocks(f) p.locality.function = localityFunction{} p.state = state // restore pkgState when compiling funcBody oldCoroValueAddrs := p.coroValueAddrs defer func() { p.fn, p.goFn, p.methodNilDerefChecks, p.patchOriginalInitIf, p.unevaluatedSSA, p.rawPlainBody = oldFn, oldGoFn, oldMethodNilDerefChecks, oldPatchOriginalInitIf, oldUnevaluatedSSA, oldRawPlainBody + p.physicalReachable = oldPhysicalReachable p.coroValueAddrs = oldCoroValueAddrs p.locality.function = oldLocalityFunction }() @@ -1188,6 +1221,10 @@ func (p *context) compileBlock(b llssa.Builder, block *ssa.BasicBlock, n int, do var instrs = block.Instrs[n:] var ret = p.sourceBlock(block.Index) b.SetBlock(ret) + if !p.coroSourceBlockReachable(block) { + b.Unreachable() + return ret + } if block.Index == 0 { p.emitFunctionPreambleWithCoroPlan(b, block.Parent()) } @@ -2019,6 +2056,9 @@ func isPhi(i ssa.Instruction) bool { func (p *context) compilePhis(b llssa.Builder, block *ssa.BasicBlock) int { ret := p.sourceBlock(block.Index) b.SetBlockEx(ret, llssa.AtEnd, false) + if !p.coroSourceBlockReachable(block) { + return 0 + } if ninstr := len(block.Instrs); ninstr > 0 { if isPhi(block.Instrs[0]) { n := 1 @@ -2053,11 +2093,22 @@ func (p *context) compilePhi(b llssa.Builder, v *ssa.Phi) (ret llssa.Expr) { finishSite := p.beginCoroSemanticInstructionEmission(v) defer finishSite() preds := v.Block().Preds + edges := v.Edges + if p.coroEmissionPlan() != nil || p.physicalReachable != nil { + livePreds := make([]*ssa.BasicBlock, 0, len(preds)) + liveEdges := make([]ssa.Value, 0, len(edges)) + for index, pred := range preds { + if p.coroSourceBlockReachable(pred) { + livePreds = append(livePreds, pred) + liveEdges = append(liveEdges, edges[index]) + } + } + preds, edges = livePreds, liveEdges + } bblks := make([]llssa.BasicBlock, len(preds)) for i, pred := range preds { bblks[i] = p.sourceBlock(pred.Index) } - edges := v.Edges phi.AddIncoming(b, bblks, func(i int, blk llssa.BasicBlock) llssa.Expr { b.SetBlockEx(blk, llssa.BeforeLast, false) return p.compileValue(b, edges[i]) @@ -2066,6 +2117,19 @@ func (p *context) compilePhi(b llssa.Builder, v *ssa.Phi) (ret llssa.Expr) { return } +func (p *context) coroSourceBlockReachable(block *ssa.BasicBlock) bool { + if block == nil { + return false + } + if physical := p.coroEmissionPlan(); physical != nil { + return physical.reachableBlocks[block] + } + if p.physicalReachable != nil { + return p.physicalReachable[block] + } + return true +} + // beginCoroSemanticInstructionEmission is the single source-instruction // boundary shared by ordinary instruction emission and Phi incoming-edge // materialization. Phi nodes are declared before their operands can be @@ -2419,10 +2483,19 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue ret = p.compileCoroTerminalResultAllocation(v) case coroPhysicalInstructionFrameBitcastAllocation: observePhysical(coroPhysicalInstructionFrameBitcastAllocation) - ret = p.coroFrameAlloca(elem) + ret = p.structuredOutcomeAlloca(elem, false) case coroPhysicalInstructionFrameAllocation: observePhysical(coroPhysicalInstructionFrameAllocation) - ret = p.coroFrameAlloc(elem) + ret = p.structuredOutcomeAlloca(elem, true) + case coroPhysicalInstructionBorrowedAllocation: + observePhysical(coroPhysicalInstructionBorrowedAllocation) + // The storage itself belongs to the entry/native frame, but a + // Heap Alloc denotes a fresh zeroed object every time its source + // instruction executes. Reinitialize here so loops and conditional + // declarations retain their Go semantics while reusing the proven + // non-escaping physical slot. + ret = p.structuredOutcomeAlloca(elem, false) + b.Store(ret, p.prog.Zero(elem)) case coroPhysicalInstructionOrdinary: default: panic(fmt.Sprintf("Alloc selected incompatible frozen physical recipe %s", physicalInstruction.recipe)) @@ -2432,6 +2505,15 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue break } } + if p.selectCoroPlainBorrowedAllocation(v) { + // The frozen SitePlan proves that every static callee borrows this + // exact address only until return. Builder.Alloc retains the ordinary + // source-point zeroing and target stack-limit check while changing the + // conservative x/tools Heap decision to native local storage. + ret = b.Alloc(elem, false) + p.debugAlloc(b, v, ret) + break + } exactBitcast := false if !physicalPlanned { bitcast, exact := coro.ProveSSAExactScalarBitcast(v.Parent()) @@ -2636,7 +2718,11 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue } } x := p.compileValue(b, v.X) - ret = b.MakeInterface(t, x) + if _, constant := v.X.(*ssa.Const); constant { + ret = b.MakeInterfaceFromConstant(t, x) + } else { + ret = b.MakeInterface(t, x) + } case *ssa.MakeSlice: t := p.type_(v.Type(), llssa.InGo) nLen := p.compileValue(b, v.Len) @@ -3127,6 +3213,12 @@ func (p *context) compileInstr(b llssa.Builder, instr ssa.Instruction) { p.emitPCLineLabel(b, p.deferRunPos(v.Pos())) outcome, outcomePlanned := p.plannedCoroPhysicalOutcome(v) if outcomePlanned { + if p.hasStructuredOutcomePhysicalBody() && outcome.semantic.evaluated && + outcome.outcome == coroPhysicalOutcomeNone { + // The static-outcome proof found no reachable Defer + // registration. x/tools' synthetic RunDefers is a no-op. + return + } if outcome.outcome != coroPhysicalOutcomeRunDefers { panic(fmt.Sprintf("RunDefers selected incompatible frozen physical outcome recipe %s", outcome.outcome)) } @@ -3521,6 +3613,9 @@ func (p *context) deferRunPos(fallback token.Pos) token.Pos { } func (p *context) returnNeedsImplicitRunDefers(ret *ssa.Return) bool { + if physical := p.coroEmissionPlan(); physical != nil && physical.cleanup == nil { + return false + } fn := ret.Parent() if fn == nil || fn.Synthetic != "" || ret.Block() == fn.Recover { return false @@ -3775,21 +3870,22 @@ func newPackageEx(prog llssa.Program, ct *CallerTracking, patches Patches, rewri ct = NewCallerTracking() } ctx := &context{ - prog: prog, - pkg: ret, - fset: pkgProg.Fset, - goProg: pkgProg, - goTyps: pkgTypes, - goPkg: pkg, - patches: patches, - options: options, - optionsSet: true, - skips: make(map[string]none), - vargs: make(map[*ssa.Alloc][]llssa.Expr), - funcs: make(map[*ssa.Function]llssa.Function), - rawPlainFuncs: make(map[*ssa.Function]llssa.Function), - linkOnceFns: make(map[*ssa.Function]none), - addrOfFieldAddrs: collectAddrOfFieldSelectors(files), + prog: prog, + pkg: ret, + fset: pkgProg.Fset, + goProg: pkgProg, + goTyps: pkgTypes, + goPkg: pkg, + patches: patches, + options: options, + optionsSet: true, + skips: make(map[string]none), + vargs: make(map[*ssa.Alloc][]llssa.Expr), + funcs: make(map[*ssa.Function]llssa.Function), + rawPlainFuncs: make(map[*ssa.Function]llssa.Function), + outcomePlainFuncs: make(map[*ssa.Function]llssa.Function), + linkOnceFns: make(map[*ssa.Function]none), + addrOfFieldAddrs: collectAddrOfFieldSelectors(files), loaded: map[*types.Package]*pkgInfo{ types.Unsafe: {kind: PkgDeclOnly}, // TODO(xsw): PkgNoInit or PkgDeclOnly? }, diff --git a/cl/coro_abi.go b/cl/coro_abi.go index a076417ac4..a2a78d6a0a 100644 --- a/cl/coro_abi.go +++ b/cl/coro_abi.go @@ -122,28 +122,34 @@ const ( coroFrameFreeHook = "__llgo_coro_frame_free_v0" coroDescriptorPrefix = "__llgo_coro_frame_descriptor_v0." - coroPhysicalABIVersionV1 uint32 = 1 - coroFrameAllocHookV1 = "__llgo_coro_frame_alloc_v1" - coroFramePublishHookV1 = "__llgo_coro_frame_publish_v1" - coroAwaitPrepareHookV1 = "__llgo_coro_await_prepare_v3" - coroAwaitInlineHookV1 = "__llgo_coro_await_inline_v1" - coroAwaitConsumeHookV1 = "__llgo_coro_await_consume_v1" - coroPreemptPollHookV1 = "__llgo_coro_preempt_poll_v1" - coroYieldPrepareHookV1 = "__llgo_coro_yield_prepare_v1" - coroCriticalEnterHookV1 = "__llgo_coro_critical_enter_v1" - coroCriticalExitHookV1 = "__llgo_coro_critical_exit_v1" - coroKeyedParkHookV2 = "__llgo_coro_keyed_park_v2" - coroKeyedResumeHookV2 = "__llgo_coro_keyed_resume_v2" - coroRunDecisionTakeHookV1 = "__llgo_coro_run_decision_take_v1" - coroRunDecisionTakeZeroHookV1 = "__llgo_coro_run_decision_take_zero_v1" - coroPanicPrepareHookV1 = "__llgo_coro_panic_prepare_v1" - coroPanicTraceReplaceHookV1 = "__llgo_coro_panic_trace_replace_v1" - coroRecoverTakeHookV1 = "__llgo_coro_recover_take_v1" - coroSpawnBeginHookV1 = "__llgo_coro_spawn_begin_v1" - coroSpawnCommitHookV1 = "__llgo_coro_spawn_commit_v1" - coroCompletePrepareHookV2 = "__llgo_coro_complete_prepare_v2" - coroFrameFreeHookV1 = "__llgo_coro_frame_free_v1" - coroDescriptorPrefixV1 = "__llgo_coro_frame_descriptor_v1." + coroPhysicalABIVersionV1 uint32 = 1 + coroFrameAllocHookV1 = "__llgo_coro_frame_alloc_v1" + coroFramePublishHookV1 = "__llgo_coro_frame_publish_v1" + coroFramePublishHookV3 = "__llgo_coro_frame_publish_v3" + coroAwaitPrepareHookV1 = "__llgo_coro_await_prepare_v3" + coroAwaitInlineHookV1 = "__llgo_coro_await_inline_v1" + coroAwaitInlineBeginHookV2 = "__llgo_coro_await_inline_begin_v2" + coroAwaitInlineFinishHookV2 = "__llgo_coro_await_inline_finish_v2" + coroAwaitInlineDestroyCommitHookV2 = "__llgo_coro_await_inline_destroy_commit_v2" + coroFrameDestroyCommitHookV2 = "__llgo_coro_frame_destroy_commit_v2" + coroAwaitConsumeHookV1 = "__llgo_coro_await_consume_v1" + coroPreemptPollHookV1 = "__llgo_coro_preempt_poll_v1" + coroYieldPrepareHookV1 = "__llgo_coro_yield_prepare_v1" + coroCriticalEnterHookV1 = "__llgo_coro_critical_enter_v1" + coroCriticalExitHookV1 = "__llgo_coro_critical_exit_v1" + coroKeyedParkHookV2 = "__llgo_coro_keyed_park_v2" + coroKeyedResumeHookV2 = "__llgo_coro_keyed_resume_v2" + coroRunDecisionTakeHookV1 = "__llgo_coro_run_decision_take_v1" + coroRunDecisionTakeZeroHookV1 = "__llgo_coro_run_decision_take_zero_v1" + coroPanicPrepareHookV1 = "__llgo_coro_panic_prepare_v1" + coroPanicTraceReplaceHookV1 = "__llgo_coro_panic_trace_replace_v1" + coroRecoverTakeHookV1 = "__llgo_coro_recover_take_v1" + coroSpawnBeginHookV1 = "__llgo_coro_spawn_begin_v1" + coroSpawnCommitHookV1 = "__llgo_coro_spawn_commit_v1" + coroCompletePrepareHookV2 = "__llgo_coro_complete_prepare_v2" + coroFrameFreeHookV1 = "__llgo_coro_frame_free_v1" + coroDescriptorPrefixV1 = "__llgo_coro_frame_descriptor_v1." + coroBorrowedFrameMetadataWordsV2 = 20 ) const ( @@ -190,14 +196,17 @@ const ( const coroPreemptInstructionBudget = 64 // coroPreemptCheckpointStride is the number of compiler-selected source -// safepoints between full runtime polls. While a frame is active its existing -// StateID word is compiler-private countdown storage; publishState overwrites -// it with the real resume state before every scheduler-visible suspension. -const coroPreemptCheckpointStride uint64 = 64 +// safepoints between full runtime polls. The countdown lives in compiler-local +// storage whose address never escapes; the ordinary SROA/mem2reg pipeline can +// therefore keep it in SSA registers on a non-suspending loop edge. Every +// activation resets it, so it is not live across a scheduler-visible suspend. +// StateID remains exclusively the published resume-state identity. +const coroPreemptCheckpointStride uint64 = 2048 type coroPhysicalABI struct { version uint32 hash [16]byte + descriptorFlags uint32 descriptorName string traceFunction string traceFile string @@ -206,6 +215,9 @@ type coroPhysicalABI struct { framePublishHook string awaitPrepareHook string awaitInlineHook string + awaitInlineFinishHook string + awaitInlineCommitHook string + frameDestroyCommitHook string awaitConsumeHook string preemptPollHook string yieldPrepareHook string @@ -250,6 +262,7 @@ type coroBodyContext struct { panicTraceReplace llssa.Expr completePrepare llssa.Expr terminalStatus llssa.Expr + preemptCountdown llssa.Expr nextState uint32 terminalState uint32 needsPreempt bool @@ -278,12 +291,27 @@ func newCoroPhysicalABI(p *context, entry plannedFunctionSymbol, sourceSig *type hasEnv = hidden == 1 } version := coroPhysicalABIVersionV1 + descriptorFlags := uint32(0) + // Absence of an execution flag is a capability only for a closed managed + // body. Unknown/open execution and any propagated getg/reentry primitive + // retain the ordinary per-resume runtime-context install. Blocking, + // affinity, preemption, and IRQ eligibility are independent scheduler + // dimensions; treating them as ambient-G requirements would discard an + // otherwise valid proof. Every remaining Go call is covered by the + // fixed-point plan (and by imported library summaries). + const contextUnproven = coro.NeedsRuntimeContext | coro.OpaqueExec + if entry.plan.External == coro.Defined && entry.plan.Exec&contextUnproven == 0 { + descriptorFlags |= coro.FrameDescriptorNoRuntimeContextV1 + } frameAllocHook := coroFrameAllocHookV1 frameFreeHook := coroFrameFreeHookV1 descriptorPrefix := coroDescriptorPrefixV1 - framePublishHook := coroFramePublishHookV1 + framePublishHook := coroFramePublishHookV3 awaitPrepareHook := coroAwaitPrepareHookV1 - awaitInlineHook := coroAwaitInlineHookV1 + awaitInlineHook := coroAwaitInlineBeginHookV2 + awaitInlineFinishHook := coroAwaitInlineFinishHookV2 + awaitInlineCommitHook := coroAwaitInlineDestroyCommitHookV2 + frameDestroyCommitHook := coroFrameDestroyCommitHookV2 awaitConsumeHook := coroAwaitConsumeHookV1 preemptPollHook := coroPreemptPollHookV1 yieldPrepareHook := coroYieldPrepareHookV1 @@ -359,9 +387,10 @@ func newCoroPhysicalABI(p *context, entry plannedFunctionSymbol, sourceSig *type } } key := fmt.Sprintf( - "llgo-coro-physical-v%d\x00%s\x00trace-function=%s\x00trace-file=%s\x00coro=%s\x00scheduler=%s\x00panic=%s\x00panic-hook=%s\x00panic-trace-replace=%s\x00recover-take=%s\x00fault-hook=%s\x00fault-payload-hook=%s\x00fault-args-hook=%s\x00fault-args-payload-hook=%s\x00fault-args-abi=x64-yword-v2\x00func-rep=%s\x00await-prepare=%s\x00await-inline=%s\x00await-consume=%s\x00resume-decision=%s\x00resume-decision-zero=%s\x00critical-enter=%s\x00critical-exit=%s\x00preempt-stride=%d\x00os-thread-lock=%s\x00os-thread-unlock=%s\x00triple=%s\x00cpu=%s\x00features=%s\x00target-abi=%s\x00data-layout=%s\x00ptr=%d\x00sig=%s\x00result=%s", + "llgo-coro-physical-v%d\x00%s\x00descriptor-flags=%#x\x00trace-function=%s\x00trace-file=%s\x00coro=%s\x00scheduler=%s\x00panic=%s\x00panic-hook=%s\x00panic-trace-replace=%s\x00recover-take=%s\x00fault-hook=%s\x00fault-payload-hook=%s\x00fault-args-hook=%s\x00fault-args-payload-hook=%s\x00fault-args-abi=x64-yword-v2\x00func-rep=%s\x00frame-publish=%s\x00await-prepare=%s\x00await-inline=%s\x00await-inline-finish=%s\x00await-inline-commit=%s\x00frame-destroy-commit=%s\x00await-consume=%s\x00resume-decision=%s\x00resume-decision-zero=%s\x00critical-enter=%s\x00critical-exit=%s\x00preempt-stride=%d\x00os-thread-lock=%s\x00os-thread-unlock=%s\x00triple=%s\x00cpu=%s\x00features=%s\x00target-abi=%s\x00data-layout=%s\x00ptr=%d\x00sig=%s\x00result=%s", version, entry.plan.ID, + descriptorFlags, traceFunction, traceFile, coroABI, @@ -375,8 +404,12 @@ func newCoroPhysicalABI(p *context, entry plannedFunctionSymbol, sourceSig *type faultPrepareArgsHook, faultPayloadArgsHook, funcRepABI, + framePublishHook, awaitPrepareHook, awaitInlineHook, + awaitInlineFinishHook, + awaitInlineCommitHook, + frameDestroyCommitHook, awaitConsumeHook, runDecisionTakeHook, runDecisionTakeZeroHook, @@ -400,6 +433,7 @@ func newCoroPhysicalABI(p *context, entry plannedFunctionSymbol, sourceSig *type return coroPhysicalABI{ version: version, hash: hash, + descriptorFlags: descriptorFlags, descriptorName: descriptorPrefix + hex.EncodeToString(hash[:]), traceFunction: traceFunction, traceFile: traceFile, @@ -408,6 +442,9 @@ func newCoroPhysicalABI(p *context, entry plannedFunctionSymbol, sourceSig *type framePublishHook: framePublishHook, awaitPrepareHook: awaitPrepareHook, awaitInlineHook: awaitInlineHook, + awaitInlineFinishHook: awaitInlineFinishHook, + awaitInlineCommitHook: awaitInlineCommitHook, + frameDestroyCommitHook: frameDestroyCommitHook, awaitConsumeHook: awaitConsumeHook, preemptPollHook: preemptPollHook, yieldPrepareHook: yieldPrepareHook, @@ -451,6 +488,7 @@ func (p *context) beginCoroBody( descriptor := p.pkg.NewCoroFrameDescriptor(abi.descriptorName, llssa.CoroFrameDescriptorOptions{ Version: abi.version, ABIHash: abi.hash, + Flags: abi.descriptorFlags, Result: resultType, Function: abi.traceFunction, File: abi.traceFile, @@ -458,25 +496,17 @@ func (p *context) beginCoroBody( descriptorPtr := b.Convert(prog.VoidPtr(), descriptor) task := p.fn.PhysicalParam(0) resultSlot := p.fn.PhysicalParam(1) - null := prog.Nil(prog.VoidPtr()) headerType := coroHeaderType(prog) header := b.AllocaT(headerType) - initialLifecycle := uint64(coroLifecycleAllocated) - if abi.version >= coroPhysicalABIVersionV1 { - initialLifecycle = coroLifecycleInitialSuspended - } - headerValues := []llssa.Expr{ - task, - null, - descriptorPtr, - null, - resultSlot, - prog.IntVal(coroSuspendNone, prog.Uint16()), - prog.IntVal(initialLifecycle, prog.Uint16()), - prog.IntVal(0, prog.Uint32()), - prog.IntVal(0, prog.Uint32()), - prog.IntVal(0, prog.Uint32()), - } + borrowedFrameMetadataType := p.type_( + types.NewArray(types.Typ[types.Uintptr], coroBorrowedFrameMetadataWordsV2), + llssa.InGo, + ) + // Dynamic ramps never consume this fallback storage. Leave it uninitialized + // here so every ordinary coroutine creation does not pay a 20-word memset; + // PublishFrameV2 initializes the complete private Frame only when LLVM has + // actually selected the allocation-elided path (storage == nil). + borrowedFrameMetadata := b.AllocaT(borrowedFrameMetadataType) allocSig := coroFrameAllocSignature(abi.version) freeSig := coroFrameFreeSignature(abi.version) alloc := p.pkg.NewFunc(abi.frameAllocHook, allocSig, llssa.InC) @@ -490,6 +520,10 @@ func (p *context) beginCoroBody( }, Free: func(b llssa.Builder, storage, size, align llssa.Expr) { if abi.version >= coroPhysicalABIVersionV1 { + // The fake use is removed after CoroSplit. Until then it makes the + // compiler-injected scheduler metadata live through final cleanup, + // so an elided child owns it inside its static parent's LLVM frame. + b.KeepAlive(borrowedFrameMetadata) b.Call(free.Expr, task, storage, size, align, descriptorPtr) return } @@ -510,6 +544,11 @@ func (p *context) beginCoroBody( // managed calls and receive their ordinary Return outcomes. body.terminalStatus = b.AllocaT(prog.Uint32()) b.Store(body.terminalStatus, prog.IntVal(coroAwaitCompletionReturn, prog.Uint32())) + // This address is compiler-private and never reaches a runtime call. It + // deliberately differs from Header.StateID: that externally visible + // field aliases runtime validation calls and therefore forces a + // load/store on every otherwise plain loop edge. + body.preemptCountdown = b.AllocaT(prog.Uint32()) } if abi.runDecisionTakeZeroHook != "" { body.runDecisionTakeZero = p.pkg.NewFunc( @@ -549,12 +588,18 @@ func (p *context) beginCoroBody( Promise: header, Frame: frame, BeforeInitialSuspend: func(b llssa.Builder, handle, storage llssa.Expr) { - for i, value := range headerValues { - b.Store(b.FieldAddr(header, i), value) - } if abi.framePublishHook != "" { publish := p.pkg.NewFunc(abi.framePublishHook, coroFramePublishSignature(), llssa.InC) - b.Call(publish.Expr, task, handle, b.Convert(prog.VoidPtr(), header), storage) + b.Call( + publish.Expr, + task, + handle, + b.Convert(prog.VoidPtr(), header), + storage, + b.Convert(prog.VoidPtr(), borrowedFrameMetadata), + descriptorPtr, + resultSlot, + ) } // A named result captured by a defer is an ordinary Go heap object, // but x/tools reloads it from compiler-owned RunDefers continuations. @@ -585,7 +630,18 @@ func (p *context) beginCoroBody( }, } if !body.runDecisionTakeZero.IsNil() { - coroOptions.AfterResumeDispatch = body.dispatchZeroRunDecision + coroOptions.AfterResumeDispatch = func(b llssa.Builder, normal llssa.BasicBlock) { + // Publication lets the scheduler retain this opaque address after the + // ramp returns. Keep one resumed use until CoroSplit has physically + // placed the storage in the LLVM frame; the post-split cleanup removes + // the fake use before instruction selection. + b.KeepAlive(borrowedFrameMetadata) + body.dispatchZeroRunDecision(b, normal) + } + } else { + coroOptions.AfterResume = func(b llssa.Builder) { + b.KeepAlive(borrowedFrameMetadata) + } } body.coro = b.BeginCoro(coroOptions) if body.unsupportedRunDecision != nil { @@ -633,6 +689,9 @@ func coroFramePublishSignature() *types.Signature { types.NewParam(token.NoPos, nil, "handle", types.Typ[types.UnsafePointer]), types.NewParam(token.NoPos, nil, "header", types.Typ[types.UnsafePointer]), types.NewParam(token.NoPos, nil, "storage", types.Typ[types.UnsafePointer]), + types.NewParam(token.NoPos, nil, "metadata", types.Typ[types.UnsafePointer]), + types.NewParam(token.NoPos, nil, "descriptor", types.Typ[types.UnsafePointer]), + types.NewParam(token.NoPos, nil, "resultSlot", types.Typ[types.UnsafePointer]), ) return types.NewSignatureType(nil, nil, nil, params, nil, false) } @@ -660,6 +719,37 @@ func coroAwaitInlineSignature() *types.Signature { return types.NewSignatureType(nil, nil, nil, params, results, false) } +func coroAwaitInlineFinishSignature() *types.Signature { + pointer := types.Typ[types.UnsafePointer] + params := types.NewTuple( + types.NewParam(token.NoPos, nil, "g", pointer), + types.NewParam(token.NoPos, nil, "parent", pointer), + types.NewParam(token.NoPos, nil, "child", pointer), + types.NewParam(token.NoPos, nil, "done", types.Typ[types.Bool]), + ) + results := types.NewTuple(types.NewParam(token.NoPos, nil, "destroy", types.Typ[types.Bool])) + return types.NewSignatureType(nil, nil, nil, params, results, false) +} + +func coroAwaitInlineCommitSignature() *types.Signature { + pointer := types.Typ[types.UnsafePointer] + params := types.NewTuple( + types.NewParam(token.NoPos, nil, "g", pointer), + types.NewParam(token.NoPos, nil, "parent", pointer), + types.NewParam(token.NoPos, nil, "child", pointer), + ) + return types.NewSignatureType(nil, nil, nil, params, nil, false) +} + +func coroFrameDestroyCommitSignature() *types.Signature { + pointer := types.Typ[types.UnsafePointer] + params := types.NewTuple( + types.NewParam(token.NoPos, nil, "g", pointer), + types.NewParam(token.NoPos, nil, "handle", pointer), + ) + return types.NewSignatureType(nil, nil, nil, params, nil, false) +} + func coroAwaitConsumeSignature() *types.Signature { pointer := types.Typ[types.UnsafePointer] params := types.NewTuple( @@ -773,10 +863,10 @@ func (c *coroBodyContext) activate(b llssa.Builder) { prog := b.Prog b.Store(b.FieldAddr(c.header, coroHeaderSuspendReason), prog.IntVal(coroSuspendNone, prog.Uint16())) b.Store(b.FieldAddr(c.header, coroHeaderLifecycle), prog.IntVal(coroLifecycleActive, prog.Uint16())) - b.Store( - b.FieldAddr(c.header, coroHeaderStateID), - prog.IntVal(coroPreemptCheckpointStride, prog.Uint32()), - ) + if c.preemptCountdown.IsNil() { + panic("coroutine activation has no private preemption countdown") + } + b.Store(c.preemptCountdown, prog.IntVal(coroPreemptCheckpointStride, prog.Uint32())) } // dispatchZeroRunDecision emits the exactly-once compiler resume gate for a @@ -814,7 +904,12 @@ func (c *coroBodyContext) dispatchZeroRunDecisionTo( mapped := b.BinOp(token.ADD, taskKind, b.Prog.IntVal(2, b.Prog.Uint32())) current := b.Load(c.terminalStatus) b.Store(c.terminalStatus, b.SelectValue(isCanceled, mapped, current)) - b.If(isCanceled, canceled, normal) + // Ordinary execution overwhelmingly has no cancellation request. Besides + // describing the scheduler's expected path, this keeps a resumed static + // child call above LLVM 22 CoroAnnotationElide's default 55% block-frequency + // threshold; an unannotated binary branch would make the normal arm look + // artificially cold even though cancellation remains fully reachable. + b.IfWithBranchWeights(isCanceled, canceled, normal, 1, 1000) } func (c *coroBodyContext) bindCancellationCompletion(b llssa.Builder) { @@ -888,12 +983,12 @@ func (c *coroBodyContext) pollAndSuspendForPreempt(b llssa.Builder) uint32 { if c.abi.version < coroPhysicalABIVersionV1 || c.preemptPoll.IsNil() || c.yieldPrepare.IsNil() { panic("coroutine preemption requires PhysicalABIV1 poll and scheduler handoff hooks") } - remaining := b.Load(b.FieldAddr(c.header, coroHeaderStateID)) + if c.preemptCountdown.IsNil() { + panic("coroutine preemption has no private countdown") + } + remaining := b.Load(c.preemptCountdown) one := b.Prog.IntVal(1, b.Prog.Uint32()) - b.Store( - b.FieldAddr(c.header, coroHeaderStateID), - b.BinOp(token.SUB, remaining, one), - ) + b.Store(c.preemptCountdown, b.BinOp(token.SUB, remaining, one)) due := b.BinOp(token.LEQ, remaining, one) stateID := c.nextState // Only the stride boundary reaches a potentially suspending block. The hot @@ -947,7 +1042,9 @@ func (p *context) compileCoroPark(b llssa.Builder, args []llssa.Expr) { } state := b.Convert(b.Prog.VoidPtr(), args[0]) body.emitCoroParkOperation(p, b, coroParkOperation{ - shouldSuspend: b.Prog.BoolVal(true), + prepare: func(active llssa.Builder, _, _ uint32) llssa.Expr { + return active.Prog.BoolVal(true) + }, park: func(suspend llssa.Builder) { park := p.pkg.NewFunc(coroKeyedParkHookV2, coroKeyedParkSignatureV2(), llssa.InC) suspend.Call( @@ -1381,10 +1478,17 @@ func validateCoroPhysicalABIForOwner( fn.Signature != nil && fn.Signature.Recv() == nil rawMethodDispatchToken := rawMethodToken && plan.FuncRep == coro.Dispatch && fn.Signature != nil && fn.Signature.Recv() != nil - outcomePlain := plan.Emission == coro.EmitOutcomePlain && plan.ManagedEntry == coro.ManagedEntryOutcomePlain && - plan.AtomicCostProof.ProvesOutcomePlain() && - plan.AtomicCost != 0 && plan.FuncRep == coro.DirectCoro && !plan.Recursive && - plan.Effect == coro.OutcomeStructured && plan.Exec&^coro.MayUnwind == 0 + outcomePlainPrimary := plan.Emission == coro.EmitOutcomePlain && plan.ManagedEntry == coro.ManagedEntryOutcomePlain + outcomePlainTwin := plan.Emission == coro.EmitCoroutine && plan.ManagedEntry == coro.ManagedEntryCoroutine && + plan.HasStaticOutcome() + outcomePlain := (outcomePlainPrimary || outcomePlainTwin) && + plan.HasStaticOutcome() && !plan.Recursive && + (plan.AtomicCostProof.ProvesOutcomePlain() && plan.AtomicCost != 0 && plan.Exec&^coro.MayUnwind == 0 || + plan.StaticOutcome && plan.Exec&(coro.BlockForeign|coro.ThreadAffine|coro.NeedsCleanupFrame|coro.OpaqueExec) == 0) && + (plan.FuncRep == coro.DirectCoro || outcomePlainTwin && plan.FuncRep == coro.Dispatch) && + (plan.Effect == coro.OutcomeStructured || outcomePlainTwin && + plan.Effect.Contains(coro.OutcomeStructured) && + plan.Effect&^(coro.YieldOnly|coro.AwaitStructured|coro.OutcomeStructured) == 0) if plan.Emission != coro.EmitCoroutine && !outcomePlain || plan.FuncRep != coro.DirectCoro && !managedDispatchTarget && !rawMethodDispatchToken { return fail("requires a direct coroutine/outcome or capability-certified Dispatch emission, got emission=%s representation=%s", plan.Emission, plan.FuncRep) @@ -1421,7 +1525,7 @@ func validateCoroPhysicalABIForOwner( // not an IRQ context. Preserve the bit in the plan/digest while allowing the // CFG lowering to execute it. Thread affinity and opaque execution still // require scheduler protocols that this ABI does not provide. - allowedExec := coro.MayUnwind | coro.NeedsPreempt | coro.IRQUnsafe + allowedExec := coro.MayUnwind | coro.NeedsPreempt | coro.IRQUnsafe | coro.NeedsRuntimeContext if cleanupPlan != nil { allowedExec |= coro.NeedsCleanupFrame } @@ -1434,7 +1538,8 @@ func validateCoroPhysicalABIForOwner( if len(fn.FreeVars) != 0 && !managedDispatchTarget && plan.FuncRep != coro.DirectCoro { return fail("captured coroutine bodies require one exact direct or capability-certified descriptor context ABI") } - if fn.Recover != nil && (cleanupPlan == nil || cleanupPlan.external) { + deadCleanupRecover := cleanupPlan == nil && !plan.Exec.Contains(coro.NeedsCleanupFrame) + if fn.Recover != nil && (cleanupPlan == nil || cleanupPlan.external) && !deadCleanupRecover { return fail("recover blocks require coroutine cleanup/unwind lowering") } if cleanupPlan != nil && !cleanupPlan.external { @@ -1574,6 +1679,7 @@ func validateCoroPhysicalABIForOwner( } wrapperOwner := universe.packages[wrapperInfo.owner] structuralKey, err := builtinSpawnWrapperStructuralKey( + universe.emissionTypeKeys.strict, wrapperInfo, wrapperOwner, universe.finalIdentity(spawn.Parent()), @@ -1800,6 +1906,10 @@ func validateCoroPhysicalABIForOwner( case *ssa.Defer, *ssa.RunDefers: want := coroPhysicalOutcomeDeferRegister if _, run := instr.(*ssa.RunDefers); run { + if cleanupPlan == nil && !plan.Exec.Contains(coro.NeedsCleanupFrame) && + instructionPlan.outcome == coroPhysicalOutcomeNone { + continue + } want = coroPhysicalOutcomeRunDefers } if instructionPlan.outcome != want { @@ -2881,7 +2991,7 @@ func resolveCoroStaticPlainCall(plan *coro.SSAPlan, call ssa.CallInstruction) (* validTrustedExternal := trustedInline && targetPlan.External == coro.ExternalUnknownForeign && targetPlan.Emission == coro.EmitExternal && targetPlan.Primary == coro.PrimaryExternal effectiveExec := targetPlan.Exec - allowedExec := coro.MayUnwind | coro.IRQUnsafe + allowedExec := coro.MayUnwind | coro.IRQUnsafe | coro.NeedsRuntimeContext if trustedInline { targetCertificate, certified := plan.CallableContractCertificate(target) if !certified { @@ -2912,7 +3022,7 @@ func resolveCoroStaticPlainCall(plan *coro.SSAPlan, call ssa.CallInstruction) (* } defaultExec := coro.CallableContractExecConstraints(targetCertificate.Contract) selectedExec := coro.CallableContractExecConstraints(targetCertificate.TrustedInlineContract) - const contractExec = coro.ThreadAffine | coro.OpaqueExec + const contractExec = coro.ThreadAffine | coro.OpaqueExec | coro.NeedsRuntimeContext if unsupported := (defaultExec | selectedExec) &^ contractExec; unsupported != 0 { return nil, coro.FunctionPlan{}, fmt.Errorf("trusted-inline target projected non-contract execution flags %s", unsupported) } diff --git a/cl/coro_abi_test.go b/cl/coro_abi_test.go index f47abe4d1a..59dfc8a544 100644 --- a/cl/coro_abi_test.go +++ b/cl/coro_abi_test.go @@ -62,7 +62,7 @@ func TestCoroLeafPhysicalABIPresplit(t *testing.T) { t.Fatalf("coro.suspend calls = %d, want initial + final:\n%s", got, leafIR) } begin := strings.Index(leafIR, "call ptr @llvm.coro.begin") - publish := strings.Index(leafIR, "call void @"+coroFramePublishHookV1) + publish := strings.Index(leafIR, "call void @"+coroFramePublishHookV3) initialSuspend := strings.Index(leafIR, "call i8 @llvm.coro.suspend") if begin < 0 || publish < 0 || initialSuspend < 0 || !(begin < publish && publish < initialSuspend) { t.Fatalf("promise/header was not published after coro.begin and before initial suspend:\n%s", leafIR) @@ -239,17 +239,20 @@ func TestCoroChildAwaitPhysicalABIV1Presplit(t *testing.T) { if got := strings.Count(childIR, "call i8 @llvm.coro.suspend"); got != 2 { t.Fatalf("Child coro.suspend calls = %d, want initial + final:\n%s", got, childIR) } - for _, forbidden := range []string{"llvm.coro.resume", "llvm.coro.done", "llvm.coro.destroy"} { - if hasLLVMCall(parentIR, forbidden) { - t.Fatalf("Parent directly owns forbidden %s operation:\n%s", forbidden, parentIR) + for _, operation := range []string{"llvm.coro.resume", "llvm.coro.done", "llvm.coro.destroy"} { + if got := len(regexp.MustCompile(`call [^\n]*@`+regexp.QuoteMeta(operation)+`\b`).FindAllString(parentIR, -1)); got != 1 { + t.Fatalf("Parent direct static-inline %s calls = %d, want 1:\n%s", operation, got, parentIR) } } for _, hook := range []string{ coroFrameAllocHookV1, - coroFramePublishHookV1, + coroFramePublishHookV3, coroAwaitPrepareHookV1, - coroAwaitInlineHookV1, + coroAwaitInlineBeginHookV2, + coroAwaitInlineFinishHookV2, + coroFrameDestroyCommitHookV2, + coroAwaitInlineDestroyCommitHookV2, coroAwaitConsumeHookV1, coroPreemptPollHookV1, coroRunDecisionTakeZeroHookV1, @@ -268,15 +271,23 @@ func TestCoroChildAwaitPhysicalABIV1Presplit(t *testing.T) { if got := strings.Count(ir, "call ptr @"+coroFrameAllocHookV1); got != 2 { t.Fatalf("task-aware v1 frame allocations = %d, want Parent + Child:\n%s", got, ir) } - if got := strings.Count(ir, "call void @"+coroFramePublishHookV1); got != 2 { + if got := strings.Count(ir, "call void @"+coroFramePublishHookV3); got != 2 { t.Fatalf("v1 frame publications = %d, want Parent + Child:\n%s", got, ir) } if got := strings.Count(ir, "call void @"+coroAwaitPrepareHookV1); got != 1 { t.Fatalf("v1 await preparations = %d, want one Parent->Child handoff:\n%s", got, ir) } - if got := strings.Count(ir, "call i1 @"+coroAwaitInlineHookV1); got != 1 { + if got := strings.Count(ir, "call i1 @"+coroAwaitInlineBeginHookV2); got != 1 { t.Fatalf("v1 inline await attempts = %d, want one Parent->Child fast path:\n%s", got, ir) } + if got := strings.Count(ir, "call i1 @"+coroAwaitInlineFinishHookV2); got != 1 { + t.Fatalf("v2 inline await finishes = %d, want one Parent->Child ownership settlement:\n%s", got, ir) + } + for _, hook := range []string{coroFrameDestroyCommitHookV2, coroAwaitInlineDestroyCommitHookV2} { + if got := strings.Count(ir, "call void @"+hook); got != 1 { + t.Fatalf("v2 inline await %s calls = %d, want one:\n%s", hook, got, ir) + } + } if got := strings.Count(ir, "call i32 @"+coroAwaitConsumeHookV1); got != 2 { t.Fatalf("v1 await outcome consume sites = %d, want normal/cancellation reconciliation:\n%s", got, ir) } @@ -417,7 +428,7 @@ func TestCoroChildAwaitPhysicalABIV1CoroSplit(t *testing.T) { } for _, function := range []string{"foo.Parent$coro", "foo.Child$coro"} { ramp := module.NamedFunction(function).String() - if !strings.Contains(ramp, "call void @"+coroFramePublishHookV1) { + if !strings.Contains(ramp, "call void @"+coroFramePublishHookV3) { t.Fatalf("%s ramp lost frame publication:\n%s", function, ramp) } destroy := module.NamedFunction(function + ".destroy").String() @@ -524,9 +535,10 @@ func Loop(limit uint32) uint32 { ) } if gates := strings.Count(body, "icmp ule i32"); gates != polls { - t.Fatalf("Loop checkpoint gates = %d, want one frame-local countdown gate per poll site (%d):\n%s", gates, polls, body) + t.Fatalf("Loop checkpoint gates = %d, want one private countdown gate per poll site (%d):\n%s", gates, polls, body) } - if resets := strings.Count(body, "store i32 64"); resets < polls+1 { + reset := "store i32 " + strconv.FormatUint(coroPreemptCheckpointStride, 10) + if resets := strings.Count(body, reset); resets < polls+1 { t.Fatalf("Loop checkpoint resets = %d, want initial activation plus at least one reset per poll site (%d):\n%s", resets, polls+1, body) } assertCoroScalarRunDecisionCalls(t, "Loop", body, polls+1) @@ -2187,16 +2199,17 @@ func assertCoroV0HeaderStateZero(t *testing.T, body string) { func assertCoroV1InitialPublish(t *testing.T, name, body string) { t.Helper() begin := strings.Index(body, "call ptr @llvm.coro.begin") - publish := strings.Index(body, "call void @"+coroFramePublishHookV1) + publish := strings.Index(body, "call void @"+coroFramePublishHookV3) suspend := strings.Index(body, "call i8 @llvm.coro.suspend") if begin < 0 || publish < 0 || suspend < 0 || !(begin < publish && publish < suspend) { t.Fatalf("%s does not publish its v1 frame after coro.begin and before initial suspend:\n%s", name, body) } call := regexp.MustCompile( - `call void @` + regexp.QuoteMeta(coroFramePublishHookV1) + `\(ptr [^,]+, ptr [^,]+, ptr [^,]+, ptr [^)]+\)`, + `call void @` + regexp.QuoteMeta(coroFramePublishHookV3) + + `\(ptr [^,]+, ptr [^,]+, ptr [^,]+, ptr [^,]+, ptr [^,]+, ptr [^,]+, ptr [^)]+\)`, ) if !call.MatchString(body) { - t.Fatalf("%s frame publication lacks (task, handle, header, storage):\n%s", name, body) + t.Fatalf("%s frame publication lacks (task, handle, header, storage, metadata, descriptor, result):\n%s", name, body) } } @@ -2234,32 +2247,21 @@ func assertCoroScalarRunDecisionCalls(t *testing.T, name, body string, want int) block := body[startOfMatch : searchOffset+end] branch := regexp.MustCompile( `(?m)^\s+br i1 ` + regexp.QuoteMeta(match[2]) + - `, label %([-a-zA-Z$._0-9]+), label %[-a-zA-Z$._0-9]+\s*$`, + `, label %([-a-zA-Z$._0-9]+), label %[-a-zA-Z$._0-9]+` + + `(?:, !prof ![0-9]+)?\s*$`, ).FindStringSubmatch(block) if len(branch) != 2 { t.Fatalf("%s scalar run-decision result does not control its block terminator:\n%s", name, block) } - label := branch[1] + ":" - start := strings.Index(body, "\n"+label) - if start < 0 { - t.Fatalf("%s cancellation target %q is absent:\n%s", name, branch[1], body) - } - start++ - targetRest := body[start+len(label):] - targetEnd := len(targetRest) - if next := regexp.MustCompile(`(?m)^[-a-zA-Z$._0-9]+:`).FindStringIndex(targetRest); next != nil { - targetEnd = next[0] - } - targetBlock := body[start : start+len(label)+targetEnd] - branches := regexp.MustCompile(`(?m)^\s+br label %([-a-zA-Z$._0-9]+)\s*$`).FindAllStringSubmatch(targetBlock, -1) - if len(branches) != 1 { - t.Fatalf("%s cancellation target %q does not unconditionally enter cleanup:\n%s", name, branch[1], targetBlock) + completionBlock, ok := coroTextCancellationCompletionBlock(body, branch[1]) + if !ok { + t.Fatalf("%s cancellation target %q does not have a unique unconditional path to completion:\n%s", name, branch[1], body) } if completion == "" { - completion = branches[0][1] - } else if branches[0][1] != completion { + completion = completionBlock + } else if completionBlock != completion { t.Fatalf("%s cancellation gates reach different cleanup entries %s and %s:\n%s", - name, completion, branches[0][1], body) + name, completion, completionBlock, body) } } if completion == "" { @@ -2267,6 +2269,42 @@ func assertCoroScalarRunDecisionCalls(t *testing.T, name, body string, want int) } } +// coroTextCancellationCompletionBlock follows only unique unconditional +// branches. LLVM 22 may split child-consumption from the shared completion +// block, so comparing only the cancellation target's immediate successor is +// too strict: every gate must instead converge on the same exact completion +// publication block. +func coroTextCancellationCompletionBlock(body, start string) (string, bool) { + labelPattern := regexp.MustCompile(`(?m)^([-a-zA-Z$._0-9]+):(?:[ \t]*;[^\n]*)?[ \t]*$`) + matches := labelPattern.FindAllStringSubmatchIndex(body, -1) + blocks := make(map[string]string, len(matches)) + for index, match := range matches { + end := len(body) + if index+1 < len(matches) { + end = matches[index+1][0] + } + blocks[body[match[2]:match[3]]] = body[match[0]:end] + } + unconditional := regexp.MustCompile(`(?m)^\s+br label %([-a-zA-Z$._0-9]+)\s*$`) + visited := make(map[string]bool) + for label := start; label != "" && !visited[label]; { + visited[label] = true + block, ok := blocks[label] + if !ok { + return "", false + } + if strings.Contains(block, "call void @"+coroCompletePrepareHookV2) { + return label, true + } + branches := unconditional.FindAllStringSubmatch(block, -1) + if len(branches) != 1 { + return "", false + } + label = branches[0][1] + } + return "", false +} + func assertCoroCancellationTerminalStatusPublication(t *testing.T, function llvm.Value) { t.Helper() if function.IsNil() { @@ -2405,6 +2443,43 @@ func compileCoroDecisionFrameProbe(t *testing.T, target *llssa.Target, scalarGat return coroFrameAllocationSize(t, ramp, prog.PointerSize()*8) } +func TestCoroPhysicalABIRuntimeContextDescriptorProof(t *testing.T) { + prog := newLLSSAProg(t) + defer prog.Dispose() + pkg := prog.NewPackage("coro_context_descriptor_probe", "llgo/test/coro-context-descriptor-probe") + defer pkg.Module().Dispose() + ctx := &context{prog: prog, pkg: pkg} + sig := types.NewSignatureType(nil, nil, nil, nil, nil, false) + + var contextFreeHash [16]byte + for _, test := range []struct { + name string + exec coro.ExecFlags + want bool + }{ + {name: "closed managed body", want: true}, + {name: "ambient runtime primitive", exec: coro.NeedsRuntimeContext}, + {name: "open target", exec: coro.OpaqueExec}, + {name: "irq eligibility is orthogonal", exec: coro.IRQUnsafe, want: true}, + {name: "thread affinity is orthogonal", exec: coro.ThreadAffine, want: true}, + {name: "worker blocking is orthogonal", exec: coro.BlockForeign, want: true}, + } { + abi := newCoroPhysicalABI(ctx, plannedFunctionSymbol{ + name: test.name, + plan: coro.FunctionPlan{ID: coro.FunctionID("llgo.test.context." + test.name), Exec: test.exec}, + }, sig) + got := abi.descriptorFlags&coro.FrameDescriptorNoRuntimeContextV1 != 0 + if got != test.want { + t.Fatalf("%s context-free descriptor = %t, want %t (exec=%s)", test.name, got, test.want, test.exec) + } + if test.want { + contextFreeHash = abi.hash + } else if abi.hash == contextFreeHash { + t.Fatalf("%s descriptor hash did not bind runtime-context mode", test.name) + } + } +} + func compileCoroPreemptCountdownFrameProbe(t *testing.T, target *llssa.Target, gated bool) uint64 { t.Helper() var prog llssa.Program @@ -2647,10 +2722,10 @@ func assertCoroV1Completion(t *testing.T, name, body string) { func assertCoroStaticChildAwait(t *testing.T, parent string) { t.Helper() childCall := regexp.MustCompile(`call ptr @"?foo\.Child\$coro"?\(`).FindStringIndex(parent) - publish := strings.Index(parent, "call void @"+coroFramePublishHookV1) + publish := strings.Index(parent, "call void @"+coroFramePublishHookV3) initialSuspend := strings.Index(parent, "call i8 @llvm.coro.suspend") await := strings.Index(parent, "call void @"+coroAwaitPrepareHookV1) - inline := strings.Index(parent, "call i1 @"+coroAwaitInlineHookV1) + inline := strings.Index(parent, "call i1 @"+coroAwaitInlineBeginHookV2) if childCall == nil || publish < 0 || initialSuspend < 0 || await < 0 || inline < 0 || !(publish < initialSuspend && initialSuspend < childCall[0] && childCall[0] < await && await < inline) { @@ -2682,10 +2757,20 @@ func assertCoroStaticChildAwait(t *testing.T, parent string) { if decisionRelative < 0 { t.Fatalf("Parent slow edge does not take its run decision after await resume:\n%s", parent) } - if !regexp.MustCompile(`(?s)call i1 @` + regexp.QuoteMeta(coroAwaitInlineHookV1) + + if !regexp.MustCompile(`(?s)call i1 @` + regexp.QuoteMeta(coroAwaitInlineBeginHookV2) + `.*xor i1 .*true.*br i1`).MatchString(parent[inline:]) { t.Fatalf("Parent does not branch to the slow suspend on an incomplete inline await:\n%s", parent) } + resume := strings.Index(parent[inline:], "call void @llvm.coro.resume") + done := strings.Index(parent[inline:], "call i1 @llvm.coro.done") + finish := strings.Index(parent[inline:], "call i1 @"+coroAwaitInlineFinishHookV2) + destroy := strings.Index(parent[inline:], "call void @llvm.coro.destroy") + frameCommit := strings.Index(parent[inline:], "call void @"+coroFrameDestroyCommitHookV2) + inlineCommit := strings.Index(parent[inline:], "call void @"+coroAwaitInlineDestroyCommitHookV2) + if resume < 0 || done <= resume || finish <= done || destroy <= finish || + frameCommit <= destroy || inlineCommit <= frameCommit { + t.Fatalf("Parent static-inline ownership order is not resume -> done -> finish -> destroy -> frame commit -> await commit:\n%s", parent) + } if !regexp.MustCompile(`(?s)store i16 0,.*store i16 2,.*call i32 @` + regexp.QuoteMeta(coroAwaitConsumeHookV1) + `.*switch i32`).MatchString(parent[inline:]) { t.Fatalf("Parent shared fast/resumed continuation does not activate and consume the child outcome:\n%s", parent) @@ -3038,7 +3123,8 @@ func assertCoroRootFactoryV1Descriptor(t *testing.T, ir, hash, parentHash string ` = linkonce_odr unnamed_addr constant ` + `\{ i32, i32, i64, i64, ` + uintptrType + `, ` + uintptrType + `, \{ ptr, ` + uintptrType + ` \}, \{ ptr, ` + uintptrType + ` \} \} ` + - `\{ i32 1, i32 0, i64 ([^,]+), i64 ([^,]+),`, + `\{ i32 1, i32 ` + strconv.FormatUint(uint64(coro.FrameDescriptorNoRuntimeContextV1), 10) + + `, i64 ([^,]+), i64 ([^,]+),`, ) frame := framePattern.FindStringSubmatch(ir) if len(frame) != 3 { diff --git a/cl/coro_await.go b/cl/coro_await.go index b2fea87f9f..00aa3a3c85 100644 --- a/cl/coro_await.go +++ b/cl/coro_await.go @@ -292,7 +292,7 @@ func validateCoroStaticMethodCallOperands(call ssa.CallInstruction, target *ssa. if err != nil { return fmt.Errorf("derive canonical static coroutine method signature: %w", err) } - if !flattenedLinkname && !coroInterfaceDispatchSignaturesIdentical(effectiveRaw, normalized) { + if !flattenedLinkname && !coroInterfaceDispatchSignaturesIdentical(effectiveRaw, normalized, universe.emissionTypeKeys.strictABI) { return fmt.Errorf("static coroutine method source ABI %s does not match canonical target ABI %s", effectiveRaw, normalized) } targetContext, err = universe.functionABIContext(target, universe.ownerOf(target)) @@ -358,10 +358,10 @@ func validateCoroAwaitTarget(caller, target coro.FunctionPlan) error { target.ID, target.External, target.Emission, target.Primary, target.FuncRep, target.Demand, ) } - if callerOutcome && target.ManagedEntry != coro.ManagedEntryOutcomePlain { + if callerOutcome && !target.HasStaticOutcome() { return fmt.Errorf( - "outcome-plain caller %q targets non-atomic managed entry %s", - caller.ID, target.ManagedEntry, + "outcome-plain caller %q targets a function without a static outcome capability", + caller.ID, ) } return nil @@ -621,6 +621,12 @@ func (p *context) compileCoroTargetEntryAwaitWithContextAndRecoveryResult( callerPlan.ID, targetPlan.ID, childFn.Name(), childType, childFn.Expr.RawType(), )) } + // This edge names one exact managed ramp and the child cannot outlive the + // caller's coroutine frame: every terminal or suspended path is reconciled + // by awaitCoroChildWithRecovery below. LLVM 22 may therefore embed the child + // frame in its static parent; dynamic/function-value calls never reach this + // proof point. + b.MarkCoroElideSafe(child) value := p.awaitCoroChildWithRecovery( b, child, resultSlot, sourceSig.Results(), cleanup, keepaliveSlots, ) @@ -652,8 +658,8 @@ func (p *context) coroAwaitResultAddress( // this edge therefore uses the same scheduler-owned child transaction as an // ordinary static synchronous-style call. func (p *context) compileCoroPatchInitAwait(b llssa.Builder) { - if !p.hasCoroPhysicalBody() || b == nil || b.Func != p.fn { - panic("coroutine patch initializer await requires an active physical body") + if !p.hasStructuredOutcomePhysicalBody() || b == nil || b.Func != p.fn { + panic("coroutine patch initializer call requires an active structured body") } if p.emissionUniverse == nil || p.immutablePlan() == nil || p.goFn == nil { panic("coroutine patch initializer await requires a frozen exact plan") @@ -666,6 +672,29 @@ func (p *context) compileCoroPatchInitAwait(b llssa.Builder) { if !frozen || original == nil || !exact || planned != original { panic("coroutine patch initializer edge disagrees between the frozen emission universe and SSA plan") } + if p.hasOutcomePlainPhysicalBody() { + entry, err := p.resolvePatchOriginalInitOutcomeSymbol(original) + if err != nil { + panic(err) + } + if err := entry.checkOutcomePlainSupported(); err != nil { + panic(err) + } + calleeFn, _, kind := p.compileFuncDeclVariantEntry(p.pkg, entry, false) + if kind != goFunc { + panic("patch original initializer outcome target did not resolve to a Go entry") + } + completion := p.structuredOutcomeAlloca(outcomePlainCompletionType(p.prog), true) + resultType := p.prog.Type(newOutcomePlainPhysicalABI(original.Signature).resultSlotType, llssa.InGo) + resultSlot := p.structuredOutcomeAlloca(resultType, false) + b.Call(calleeFn.Expr, + p.managedPhysicalTask(), + b.Convert(p.prog.VoidPtr(), resultSlot), + b.Convert(p.prog.VoidPtr(), completion), + ) + p.dispatchOutcomePlainCompletion(b, completion) + return + } entry := p.mustPatchOriginalInitFunctionSymbol(original) if entry.function != original || !entry.patchOriginalInit { panic("coroutine patch initializer edge lost its exact private original-init role") @@ -693,6 +722,31 @@ func (p *context) coroFrameAlloca(typ llssa.Type) llssa.Expr { return alloc.AllocaT(typ) } +// structuredOutcomeAlloca emits function-lifetime storage for either managed +// physical body. A real coroutine must define the slot in its ramp entry so +// CoroSplit can retain it across suspension. An outcome-plain body is a normal +// synchronous function, so the same proven-local storage belongs on its native +// entry stack and remains available to LLVM's ordinary SROA/mem2reg pipeline. +func (p *context) structuredOutcomeAlloca(typ llssa.Type, zeroed bool) llssa.Expr { + if !p.hasStructuredOutcomePhysicalBody() || p.fn == nil || typ == nil { + panic("structured outcome alloca requires an active physical body and type") + } + if p.hasCoroPhysicalBody() { + if zeroed { + return p.coroFrameAlloc(typ) + } + return p.coroFrameAlloca(typ) + } + entry := p.fn.Block(0) + alloc := p.fn.NewBuilder() + defer alloc.Dispose() + alloc.SetBlockEx(entry, llssa.AtStart, true) + if zeroed { + return alloc.AllocaZeroedT(typ) + } + return alloc.AllocaT(typ) +} + func (p *context) coroFrameByteAlloca(b llssa.Builder, size int64) llssa.Expr { if size < 0 { panic("coroutine byte alloca requires a non-negative constant size") @@ -789,11 +843,7 @@ func (p *context) awaitCoroChildWithRecovery( recoverType, recoverData, ) - if body.abi.awaitInlineHook == "" { - panic("coroutine child await has no inline completion hook") - } - inline := p.pkg.NewFunc(body.abi.awaitInlineHook, coroAwaitInlineSignature(), llssa.InC) - completedInline := b.Call(inline.Expr, body.task, body.coro.Handle(), child) + completedInline := p.emitCoroStaticInlineAwait(b, child) if body.abi.awaitConsumeHook == "" { panic("coroutine child await has no outcome consume hook") } @@ -949,6 +999,55 @@ func (p *context) awaitCoroChildWithRecovery( return p.loadCoroAwaitResult(b, resultSlot, results) } +// emitCoroStaticInlineAwait keeps LLVM's handle operations in the exact static +// caller while the runtime owns only scheduler-state transitions. Besides +// removing one opaque runtime round trip, this is the shape required for LLVM +// 22 to select a coro_elide_safe no-allocation ramp and embed the child frame +// in its parent. A declined/deep or genuinely suspended child converges on the +// existing false result and parent suspend path. +func (p *context) emitCoroStaticInlineAwait(b llssa.Builder, child llssa.Expr) llssa.Expr { + body := p.coroBody() + if body == nil || b == nil || b.Func != p.fn || child.IsNil() || + body.abi.awaitInlineHook == "" || body.abi.awaitInlineFinishHook == "" || + body.abi.awaitInlineCommitHook == "" || body.abi.frameDestroyCommitHook == "" { + panic("coroutine static inline await has an incomplete physical contract") + } + begin := p.pkg.NewFunc(body.abi.awaitInlineHook, coroAwaitInlineSignature(), llssa.InC) + finish := p.pkg.NewFunc( + body.abi.awaitInlineFinishHook, coroAwaitInlineFinishSignature(), llssa.InC, + ) + frameCommit := p.pkg.NewFunc( + body.abi.frameDestroyCommitHook, coroFrameDestroyCommitSignature(), llssa.InC, + ) + inlineCommit := p.pkg.NewFunc( + body.abi.awaitInlineCommitHook, coroAwaitInlineCommitSignature(), llssa.InC, + ) + + started, declined, destroy, joined := p.fn.MakeBlock(), p.fn.MakeBlock(), p.fn.MakeBlock(), p.fn.MakeBlock() + parent := body.coro.Handle() + b.If(b.Call(begin.Expr, body.task, parent, child), started, declined) + + b.SetBlockEx(started, llssa.AtEnd, false) + b.CoroResume(child) + done := b.CoroDone(child) + b.If(b.Call(finish.Expr, body.task, parent, child, done), destroy, declined) + + b.SetBlockEx(declined, llssa.AtEnd, false) + b.Jump(joined) + b.SetBlockEx(destroy, llssa.AtEnd, false) + b.CoroDestroy(child) + b.Call(frameCommit.Expr, body.task, child) + b.Call(inlineCommit.Expr, body.task, parent, child) + b.Jump(joined) + + b.SetBlockContinuation(joined) + completed := b.Phi(p.prog.Bool()) + completed.AddIncoming(b, []llssa.BasicBlock{declined, destroy}, func(index int, _ llssa.BasicBlock) llssa.Expr { + return p.prog.BoolVal(index == 1) + }) + return completed.Expr +} + // enterCoroPropagatedPanic and enterCoroPropagatedGoexit are the narrow parent // capabilities shared by scheduler-owned child completion and synchronous // outcome-plain completion. Call-site lowerers do not inspect the complete diff --git a/cl/coro_call_site_plan.go b/cl/coro_call_site_plan.go index f7f1943c59..67fe494871 100644 --- a/cl/coro_call_site_plan.go +++ b/cl/coro_call_site_plan.go @@ -434,6 +434,9 @@ func (ir *coroProgramIR) freezeCallSites(u *EmissionUniverse) error { } } } + if err := ir.finalizeOutcomePlainIntrinsicSemantics(u.prog, u.functions, u.sortedUseOwners); err != nil { + return fmt.Errorf("finalize outcome-plain intrinsic semantics: %w", err) + } ir.callsFrozen = true // These maps are mutable builder scratch. All production call-site // certificate payloads now live in ProgramIR; retaining a second readable diff --git a/cl/coro_callable_identity.go b/cl/coro_callable_identity.go index 8195725e6c..98625dc931 100644 --- a/cl/coro_callable_identity.go +++ b/cl/coro_callable_identity.go @@ -172,6 +172,7 @@ func (u *EmissionUniverse) freezeCoroCallableIdentityCertificates() error { freezeCallableShape: u.freezeCoroCallableShape, entrySourceSignature: u.coroPhysicalEntrySourceSignature, finalFunctionIdentity: u.finalIdentity, + cFunctionABITypeKey: u.emissionTypeKeys.cFunctionABI, }) if err != nil { return err diff --git a/cl/coro_callable_shadow.go b/cl/coro_callable_shadow.go index 2188a95e35..11e0669ffd 100644 --- a/cl/coro_callable_shadow.go +++ b/cl/coro_callable_shadow.go @@ -315,7 +315,7 @@ func coroWorkerCallableTarget( coroWorkerAddressFunctionIdentity(universe, sourceTarget), coroWorkerAddressFunctionIdentity(universe, target), physical, - structuralGoLinknameABITypeKey(target.Signature), + universe.emissionTypeKeys.goLinknameABI(target.Signature), parsed.Canonical, behaviorDigest, )) @@ -368,7 +368,7 @@ func coroWorkerCallableTarget( coroWorkerAddressFunctionIdentity(universe, sourceTarget), coroWorkerAddressFunctionIdentity(universe, target), physical, - structuralGoLinknameABITypeKey(target.Signature), + universe.emissionTypeKeys.goLinknameABI(target.Signature), defaultContract.Canonical, behaviorDigest, )) diff --git a/cl/coro_channel.go b/cl/coro_channel.go index 7fc4b277b3..72a5e28fbc 100644 --- a/cl/coro_channel.go +++ b/cl/coro_channel.go @@ -26,13 +26,14 @@ import ( ) const ( - coroChanSendParkHookV1 = "__llgo_coro_chan_send_park_v1" - coroChanRecvParkHookV1 = "__llgo_coro_chan_recv_park_v1" - coroChanResumeHookV1 = "__llgo_coro_chan_resume_v1" + coroChanSendTryParkHookV2 = "__llgo_coro_chan_send_try_park_v2" + coroChanRecvTryParkHookV2 = "__llgo_coro_chan_recv_try_park_v2" + coroChanResumeHookV2 = "__llgo_coro_chan_resume_v2" ) const ( - coroChanResumeSendOK uint64 = iota + 1 + coroChanResumeInvalid uint64 = iota + coroChanResumeSendOK coroChanResumeRecvOK coroChanResumeRecvClosed coroChanResumeSendClosed @@ -54,7 +55,7 @@ func isCoroCloseBuiltinCall(call *ssa.Call) bool { return ok && builtin.Name() == "close" } -func coroChanParkSignature() *types.Signature { +func coroChanTryParkSignatureV2() *types.Signature { pointer := types.Typ[types.UnsafePointer] params := types.NewTuple( types.NewParam(token.NoPos, nil, "g", pointer), @@ -64,8 +65,11 @@ func coroChanParkSignature() *types.Signature { types.NewParam(token.NoPos, nil, "elem", pointer), types.NewParam(token.NoPos, nil, "state", pointer), types.NewParam(token.NoPos, nil, "size", types.Typ[types.Uintptr]), + types.NewParam(token.NoPos, nil, "stateID", types.Typ[types.Uint32]), + types.NewParam(token.NoPos, nil, "line", types.Typ[types.Uint32]), ) - return types.NewSignatureType(nil, nil, nil, params, nil, false) + results := types.NewTuple(types.NewParam(token.NoPos, nil, "status", types.Typ[types.Uint32])) + return types.NewSignatureType(nil, nil, nil, params, results, false) } func coroChanResumeSignature() *types.Signature { @@ -93,13 +97,18 @@ func (p *context) requireCoroChannelBody(b llssa.Builder) *coroBodyContext { func (p *context) newCoroChannelStorage(b llssa.Builder, elemType llssa.Type) (elem, state llssa.Expr) { // These addresses may be published to hchan immediately before suspend. // Allocate them in the physical ramp entry so no waiter can retain a - // resume-local M stack address. Reset at the logical operation point because - // the same static channel instruction may execute repeatedly in a loop. + // resume-local M stack address. The coroutine frame allocator zero-fills the + // complete frame, and channel resume clears state before the same static + // instruction can execute again in a loop. elem = p.coroFrameAlloca(elemType) stateType := p.prog.RuntimeType("CoroChanParkV1") state = p.coroFrameAlloca(stateType) b.Store(elem, b.Prog.Zero(elemType)) - b.Store(state, b.Prog.Zero(stateType)) + // State is unreachable until the conditional park hook runs, and that hook + // initializes the complete CoroChanParkV1 before publishing any frame + // pointer. Resume clears it before a loop can reuse this static operation. + // Eagerly zeroing the large aggregate here made both successful try paths + // and actual parks pay for a store which the runtime immediately repeated. return } @@ -107,24 +116,27 @@ func (p *context) compileCoroChanSend(b llssa.Builder, channel, value llssa.Expr body := p.requireCoroChannelBody(b) elem, state := p.newCoroChannelStorage(b, value.Type) b.Store(elem, value) - ready := b.CoroChanTrySend(channel, elem) body.emitCoroParkOperation(p, b, coroParkOperation{ - shouldSuspend: b.UnOp(token.NOT, ready), - park: func(suspend llssa.Builder) { - park := p.pkg.NewFunc(coroChanSendParkHookV1, coroChanParkSignature(), llssa.InC) - suspend.Call( + prepare: func(active llssa.Builder, stateID, sourceLine uint32) llssa.Expr { + park := p.pkg.NewFunc(coroChanSendTryParkHookV2, coroChanTryParkSignatureV2(), llssa.InC) + status := active.Call( park.Expr, body.task, body.coro.Handle(), - suspend.Convert(suspend.Prog.VoidPtr(), body.header), - suspend.Convert(suspend.Prog.VoidPtr(), channel), - suspend.Convert(suspend.Prog.VoidPtr(), elem), - suspend.Convert(suspend.Prog.VoidPtr(), state), - p.prog.IntVal(p.prog.SizeOf(value.Type), p.prog.Uintptr()), + active.Convert(active.Prog.VoidPtr(), body.header), + active.Convert(active.Prog.VoidPtr(), channel), + active.Convert(active.Prog.VoidPtr(), elem), + active.Convert(active.Prog.VoidPtr(), state), + active.Prog.IntVal(active.Prog.SizeOf(value.Type), active.Prog.Uintptr()), + active.Prog.IntVal(uint64(stateID), active.Prog.Uint32()), + active.Prog.IntVal(uint64(sourceLine), active.Prog.Uint32()), + ) + return active.BinOp( + token.EQL, status, active.Prog.IntVal(coroChanResumeInvalid, active.Prog.Uint32()), ) }, resume: func(resume llssa.Builder) llssa.Expr { - statusHook := p.pkg.NewFunc(coroChanResumeHookV1, coroChanResumeSignature(), llssa.InC) + statusHook := p.pkg.NewFunc(coroChanResumeHookV2, coroChanResumeSignature(), llssa.InC) return resume.Call(statusHook.Expr, body.task, resume.Convert(resume.Prog.VoidPtr(), state)) }, normal: []uint64{coroChanResumeSendOK}, @@ -143,28 +155,36 @@ func (p *context) compileCoroChanRecv(b llssa.Builder, instruction *ssa.UnOp, ch body := p.requireCoroChannelBody(b) elemType := p.prog.Elem(channel.Type) elem, state := p.newCoroChannelStorage(b, elemType) - result := b.CoroChanTryRecv(channel, elem) - recvOK := b.Extract(result, 0) - tryOK := b.Extract(result, 1) recvOKSlot := p.coroFrameAlloca(p.prog.Bool()) - b.Store(recvOKSlot, recvOK) body.emitCoroParkOperation(p, b, coroParkOperation{ - shouldSuspend: b.UnOp(token.NOT, tryOK), - park: func(suspend llssa.Builder) { - park := p.pkg.NewFunc(coroChanRecvParkHookV1, coroChanParkSignature(), llssa.InC) - suspend.Call( + prepare: func(active llssa.Builder, stateID, sourceLine uint32) llssa.Expr { + park := p.pkg.NewFunc(coroChanRecvTryParkHookV2, coroChanTryParkSignatureV2(), llssa.InC) + status := active.Call( park.Expr, body.task, body.coro.Handle(), - suspend.Convert(suspend.Prog.VoidPtr(), body.header), - suspend.Convert(suspend.Prog.VoidPtr(), channel), - suspend.Convert(suspend.Prog.VoidPtr(), elem), - suspend.Convert(suspend.Prog.VoidPtr(), state), - p.prog.IntVal(p.prog.SizeOf(elemType), p.prog.Uintptr()), + active.Convert(active.Prog.VoidPtr(), body.header), + active.Convert(active.Prog.VoidPtr(), channel), + active.Convert(active.Prog.VoidPtr(), elem), + active.Convert(active.Prog.VoidPtr(), state), + active.Prog.IntVal(active.Prog.SizeOf(elemType), active.Prog.Uintptr()), + active.Prog.IntVal(uint64(stateID), active.Prog.Uint32()), + active.Prog.IntVal(uint64(sourceLine), active.Prog.Uint32()), + ) + active.Store( + recvOKSlot, + active.BinOp( + token.EQL, + status, + active.Prog.IntVal(coroChanResumeRecvOK, active.Prog.Uint32()), + ), + ) + return active.BinOp( + token.EQL, status, active.Prog.IntVal(coroChanResumeInvalid, active.Prog.Uint32()), ) }, resume: func(resume llssa.Builder) llssa.Expr { - statusHook := p.pkg.NewFunc(coroChanResumeHookV1, coroChanResumeSignature(), llssa.InC) + statusHook := p.pkg.NewFunc(coroChanResumeHookV2, coroChanResumeSignature(), llssa.InC) status := resume.Call(statusHook.Expr, body.task, resume.Convert(resume.Prog.VoidPtr(), state)) resume.Store( recvOKSlot, @@ -212,7 +232,7 @@ func (p *context) compileCoroChanCloseWithRecovery( if cleanup != nil && body.cleanup != cleanup { panic("coroutine deferred close does not belong to the active cleanup drainer") } - status := b.CoroChanTryClose(channel) + status := b.CoroChanTryCloseTask(body.task, channel) nilChannel := b.Func.MakeBlock() alreadyClosed := b.Func.MakeBlock() normal := b.Func.MakeBlock() @@ -251,7 +271,9 @@ func (p *context) compileCoroChanSelect(b llssa.Builder, states []*llssa.SelectS b.Store(recvOKSlot, b.Extract(attempt, 1)) tryOK := b.Extract(attempt, 2) body.emitCoroParkOperation(p, b, coroParkOperation{ - shouldSuspend: b.UnOp(token.NOT, tryOK), + prepare: func(active llssa.Builder, _, _ uint32) llssa.Expr { + return active.UnOp(token.NOT, tryOK) + }, park: func(suspend llssa.Builder) { suspend.CoroChanSelectPark( plan, diff --git a/cl/coro_channel_test.go b/cl/coro_channel_test.go index 80b13a468a..a3338baa60 100644 --- a/cl/coro_channel_test.go +++ b/cl/coro_channel_test.go @@ -121,14 +121,16 @@ func TestCoroChannelNativeAndWasm32(t *testing.T) { sendPhysical := requireCoroPhysicalFunction(t, module, "foo.Send") send := sendPhysical.String() + assertCoroChannelTryParkUsesCompilerTask(t, sendPhysical, coroChanSendTryParkHookV2) + assertCoroChannelParkStateHasNoCallerStore(t, sendPhysical, coroChanSendTryParkHookV2) assertCoroCancellationTerminalStatusPublication(t, sendPhysical) - assertCoroChannelBody(t, "Send", send, coroChanSendParkHookV1, []uint64{ + assertCoroChannelBody(t, "Send", send, coroChanSendTryParkHookV2, []uint64{ coroChanResumeSendOK, coroChanResumeSendClosed, coroChanResumeTaskAbort, coroChanResumeShutdown, }) - for _, symbol := range []string{"github.com/goplus/llgo/runtime/internal/runtime.CoroChanTrySend", coroFaultPrepareHookV1} { + for _, symbol := range []string{coroChanSendTryParkHookV2, coroFaultPrepareHookV1} { if !strings.Contains(send, symbol) { t.Fatalf("Send coroutine lacks %q:\n%s", symbol, send) } @@ -138,15 +140,18 @@ func TestCoroChannelNativeAndWasm32(t *testing.T) { t.Fatalf("Send coroutine did not select the send-closed fault kind:\n%s", send) } for _, name := range []string{"Recv", "RecvOK"} { - recv := requireCoroPhysicalFunction(t, module, "foo."+name).String() - assertCoroChannelBody(t, name, recv, coroChanRecvParkHookV1, []uint64{ + recvPhysical := requireCoroPhysicalFunction(t, module, "foo."+name) + recv := recvPhysical.String() + assertCoroChannelTryParkUsesCompilerTask(t, recvPhysical, coroChanRecvTryParkHookV2) + assertCoroChannelParkStateHasNoCallerStore(t, recvPhysical, coroChanRecvTryParkHookV2) + assertCoroChannelBody(t, name, recv, coroChanRecvTryParkHookV2, []uint64{ coroChanResumeRecvOK, coroChanResumeRecvClosed, coroChanResumeTaskAbort, coroChanResumeShutdown, }) - if !strings.Contains(recv, "@\"github.com/goplus/llgo/runtime/internal/runtime.CoroChanTryRecv\"") { - t.Fatalf("%s coroutine lacks nonblocking receive helper:\n%s", name, recv) + if !strings.Contains(recv, "@"+coroChanRecvTryParkHookV2) { + t.Fatalf("%s coroutine lacks try-or-park receive helper:\n%s", name, recv) } } selectBody := requireCoroPhysicalFunction(t, module, "foo.Select").String() @@ -178,7 +183,7 @@ func TestCoroChannelNativeAndWasm32(t *testing.T) { runCoroABITestPipeline(t, prog, module) for _, name := range []string{"foo.Send$coro", "foo.Recv$coro", "foo.RecvOK$coro"} { resume := module.NamedFunction(name + ".resume") - if resume.IsNil() || !strings.Contains(resume.String(), "call i32 @"+coroChanResumeHookV1) { + if resume.IsNil() || !strings.Contains(resume.String(), "call i32 @"+coroChanResumeHookV2) { t.Fatalf("CoroSplit lost channel resume dispatch in %s:\n%s", name, module.String()) } if name == "foo.Send$coro" { @@ -233,9 +238,9 @@ func TestCoroChannelNativeAndWasm32(t *testing.T) { } defer object.Dispose() for _, symbol := range []string{ - coroChanSendParkHookV1, - coroChanRecvParkHookV1, - coroChanResumeHookV1, + coroChanSendTryParkHookV2, + coroChanRecvTryParkHookV2, + coroChanResumeHookV2, coroFaultPrepareHookV1, coroFaultPayloadHookV1, "github.com/goplus/llgo/runtime/internal/runtime.CoroChanSelectTry", @@ -250,6 +255,85 @@ func TestCoroChannelNativeAndWasm32(t *testing.T) { } } +// assertCoroChannelTryParkUsesCompilerTask freezes the internal channel-helper +// boundary: the first physical coroutine parameter is the scheduler G, and it +// must be forwarded as the first helper argument. Reconstructing that identity +// through TLS or a code-address lookup would add native hot-path work and make +// the same owner-local optimization unavailable to WASM/bare-metal targets. +func assertCoroChannelTryParkUsesCompilerTask(t *testing.T, function llvm.Value, callee string) { + t.Helper() + var call llvm.Value + for _, block := range function.BasicBlocks() { + for instruction := block.FirstInstruction(); !instruction.IsNil(); instruction = llvm.NextInstruction(instruction) { + if instruction.InstructionOpcode() != llvm.Call || instruction.CalledValue().Name() != callee { + continue + } + if !call.IsNil() { + t.Fatalf("%s calls %q more than once:\n%s", function.Name(), callee, function.String()) + } + call = instruction + } + } + if call.IsNil() { + t.Fatalf("%s does not call %q:\n%s", function.Name(), callee, function.String()) + } + if got := call.OperandsCount() - 1; got != 9 { + t.Fatalf("%s arguments = %d, want task/handle/header/channel/element/state/size/state-id/line:\n%s", + callee, got, call.String()) + } + if function.ParamsCount() < 1 || call.Operand(0) != function.Param(0) { + t.Fatalf("%s does not receive the compiler-carried task as argument zero:\n%s", callee, call.String()) + } +} + +// assertCoroChannelParkStateHasNoCallerStore freezes the zero-initialization +// ownership boundary. Coroutine frame allocation supplies the initial zero +// state, the park hook initializes it only after a nonblocking try fails, and +// resume clears it before reuse. A caller-side aggregate store would make +// every successful try pay the cost of initializing slow-path-only state. +func assertCoroChannelParkStateHasNoCallerStore(t *testing.T, function llvm.Value, parkHook string) { + t.Helper() + var call llvm.Value + for _, block := range function.BasicBlocks() { + for instruction := block.FirstInstruction(); !instruction.IsNil(); instruction = llvm.NextInstruction(instruction) { + if instruction.InstructionOpcode() != llvm.Call || instruction.CalledValue().Name() != parkHook { + continue + } + if !call.IsNil() { + t.Fatalf("%s calls %q more than once:\n%s", function.Name(), parkHook, function.String()) + } + call = instruction + } + } + if call.IsNil() { + t.Fatalf("%s does not call %q:\n%s", function.Name(), parkHook, function.String()) + } + if got := call.OperandsCount() - 1; got != 9 { + t.Fatalf("%s arguments = %d, want task/handle/header/channel/element/state/size/state-id/line:\n%s", + parkHook, got, call.String()) + } + state := coroChannelPointerBase(call.Operand(5)) + if state.IsAAllocaInst().IsNil() { + t.Fatalf("%s park state is not physical-ramp frame storage:\n%s", function.Name(), call.String()) + } + for _, block := range function.BasicBlocks() { + for instruction := block.FirstInstruction(); !instruction.IsNil(); instruction = llvm.NextInstruction(instruction) { + if instruction.InstructionOpcode() == llvm.Store && + coroChannelPointerBase(instruction.Operand(1)) == state { + t.Fatalf("%s eagerly stores channel park state before the slow-path hook:\n%s", + function.Name(), instruction.String()) + } + } + } +} + +func coroChannelPointerBase(value llvm.Value) llvm.Value { + for !value.IsABitCastInst().IsNil() && value.OperandsCount() == 1 { + value = value.Operand(0) + } + return value +} + func assertCoroSelectBody(t *testing.T, body string) { t.Helper() if got := strings.Count(body, "call i8 @llvm.coro.suspend"); got != 3 { @@ -307,7 +391,7 @@ func assertCoroChannelBody(t *testing.T, name, body, parkHook string, statuses [ if got := strings.Count(body, "call i8 @llvm.coro.suspend"); got != 3 { t.Fatalf("%s coro.suspend calls = %d, want initial + channel + final:\n%s", name, got, body) } - for _, symbol := range []string{parkHook, coroChanResumeHookV1} { + for _, symbol := range []string{parkHook, coroChanResumeHookV2} { if got := strings.Count(body, "@"+symbol); got != 1 { t.Fatalf("%s references to %q = %d, want 1:\n%s", name, symbol, got, body) } @@ -316,7 +400,7 @@ func assertCoroChannelBody(t *testing.T, name, body, parkHook string, statuses [ t.Fatalf("%s has no exact typed resume-status dispatch:\n%s", name, body) } statusCall := regexp.MustCompile( - `(?m)^\s+(%[-A-Za-z0-9._]+) = call i32 @` + regexp.QuoteMeta(coroChanResumeHookV1) + `\([^\n]+\)\s*$`, + `(?m)^\s+(%[-A-Za-z0-9._]+) = call i32 @` + regexp.QuoteMeta(coroChanResumeHookV2) + `\([^\n]+\)\s*$`, ).FindStringSubmatchIndex(body) if len(statusCall) != 4 { t.Fatalf("%s has no unique channel resume status value:\n%s", name, body) @@ -339,12 +423,12 @@ func assertCoroChannelBody(t *testing.T, name, body, parkHook string, statuses [ t.Fatalf("%s channel resume switch lacks status %d:\n%s", name, status, dispatch[0]) } } - hook := strings.Index(body, "call void @"+parkHook) + hook := strings.Index(body, "call i32 @"+parkHook) if hook < 0 { t.Fatalf("%s does not publish its physical park:\n%s", name, body) } suspend := strings.Index(body[hook:], "call i8 @llvm.coro.suspend") - resume := strings.Index(body[hook:], "call i32 @"+coroChanResumeHookV1) + resume := strings.Index(body[hook:], "call i32 @"+coroChanResumeHookV2) if suspend < 0 || resume < 0 || suspend >= resume { t.Fatalf("%s does not publish park before suspend and dispatch after resume:\n%s", name, body) } @@ -393,6 +477,7 @@ func compileCoroChannelFixture(t *testing.T, target *llssa.Target) ( EmissionUniverse: ssaUniverse, FunctionIDs: functionIDs, MaxPlainInstructions: -1, + ClassifyLocalBody: universe.CoroLocalBodyFacts, }) if err != nil { prog.Dispose() diff --git a/cl/coro_child_keepalive_test.go b/cl/coro_child_keepalive_test.go index 1bbb4cff0a..e311cd2dc9 100644 --- a/cl/coro_child_keepalive_test.go +++ b/cl/coro_child_keepalive_test.go @@ -19,7 +19,6 @@ package cl import ( - "strings" "testing" "github.com/goplus/llgo/internal/coro" @@ -109,35 +108,46 @@ func Parent(pointer *byte) { t.Fatalf("verify child keepalive before CoroSplit: %v\n%s", err, module.String()) } - parentIR := requireCoroPhysicalFunction(t, module, "foo.Parent").String() - consume := "call i32 @" + coroAwaitConsumeHookV1 - fakeUse := "call void (...) @llvm.fake.use(ptr " - consumeAt, fakeUseAt := allTextIndexes(parentIR, consume), allTextIndexes(parentIR, fakeUse) - if len(consumeAt) != 2 || len(fakeUseAt) != 2 { - t.Fatalf("child completion consume/fake-use sites = %d/%d, want 2/2:\n%s", len(consumeAt), len(fakeUseAt), parentIR) - } - for index := range consumeAt { - if fakeUseAt[index] <= consumeAt[index] || index+1 < len(consumeAt) && fakeUseAt[index] >= consumeAt[index+1] { - t.Fatalf("fake-use %d does not follow its exact completion consume:\n%s", index, parentIR) - } + parentRamp := requireCoroPhysicalFunction(t, module, "foo.Parent") + consumeCount, ownerUseCount := coroChildAwaitCompletionOwnerUseCounts(parentRamp) + if consumeCount != 2 || ownerUseCount != 2 { + t.Fatalf("child completion consume/owner fake-use sites = %d/%d, want 2/2:\n%s", + consumeCount, ownerUseCount, parentRamp.String()) } runCoroABITestPipeline(t, prog, module) resume := module.NamedFunction("foo.Parent$coro.resume") - if resume.IsNil() || strings.Count(resume.String(), fakeUse) != 2 { + consumeCount, ownerUseCount = coroChildAwaitCompletionOwnerUseCounts(resume) + if resume.IsNil() || consumeCount != 2 || ownerUseCount != 2 { t.Fatalf("CoroSplit did not retain both completion-bound pointer owners:\n%s", module.String()) } } -func allTextIndexes(text, marker string) []int { - var indexes []int - for offset := 0; ; { - index := strings.Index(text[offset:], marker) - if index < 0 { - return indexes +func coroChildAwaitCompletionOwnerUseCounts(function llvm.Value) (consumes, ownerUses int) { + if function.IsNil() { + return 0, 0 + } + for _, block := range function.BasicBlocks() { + awaitingOwnerUse := false + for instruction := block.FirstInstruction(); !instruction.IsNil(); instruction = llvm.NextInstruction(instruction) { + if instruction.InstructionOpcode() != llvm.Call { + continue + } + switch instruction.CalledValue().Name() { + case coroAwaitConsumeHookV1: + consumes++ + awaitingOwnerUse = true + case "llvm.fake.use": + // The scheduler's scalar run-decision scratch may also need an + // llvm.fake.use. The pointer owner retained across this exact + // child await is distinguished by its post-consume frame load. + if awaitingOwnerUse && instruction.OperandsCount() > 1 && + instruction.Operand(0).InstructionOpcode() == llvm.Load { + ownerUses++ + awaitingOwnerUse = false + } + } } - index += offset - indexes = append(indexes, index) - offset = index + len(marker) } + return consumes, ownerUses } diff --git a/cl/coro_defer.go b/cl/coro_defer.go index cc0802f662..20b68275e3 100644 --- a/cl/coro_defer.go +++ b/cl/coro_defer.go @@ -271,7 +271,11 @@ func prepareCoroStaticCleanupPlan( return nil, fmt.Errorf("cleanup site function %q has no compilation plan", siteFunction.Name()) } siteInfos := blocks.Infos(siteFunction.Blocks) + reachable := coroPhysicalConstantReachableBlocks(siteFunction) for _, block := range siteFunction.Blocks { + if !reachable[block] { + continue + } for instructionIndex, raw := range block.Instrs { switch instruction := raw.(type) { case *ssa.Defer: @@ -441,7 +445,7 @@ func prepareCoroStaticCleanupPlan( } if len(byInstruction) == 0 { - if caller.Exec.Contains(coro.NeedsCleanupFrame) || runDefers != 0 { + if caller.Exec.Contains(coro.NeedsCleanupFrame) { return nil, fmt.Errorf("needs-cleanup-frame body has no supported static defer site") } return nil, nil @@ -529,11 +533,20 @@ func prepareCoroStaticCleanupPlan( // blocks.Infos' Next chain is a topological order outside SCCs. Defer // sites in SCCs were rejected above, so reversing this list later is the // exact registration order for every path on which two sites both ran. + reachable := coroPhysicalConstantReachableBlocks(fn) ordered := make([]*coroStaticCleanupSitePlan, 0, len(byInstruction)) for index := 0; index >= 0; index = infos[index].Next { - for _, raw := range fn.Blocks[index].Instrs { + block := fn.Blocks[index] + if !reachable[block] { + continue + } + for _, raw := range block.Instrs { if instruction, ok := raw.(*ssa.Defer); ok { - ordered = append(ordered, byInstruction[instruction]) + site, planned := byInstruction[instruction] + if !planned { + return nil, fmt.Errorf("reachable defer in block %d is absent from the cleanup plan", block.Index) + } + ordered = append(ordered, site) } } } diff --git a/cl/coro_defer_test.go b/cl/coro_defer_test.go index bed35c9020..ff3f0de9e4 100644 --- a/cl/coro_defer_test.go +++ b/cl/coro_defer_test.go @@ -160,6 +160,44 @@ func Root() { } } +func TestCoroStaticCleanupOrderIgnoresConstantUnreachableDefer(t *testing.T) { + const source = `package foo +func live() {} +func dead() {} +func Root() { + defer live() + if false { + defer dead() + } +} +` + prog, universe, plan, root, _ := buildCoroStaticCleanupPlanFixture(t, source) + defer prog.Dispose() + + reachable := coroPhysicalConstantReachableBlocks(root) + allDefers, reachableDefers := 0, 0 + for _, block := range root.Blocks { + for _, instruction := range block.Instrs { + if _, ok := instruction.(*ssa.Defer); ok { + allDefers++ + if reachable[block] { + reachableDefers++ + } + } + } + } + if allDefers != 2 || reachableDefers != 1 { + t.Fatalf("constant-unreachable defer shape = all:%d reachable:%d", allDefers, reachableDefers) + } + cleanup, err := prepareCoroStaticCleanupPlan(root, plan, universe, "", true) + if err != nil { + t.Fatal(err) + } + if cleanup == nil || len(cleanup.sites) != 1 || cleanup.sites[0].target == nil || cleanup.sites[0].target.Name() != "live" { + t.Fatalf("constant-unreachable cleanup plan = %+v", cleanup) + } +} + func TestCoroStaticCleanupIRNativeAndWasm32(t *testing.T) { llssa.Initialize(llssa.InitAll) for _, test := range []struct { @@ -590,7 +628,7 @@ func assertCoroAwaitCompletionCleanupControlFlow(t *testing.T, function llvm.Val first == coroTestAwaitConsumeCanceled && second == coroTestAwaitConsumeNormal { gateFound = true } - case coroAwaitInlineHookV1: + case coroAwaitInlineFinishHookV2: // The inline-complete edge reaches only normal consume. The // suspend edge re-enters the run-decision gate and can first // reach cancellation as well as the normal continuation. @@ -748,17 +786,35 @@ const ( func coroTestFirstReachableAwaitConsumes( entry, normal, canceled llvm.BasicBlock, ) uint8 { - seen := make(map[llvm.BasicBlock]bool) - pending := []llvm.BasicBlock{entry} + type edge struct { + block llvm.BasicBlock + predecessor llvm.BasicBlock + } + type state struct { + edge + constants map[llvm.Value]uint64 + } + seen := make(map[edge][]map[llvm.Value]uint64) + pending := []state{{edge: edge{block: entry}, constants: make(map[llvm.Value]uint64)}} var result uint8 for len(pending) != 0 { - block := pending[len(pending)-1] + current := pending[len(pending)-1] pending = pending[:len(pending)-1] - if block.IsNil() || seen[block] { + if current.block.IsNil() { continue } - seen[block] = true - switch block { + alreadySeen := false + for _, constants := range seen[current.edge] { + if sameCoroCFGConstants(constants, current.constants) { + alreadySeen = true + break + } + } + if alreadySeen { + continue + } + seen[current.edge] = append(seen[current.edge], current.constants) + switch current.block { case normal: result |= coroTestAwaitConsumeNormal continue @@ -766,9 +822,34 @@ func coroTestFirstReachableAwaitConsumes( result |= coroTestAwaitConsumeCanceled continue } - terminator := block.LastInstruction() - for successor := 0; successor < terminator.SuccessorsCount(); successor++ { - pending = append(pending, terminator.Successor(successor)) + constants := copyCoroCFGConstants(current.constants) + for instruction := current.block.FirstInstruction(); !instruction.IsNil(); instruction = llvm.NextInstruction(instruction) { + if !instruction.IsAPHINode().IsNil() { + value, ok := coroCFGPHIIncomingConstant(instruction, current.predecessor, constants) + if ok { + constants[instruction] = value + } else { + delete(constants, instruction) + } + continue + } + // Conditional stack cuts encode !completed as xor i1 %phi, true. + // Preserve that exact path fact so the inline-complete predecessor + // is not incorrectly considered able to enter the resume gate. + if instruction.InstructionOpcode() == llvm.Xor { + left, leftOK := coroCFGConstant(instruction.Operand(0), constants) + right, rightOK := coroCFGConstant(instruction.Operand(1), constants) + if leftOK && rightOK { + constants[instruction] = left ^ right + } + } + } + terminator := current.block.LastInstruction() + for _, successor := range executableTerminatorSuccessors(terminator, constants) { + pending = append(pending, state{ + edge: edge{block: successor, predecessor: current.block}, + constants: constants, + }) } } return result diff --git a/cl/coro_dispatch.go b/cl/coro_dispatch.go index 010ee6c42e..8d8843089d 100644 --- a/cl/coro_dispatch.go +++ b/cl/coro_dispatch.go @@ -899,15 +899,20 @@ func newCoroPlainDispatchEffectiveABI(p *context, patched *types.Signature) (cor // The ABI identity is structural at every function nesting depth. Parameter // and result names are source decoration, including inside callback types; // they must not make an otherwise identical producer and consumer disagree. - writeDispatchHashField(&key, "logical-signature", structuralEmissionABITypeKey(patched)) - writeDispatchHashField(&key, "physical-signature", structuralEmissionABITypeKey(physical)) - if err := appendCoroPlainDispatchTupleLayout(&key, p.prog, "params", physical.Params(), qualified); err != nil { + writeDispatchHashField(&key, "logical-signature", p.cachedStrictEmissionABITypeKey(patched)) + writeDispatchHashField(&key, "physical-signature", p.cachedStrictEmissionABITypeKey(physical)) + universe := p.immutableEmissionUniverse() + var typeKeys *emissionTypeKeyCache + if universe != nil { + typeKeys = &universe.emissionTypeKeys + } + if err := appendCoroPlainDispatchTupleLayout(&key, p.prog, "params", physical.Params(), qualified, typeKeys.strictABI); err != nil { return coroPlainDispatchABI{}, err } - if err := appendCoroPlainDispatchTupleLayout(&key, p.prog, "results", physical.Results(), qualified); err != nil { + if err := appendCoroPlainDispatchTupleLayout(&key, p.prog, "results", physical.Results(), qualified, typeKeys.strictABI); err != nil { return coroPlainDispatchABI{}, err } - if err := appendCoroPlainDispatchTypeLayout(&key, p.prog, "result-slot", resultSlot, qualified, make(map[types.Type]bool)); err != nil { + if err := appendCoroPlainDispatchTypeLayout(&key, p.prog, "result-slot", resultSlot, qualified, make(map[types.Type]bool), typeKeys.strictABI); err != nil { return coroPlainDispatchABI{}, err } sum := sha256.Sum256([]byte(key.String())) @@ -954,22 +959,43 @@ func writeDispatchHashField(builder *strings.Builder, name, value string) { builder.WriteByte('\n') } -func appendCoroPlainDispatchTupleLayout(builder *strings.Builder, prog llssa.Program, path string, tuple *types.Tuple, qualified types.Qualifier) error { +func appendCoroPlainDispatchTupleLayout( + builder *strings.Builder, + prog llssa.Program, + path string, + tuple *types.Tuple, + qualified types.Qualifier, + typeKeys ...func(types.Type) string, +) error { writeDispatchHashField(builder, path+".count", strconv.Itoa(tuple.Len())) for i := 0; i < tuple.Len(); i++ { - if err := appendCoroPlainDispatchTypeLayout(builder, prog, fmt.Sprintf("%s[%d]", path, i), tuple.At(i).Type(), qualified, make(map[types.Type]bool)); err != nil { + if err := appendCoroPlainDispatchTypeLayout(builder, prog, fmt.Sprintf("%s[%d]", path, i), tuple.At(i).Type(), qualified, make(map[types.Type]bool), typeKeys...); err != nil { return err } } return nil } -func appendCoroPlainDispatchTypeLayout(builder *strings.Builder, prog llssa.Program, path string, typ types.Type, qualified types.Qualifier, visiting map[types.Type]bool) error { +func appendCoroPlainDispatchTypeLayout( + builder *strings.Builder, + prog llssa.Program, + path string, + typ types.Type, + qualified types.Qualifier, + visiting map[types.Type]bool, + typeKeys ...func(types.Type) string, +) error { if typ == nil { return fmt.Errorf("coroutine plain dispatch ABI: nil type at %s", path) } typ = types.Unalias(typ) - writeDispatchHashField(builder, path+".type", structuralEmissionABITypeKey(typ)) + var typeKey string + if len(typeKeys) != 0 && typeKeys[0] != nil { + typeKey = typeKeys[0](typ) + } else { + typeKey = structuralEmissionABITypeKey(typ) + } + writeDispatchHashField(builder, path+".type", typeKey) physical := prog.Type(typ, llssa.InC) writeDispatchHashField(builder, path+".size", strconv.FormatUint(prog.SizeOf(physical), 10)) writeDispatchHashField(builder, path+".align", strconv.FormatUint(prog.AlignOf(physical), 10)) @@ -990,20 +1016,20 @@ func appendCoroPlainDispatchTypeLayout(builder *strings.Builder, prog llssa.Prog writeDispatchHashField(builder, path+".named-interface", "two-word-header") return nil } - return appendCoroPlainDispatchTypeLayout(builder, prog, path+".underlying", value.Underlying(), qualified, visiting) + return appendCoroPlainDispatchTypeLayout(builder, prog, path+".underlying", value.Underlying(), qualified, visiting, typeKeys...) case *types.Pointer: writeDispatchHashField(builder, path+".pointer", "opaque") case *types.Struct: writeDispatchHashField(builder, path+".fields", strconv.Itoa(value.NumFields())) for i := 0; i < value.NumFields(); i++ { writeDispatchHashField(builder, fmt.Sprintf("%s.field[%d].offset", path, i), strconv.FormatUint(prog.OffsetOf(physical, i), 10)) - if err := appendCoroPlainDispatchTypeLayout(builder, prog, fmt.Sprintf("%s.field[%d]", path, i), value.Field(i).Type(), qualified, visiting); err != nil { + if err := appendCoroPlainDispatchTypeLayout(builder, prog, fmt.Sprintf("%s.field[%d]", path, i), value.Field(i).Type(), qualified, visiting, typeKeys...); err != nil { return err } } case *types.Array: writeDispatchHashField(builder, path+".length", strconv.FormatInt(value.Len(), 10)) - return appendCoroPlainDispatchTypeLayout(builder, prog, path+".element", value.Elem(), qualified, visiting) + return appendCoroPlainDispatchTypeLayout(builder, prog, path+".element", value.Elem(), qualified, visiting, typeKeys...) case *types.Signature: // A signature here is the first field of LLGo's already-converted // two-pointer closure aggregate. LLVM opaque pointers make the code word @@ -1294,7 +1320,7 @@ func (p *context) newCoroDynamicDispatchEntryThunk( } targetClosureCtx := targetSig.Params().At(targetParam).Type() if !types.Identical(targetClosureCtx, closureCtx) && - structuralEmissionABITypeKey(targetClosureCtx) != structuralEmissionABITypeKey(closureCtx) { + p.cachedStrictEmissionABITypeKey(targetClosureCtx) != p.cachedStrictEmissionABITypeKey(closureCtx) { got := "" if targetSig.Params().Len() > targetParam { got = types.TypeString(targetSig.Params().At(targetParam).Type(), nil) diff --git a/cl/coro_dynamic_await_test.go b/cl/coro_dynamic_await_test.go index 860a09138c..7dfb70567a 100644 --- a/cl/coro_dynamic_await_test.go +++ b/cl/coro_dynamic_await_test.go @@ -152,11 +152,11 @@ func Apply(callback func(int) int, value int) int { } if !strings.Contains(applyIR, "@llvm.coro.promise") || !strings.Contains(applyIR, "call void @"+coroAwaitPrepareHookV1) || - !strings.Contains(applyIR, "call i1 @"+coroAwaitInlineHookV1) { + !strings.Contains(applyIR, "call i1 @"+coroAwaitInlineBeginHookV2) { t.Fatalf("Apply coroutine descriptor branch does not enter the shared child-await handoff:\n%s", applyIR) } await := strings.Index(applyIR, "call void @"+coroAwaitPrepareHookV1) - inline := strings.Index(applyIR, "call i1 @"+coroAwaitInlineHookV1) + inline := strings.Index(applyIR, "call i1 @"+coroAwaitInlineBeginHookV2) if await < coroCall[0] || inline < await || strings.Index(applyIR[inline:], "call i8 @llvm.coro.suspend") < 0 { t.Fatalf("Apply does not publish, try inline completion, and retain its dynamic-child slow suspend:\n%s", applyIR) } @@ -287,7 +287,7 @@ func Apply( } applyIR := requireCoroPhysicalFunction(t, module, "foo.Apply").String() if !strings.Contains(applyIR, "call void @"+coroAwaitPrepareHookV1) || - !strings.Contains(applyIR, "call i1 @"+coroAwaitInlineHookV1) || + !strings.Contains(applyIR, "call i1 @"+coroAwaitInlineBeginHookV2) || strings.Count(applyIR, "extractvalue") < 6 { t.Fatalf("aggregate descriptor branches did not hand off child and merge six typed results:\n%s", applyIR) } diff --git a/cl/coro_emitter_adapter.go b/cl/coro_emitter_adapter.go index 3cd7b262fc..201e250050 100644 --- a/cl/coro_emitter_adapter.go +++ b/cl/coro_emitter_adapter.go @@ -84,7 +84,7 @@ func (p *context) compileCoroInstructionPrologue(b llssa.Builder, instr ssa.Inst } func (p *context) compileCoroPatchInitAtBlock(b llssa.Builder) bool { - if !p.hasCoroPhysicalBody() { + if !p.hasStructuredOutcomePhysicalBody() { return false } p.compileCoroPatchInitAwait(b) @@ -132,7 +132,17 @@ func (p *context) tryCompileCoroPhysicalCall(b llssa.Builder, call *ssa.Call) (l p.observeCoroPhysicalControl(call, instructionPlan.control) } if p.hasOutcomePlainPhysicalBody() && instructionPlan.control != coroPhysicalControlDirectOutcome { - panic(fmt.Sprintf("outcome-plain DAG call selected incompatible frozen control recipe %s", instructionPlan.control)) + // A ProgramIR-finalized helper-free intrinsic remains a source *ssa.Call + // but has no physical callee edge. Let the ordinary intrinsic emitter own + // it. Every unrefined source call still fails closed here. + staticOutcome := false + if physical := p.coroEmissionPlan(); physical != nil { + staticOutcome = physical.staticOutcome + } + if instructionPlan.control != coroPhysicalControlNone || + !staticOutcome && !coroOutcomePlainLeafSemanticRecipe(instructionPlan.semantic) { + panic(fmt.Sprintf("outcome-plain DAG call selected incompatible frozen control recipe %s", instructionPlan.control)) + } } switch instructionPlan.control { case coroPhysicalControlDirectAwait: diff --git a/cl/coro_entry.go b/cl/coro_entry.go index 58e9e30e46..ad27360617 100644 --- a/cl/coro_entry.go +++ b/cl/coro_entry.go @@ -52,12 +52,19 @@ type plannedFunctionSymbol struct { patchOriginalInit bool wasmImport wasmImportSpec hasWasmImport bool + // outcomePlainTwin selects the compiler-owned synchronous static-call + // symbol while leaving plan.ManagedEntry and the dynamic descriptor on the + // ordinary coroutine primary. + outcomePlainTwin bool } func (e plannedFunctionSymbol) usesCoroPhysicalABI() bool { if !e.planned || !e.physical { return false } + if e.outcomePlainTwin { + return false + } if e.plan.Emission == coro.EmitCoroutine { return true } @@ -70,6 +77,9 @@ func (e plannedFunctionSymbol) usesOutcomePlainPhysicalABI() bool { if !e.planned || !e.physical { return false } + if e.outcomePlainTwin { + return true + } if e.plan.Emission == coro.EmitOutcomePlain { return true } @@ -77,6 +87,21 @@ func (e plannedFunctionSymbol) usesOutcomePlainPhysicalABI() bool { e.libraryEffect.ManagedEntry == coro.ManagedEntryOutcomePlain } +func (e plannedFunctionSymbol) hasOutcomePlainCapability() bool { + if !e.planned || !e.physical || !e.plan.HasStaticOutcome() { + return false + } + if !e.plan.StaticOutcome && (e.plan.AtomicCost == 0 || e.plan.AtomicCostCertificate == "") { + return false + } + if e.importedLibrary { + return e.plan.External == coro.ExternalKnown && e.plan.Emission == coro.EmitExternal && + e.libraryEffect.OutcomePlainSymbol != "" + } + return e.plan.External == coro.Defined && + (e.plan.Emission == coro.EmitOutcomePlain || e.plan.Emission == coro.EmitCoroutine) +} + // resolveFunctionSymbol is shared by function definitions and declarations so // they cannot independently choose different primary symbols. The physical // descriptor derives the signature from this exact entry. A nil Compilation is @@ -204,6 +229,23 @@ func (p *context) resolvePatchOriginalInitSymbol(fn *ssa.Function) (plannedFunct return entry, nil } +// resolvePatchOriginalInitOutcomeSymbol preserves the private original-init +// role while selecting its synchronous outcome twin. The generic outcome +// resolver starts from the public initializer base name and therefore cannot be +// used for this compiler-owned patch edge. +func (p *context) resolvePatchOriginalInitOutcomeSymbol(fn *ssa.Function) (plannedFunctionSymbol, error) { + entry, err := p.resolvePatchOriginalInitSymbol(fn) + if err != nil { + return plannedFunctionSymbol{}, err + } + if !entry.plan.HasStaticOutcome() || entry.baseName == "" { + return plannedFunctionSymbol{}, fmt.Errorf("patch original initializer %q has no synchronous outcome capability", entry.plan.ID) + } + entry.name = entry.baseName + coroOutcomePlainPrimarySuffix + entry.outcomePlainTwin = true + return entry, nil +} + func (p *context) mustPatchOriginalInitFunctionSymbol(fn *ssa.Function) plannedFunctionSymbol { entry, err := p.resolvePatchOriginalInitSymbol(fn) if err == nil { @@ -494,6 +536,7 @@ func (e plannedFunctionSymbol) checkSupportedWithPhysicalPlan(accept func(*coroP e.plan.AtomicCost != e.libraryEffect.AtomicCost || e.plan.AtomicCostProof != e.libraryEffect.AtomicCostProof || e.plan.AtomicCostCertificate != e.libraryEffect.AtomicCostCertificate || + e.plan.StaticOutcome != e.libraryEffect.StaticOutcome || e.libraryEffect.PrimarySymbol == "" { return fmt.Errorf( "external coroutine emission %q disagrees with its producer library fact", @@ -522,6 +565,19 @@ func (e plannedFunctionSymbol) checkSupportedWithPhysicalPlan(accept func(*coroP return nil } +func (e plannedFunctionSymbol) checkOutcomePlainSupported() error { + if !e.outcomePlainTwin || !e.hasOutcomePlainCapability() || e.importedLibrary || + e.plan.External != coro.Defined || e.plan.Emission != coro.EmitCoroutine || + e.plan.ManagedEntry != coro.ManagedEntryCoroutine { + return fmt.Errorf("outcome-plain twin %q has no owned coroutine-primary capability", e.plan.ID) + } + physical, err := e.sealedPhysicalFunctionPlan() + if err != nil { + return err + } + return validateOutcomePlainFrozenPlan(physical, e.plan) +} + // sealedPhysicalFunctionPlan is the single post-freeze lookup authority shared // by every managed physical entry ABI. Entry variants may validate different // capabilities, but none may create a second ProgramIR access path. @@ -696,6 +752,24 @@ func (c *Compilation) preflightCoroPlan() error { return c.coroPreflightErr } +// CoroProgramCapabilities returns the optional runtime services demanded by +// the final reachable physical recipes. It deliberately runs the same atomic +// preflight used by codegen, so a build cannot publish bootstrap capability +// bits from a partial semantic plan and later emit a different operation set. +func (c *Compilation) CoroProgramCapabilities() (coro.ProgramCapabilities, error) { + if c == nil { + return 0, fmt.Errorf("coroutine program capabilities require a compilation") + } + if err := c.preflightCoroPlan(); err != nil { + return 0, err + } + universe := c.immutableEmissionUniverse() + if universe == nil || universe.coroProgramIR == nil { + return 0, fmt.Errorf("coroutine program capabilities require a prepared ProgramIR") + } + return universe.coroProgramIR.programCapabilities() +} + func (p *context) mustFunctionSymbol(fn *ssa.Function) plannedFunctionSymbol { entry, err := p.resolveFunctionSymbol(fn) if err == nil { @@ -707,6 +781,41 @@ func (p *context) mustFunctionSymbol(fn *ssa.Function) plannedFunctionSymbol { return entry } +// mustOutcomePlainFunctionSymbol selects the proof-carrying synchronous entry +// used only by one exact static call. It never changes the managed primary +// published through a function value, method table, interface, or reflection. +func (p *context) mustOutcomePlainFunctionSymbol(fn *ssa.Function) plannedFunctionSymbol { + entry, err := p.resolveFunctionSymbol(fn) + if err == nil { + err = entry.checkSupported() + } + if err == nil && !entry.hasOutcomePlainCapability() { + err = fmt.Errorf("outcome-plain static entry %q has no frozen atomic-cost capability", entry.plan.ID) + } + if err == nil { + if entry.importedLibrary { + entry.name = entry.libraryEffect.OutcomePlainSymbol + } else { + if entry.baseName == "" { + err = fmt.Errorf("outcome-plain static entry %q has no frozen base symbol", entry.plan.ID) + } else { + entry.name = entry.baseName + coroOutcomePlainPrimarySuffix + } + } + entry.outcomePlainTwin = entry.plan.Emission != coro.EmitOutcomePlain || + entry.plan.ManagedEntry != coro.ManagedEntryOutcomePlain + } + if err == nil && entry.outcomePlainTwin && !entry.importedLibrary && + entry.emission != nil && entry.emission.coroProgramIR != nil && + entry.emission.coroProgramIR.physicalPlansSealed { + err = entry.checkOutcomePlainSupported() + } + if err != nil { + panic(err) + } + return entry +} + // mustRawPlainFunctionSymbol selects the separately planned legacy Go-ABI body // for one member of an exactly validated raw synchronous closure. It never // changes the managed primary selected by mustFunctionSymbol. Captured diff --git a/cl/coro_entry_test.go b/cl/coro_entry_test.go index cdced742f2..6e7f8b099c 100644 --- a/cl/coro_entry_test.go +++ b/cl/coro_entry_test.go @@ -694,7 +694,7 @@ func Complex(ch chan int) int { t.Fatalf("verify demanded channel/control-flow coroutine: %v\n%s", err, module.String()) } body := requireCoroPhysicalFunction(t, module, "foo.Complex").String() - if !strings.Contains(body, coroChanRecvParkHookV1) || !strings.Contains(body, "switch i32") { + if !strings.Contains(body, coroChanRecvTryParkHookV2) || !strings.Contains(body, "switch i32") { t.Fatalf("complex coroutine lacks channel park/control-flow lowering:\n%s", body) } } diff --git a/cl/coro_foreign_funcval.go b/cl/coro_foreign_funcval.go index a4b8c1ecc6..5714c08cc7 100644 --- a/cl/coro_foreign_funcval.go +++ b/cl/coro_foreign_funcval.go @@ -115,7 +115,7 @@ func (u *EmissionUniverse) managedForeignFunctionValueWrapperStructuralKey( "llgo-managed-foreign-function-value-wrapper-v1", owner.identity, identity, - structuralEmissionTypeKey(effective), + u.emissionTypeKeys.strict(effective), ), nil } diff --git a/cl/coro_foreign_reentry.go b/cl/coro_foreign_reentry.go index 82fb694dd4..10c131f720 100644 --- a/cl/coro_foreign_reentry.go +++ b/cl/coro_foreign_reentry.go @@ -85,7 +85,7 @@ func (p *context) coroForeignReentryPlainRamp( "cl-coro-foreign-reentry-plain-ramp-v1", string(entry.plan.ID), entry.name, - structuralEmissionABITypeKey(sourceSignature), + p.cachedStrictEmissionABITypeKey(sourceSignature), strconv.Itoa(p.prog.PointerSize()), ) name := coroForeignReentryPlainRampPrefixV1 + emissionDigest(key) @@ -223,7 +223,7 @@ func (p *context) coroForeignReentryAdapter( "cl-coro-foreign-reentry-adapter-v1", string(entry.plan.ID), entry.name, - structuralEmissionABITypeKey(callbackSignature), + p.cachedStrictEmissionABITypeKey(callbackSignature), strconv.Itoa(p.prog.PointerSize()), ) name := coroForeignReentryAdapterPrefixV1 + emissionDigest(key) diff --git a/cl/coro_frame_retention.go b/cl/coro_frame_retention.go index 30a59088a8..cd8e228656 100644 --- a/cl/coro_frame_retention.go +++ b/cl/coro_frame_retention.go @@ -38,6 +38,12 @@ type coroFrameRetentionProof struct { // allocations are the exact park-transaction cells reclassified from an // x/tools Heap Alloc into storage owned by the LLVM coroutine frame. allocations map[*ssa.Alloc]struct{} + // borrowedAllocations are ordinary fresh Go cells whose complete static + // address graph is proven not to survive the owning function. x/tools marks + // them Heap at a conservative interprocedural call boundary; the stronger + // closed-world proof permits target-bounded LLVM frame/native-stack storage + // without source annotations or per-operation runtime scratch fields. + borrowedAllocations map[*ssa.Alloc]coro.SSABorrowedAllocationProof // managedHeapAllocations remain ordinary Go heap allocations. Each fact is // admitted only after the frozen lowered-call plan proves its exact AllocZ // path; the resulting pointer may then be conservatively scanned from this @@ -271,7 +277,19 @@ func (a *coroPhysicalPureSSAAudit) frameRetainsAllocation(alloc *ssa.Alloc) bool if proof == nil { return false } - _, ok := proof.allocations[alloc] + if _, ok := proof.allocations[alloc]; ok { + return true + } + _, ok := proof.borrowedAllocations[alloc] + return ok +} + +func (a *coroPhysicalPureSSAAudit) frameRetainsBorrowedAllocation(alloc *ssa.Alloc) bool { + proof := a.currentFrameRetentionProof() + if proof == nil { + return false + } + _, ok := proof.borrowedAllocations[alloc] return ok } @@ -298,6 +316,7 @@ func (a *coroPhysicalPureSSAAudit) currentFrameRetentionProof() *coroFrameRetent func (a *coroPhysicalPureSSAAudit) proveCurrentFrameRetention() *coroFrameRetentionProof { proof := &coroFrameRetentionProof{ allocations: make(map[*ssa.Alloc]struct{}), + borrowedAllocations: make(map[*ssa.Alloc]coro.SSABorrowedAllocationProof), managedHeapAllocations: make(map[*ssa.Alloc]coroFrameRetentionManagedHeapAllocation), terminalResultAllocations: make(map[*ssa.Alloc]struct{}), exactRoots: make(map[ssa.Value]coroFrameRetentionExactRoot), @@ -311,7 +330,6 @@ func (a *coroPhysicalPureSSAAudit) proveCurrentFrameRetention() *coroFrameRetent if a.frameRetentionABI == CoroFrameRetentionParkABIV2 { a.proveParkFrameRetention(proof) } - a.proveManagedHeapAllocations(proof) terminalAllocations, err := coroStaticTerminalReconstructionAllocations(a.fn) if err != nil { // Static-cleanup preflight reports the precise structural error. Do not @@ -321,11 +339,54 @@ func (a *coroPhysicalPureSSAAudit) proveCurrentFrameRetention() *coroFrameRetent for _, allocation := range terminalAllocations { proof.terminalResultAllocations[allocation] = struct{}{} } + a.proveBorrowedHeapAllocations(proof) + a.proveManagedHeapAllocations(proof) newCoroFrameRetentionRootBuilder(a, proof).prove() proof.rootDigest = coroFrameRetentionRootDigest(a, proof) return proof } +// proveBorrowedHeapAllocations strengthens x/tools' conservative Heap bit only +// for fresh cells whose complete interprocedural address graph remains bounded +// by the owning call. The exact physical type must fit the target's native +// local limit: outcome-plain functions use an entry alloca, while CoroSplit +// incorporates the same storage into a stackless frame. Each physical recipe +// still zeroes the cell at the source Alloc instruction, preserving loop and +// conditional allocation semantics despite entry-owned storage. +func (a *coroPhysicalPureSSAAudit) proveBorrowedHeapAllocations(proof *coroFrameRetentionProof) { + if a == nil || a.ctx == nil || a.ctx.prog == nil || a.universe == nil || a.fn == nil || proof == nil { + return + } + bitcastAllocation := (*ssa.Alloc)(nil) + if bitcast, exact := coro.ProveSSAExactScalarBitcast(a.fn); exact { + bitcastAllocation = bitcast.Allocation + } + for _, block := range a.fn.Blocks { + for _, instruction := range block.Instrs { + allocation, ok := instruction.(*ssa.Alloc) + if !ok || !allocation.Heap || allocation == bitcastAllocation || + a.ctx.skipSyntheticMakeSliceAlloc(allocation) || isEmissionVargsAlloc(a.ctx, allocation) { + continue + } + if _, park := proof.allocations[allocation]; park { + continue + } + if _, terminal := proof.terminalResultAllocations[allocation]; terminal { + continue + } + pointer, ok := types.Unalias(a.typeOf(allocation.Type())).Underlying().(*types.Pointer) + if !ok || a.ctx.prog.LocalGoTypeExceedsNativeStack(a.typeOf(pointer.Elem())) || + validateCoroPhysicalSSAValueType(a.typeOf(pointer.Elem())) != nil { + continue + } + borrow, exact := proveCoroBorrowedAllocation(a.universe, allocation) + if exact { + proof.borrowedAllocations[allocation] = borrow + } + } + } +} + func (a *coroPhysicalPureSSAAudit) proveParkFrameRetention(proof *coroFrameRetentionProof) { if a == nil || proof == nil || a.universe == nil || a.fn == nil { return @@ -468,6 +529,9 @@ func (a *coroPhysicalPureSSAAudit) proveManagedHeapAllocations(proof *coroFrameR if _, frameLocal := proof.allocations[alloc]; frameLocal { continue } + if _, borrowed := proof.borrowedAllocations[alloc]; borrowed { + continue + } fact, reason := a.managedHeapAllocationCapability(alloc) if reason == "" { proof.managedHeapAllocations[alloc] = fact @@ -817,7 +881,9 @@ func (b *coroFrameRetentionRootBuilder) traceAddress(value ssa.Value, use ssa.In if _, managed := b.proof.managedHeapAllocations[value]; managed { kind = coroFrameRetentionRootManagedHeapAllocation } else if value.Heap { - if _, retained := b.proof.allocations[value]; !retained { + _, parkRetained := b.proof.allocations[value] + _, borrowRetained := b.proof.borrowedAllocations[value] + if !parkRetained && !borrowRetained { return trace, false } } @@ -2149,6 +2215,10 @@ func coroFrameRetentionRootDigest(a *coroPhysicalPureSSAAudit, proof *coroFrameR return "" } builder := newCoroFrameRetentionRootBuilder(a, proof) + typeKey := structuralEmissionTypeKey + if a.universe != nil { + typeKey = a.universe.emissionTypeKeys.strict + } valueID := func(value ssa.Value) string { if value == nil { return "none" @@ -2171,7 +2241,7 @@ func coroFrameRetentionRootDigest(a *coroPhysicalPureSSAAudit, proof *coroFrameR for _, rootValue := range proof.exactRetainedRoots() { root := proof.exactRoots[rootValue] fields = append(fields, framedEmissionKey( - "root", valueID(root.value), strconv.Itoa(int(root.kind)), structuralEmissionTypeKey(a.typeOf(root.value.Type())), + "root", valueID(root.value), strconv.Itoa(int(root.kind)), typeKey(a.typeOf(root.value.Type())), )) } managedAllocationKeys := make([]*ssa.Alloc, 0, len(proof.managedHeapAllocations)) @@ -2188,7 +2258,7 @@ func coroFrameRetentionRootDigest(a *coroPhysicalPureSSAAudit, proof *coroFrameR mode = "module-zero-sentinel" } fields = append(fields, framedEmissionKey( - "managed-heap-allocation", valueID(allocation), structuralEmissionTypeKey(a.typeOf(allocation.Type())), + "managed-heap-allocation", valueID(allocation), typeKey(a.typeOf(allocation.Type())), mode, fact.helper, string(fact.helperTarget), fact.helperEmission.String(), )) } @@ -2201,7 +2271,7 @@ func coroFrameRetentionRootDigest(a *coroPhysicalPureSSAAudit, proof *coroFrameR }) for _, allocation := range terminalAllocationKeys { fields = append(fields, framedEmissionKey( - "cleanup-terminal-result-allocation", valueID(allocation), structuralEmissionTypeKey(a.typeOf(allocation.Type())), + "cleanup-terminal-result-allocation", valueID(allocation), typeKey(a.typeOf(allocation.Type())), )) } addressKeys := make([]coroFrameRetentionAddressUse, 0, len(proof.stableAddresses)) @@ -2221,7 +2291,7 @@ func coroFrameRetentionRootDigest(a *coroPhysicalPureSSAAudit, proof *coroFrameR if fact.nonNil { nilMode = "non-nil" } - entry := []string{"address", valueID(key.value), instructionID(key.use), structuralEmissionTypeKey(a.typeOf(key.value.Type())), nilMode} + entry := []string{"address", valueID(key.value), instructionID(key.use), typeKey(a.typeOf(key.value.Type())), nilMode} for _, evidence := range fact.evidence { entry = append(entry, "evidence="+instructionID(evidence)) } @@ -2265,6 +2335,21 @@ func coroFrameRetentionRootDigest(a *coroPhysicalPureSSAAudit, proof *coroFrameR for _, allocation := range allocationKeys { fields = append(fields, framedEmissionKey("park-allocation", valueID(allocation))) } + borrowedAllocationKeys := make([]*ssa.Alloc, 0, len(proof.borrowedAllocations)) + for allocation := range proof.borrowedAllocations { + borrowedAllocationKeys = append(borrowedAllocationKeys, allocation) + } + sort.Slice(borrowedAllocationKeys, func(i, j int) bool { + return builder.valueOrder[borrowedAllocationKeys[i]] < builder.valueOrder[borrowedAllocationKeys[j]] + }) + for _, allocation := range borrowedAllocationKeys { + borrow := proof.borrowedAllocations[allocation] + fields = append(fields, framedEmissionKey( + "borrowed-allocation", valueID(allocation), typeKey(a.typeOf(allocation.Type())), + strconv.FormatUint(uint64(borrow.FunctionsVisited), 10), + strconv.FormatUint(uint64(borrow.ParametersProven), 10), + )) + } sum := sha256.Sum256([]byte(framedEmissionKey(fields...))) return hex.EncodeToString(sum[:]) } diff --git a/cl/coro_frame_retention_test.go b/cl/coro_frame_retention_test.go index 75ac4bc706..d400991925 100644 --- a/cl/coro_frame_retention_test.go +++ b/cl/coro_frame_retention_test.go @@ -62,6 +62,98 @@ func Root(addr *uint32) uint32 { } ` +const coroBorrowedAllocationFixture = `package foo + +type transaction struct { + value uint32 +} + +func borrow(value *transaction) bool { return value != nil } + +func Root(limit uint32) bool { + var value transaction + if !borrow(&value) { + return false + } + for value.value < limit { + value.value++ + } + return value.value == limit +} +` + +func TestCoroBorrowedAllocationUsesPhysicalLocalStorage(t *testing.T) { + prog, ssaPkg, files, universe, plan := prepareCoroPreemptTestPlan( + t, + coroBorrowedAllocationFixture, + []coroRootFactoryTestRoot{{name: "Root", demand: coro.AsyncDemand}}, + nil, + -1, + ) + defer prog.Dispose() + root := ssaPkg.Func("Root") + audit, err := newCoroPhysicalPureSSAAudit(universe, plan, root, "") + if err != nil { + t.Fatal(err) + } + proof := audit.currentFrameRetentionProof() + allocations := coroFrameRetentionHeapAllocs(root) + if len(allocations) != 1 || len(proof.borrowedAllocations) != 1 || + len(proof.managedHeapAllocations) != 0 { + t.Fatalf("borrowed fixture SSA/borrowed/managed allocations = %d/%d/%d, want 1/1/0", + len(allocations), len(proof.borrowedAllocations), len(proof.managedHeapAllocations)) + } + borrow, retained := proof.borrowedAllocations[allocations[0]] + rootFact, rooted := proof.exactRoots[allocations[0]] + if !retained || borrow.FunctionsVisited < 2 || borrow.ParametersProven < 1 || + !rooted || rootFact.kind != coroFrameRetentionRootLocalAddress { + t.Fatalf("borrowed allocation fact = %+v, retained=%t root=%+v rooted=%t", + borrow, retained, rootFact, rooted) + } + if reason := audit.validateAlloc(allocations[0]); reason != "" { + t.Fatalf("borrowed allocation rejected: %s", reason) + } + owners := universe.sortedUseOwners(root) + if len(owners) != 1 { + t.Fatalf("borrowed Root owners = %d, want 1", len(owners)) + } + plainContext, err := universe.functionABIContext(root, owners[0]) + if err != nil { + t.Fatal(err) + } + plainContext.compilation = &Compilation{CoroPlan: plan, EmissionUniverse: universe} + plainContext.rawPlainBody = true + if !plainContext.selectCoroPlainBorrowedAllocation(allocations[0]) { + frozen, frozenErr := universe.coroProgramIR.sitePlan(plainContext, allocations[0]) + t.Fatalf("raw-plain body did not consume its frozen borrowed-allocation SitePlan: plan=%+v err=%v", + frozen.plainAllocation, frozenErr) + } + + compilation := &Compilation{CoroPlan: plan, EmissionUniverse: universe} + enableCoroPreemptCompilation(compilation) + pkg, _, err := NewPackageExWithEmbedOptions( + prog, nil, nil, nil, ssaPkg, files, goembed.VarMap{}, + PackageOptions{Compilation: compilation}, + ) + if err != nil { + t.Fatal(err) + } + module := pkg.Module() + defer module.Dispose() + rootIR := requireCoroPhysicalFunction(t, module, "foo.Root").String() + if strings.Contains(rootIR, "runtime.AllocZ") || !strings.Contains(rootIR, "alloca %foo.transaction") { + t.Fatalf("borrowed allocation did not lower to local physical storage:\n%s", rootIR) + } + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify borrowed allocation before CoroSplit: %v\n%s", err, module.String()) + } + runCoroABITestPipeline(t, prog, module) + if resume := module.NamedFunction("foo.Root$coro.resume"); resume.IsNil() || + strings.Contains(resume.String(), "runtime.AllocZ") { + t.Fatalf("CoroSplit lost borrowed local storage:\n%s", module.String()) + } +} + func TestCoroGenericParkStateRetentionIsSourceIndependent(t *testing.T) { for _, symbol := range []string{"__llgo_coro_fixture_prepare", "__llgo_coro_another_source_prepare"} { t.Run(symbol, func(t *testing.T) { diff --git a/cl/coro_funcpc.go b/cl/coro_funcpc.go index 267eca9fae..5d432b161a 100644 --- a/cl/coro_funcpc.go +++ b/cl/coro_funcpc.go @@ -103,7 +103,7 @@ func (u *EmissionUniverse) aliasPatchedFuncPCABI0Declarations() error { intrinsic.Signature == nil { continue } - intrinsicSignature := structuralEmissionABITypeKey(u.effectiveType(prepared, intrinsic, intrinsic.Signature, false)) + intrinsicSignature := u.emissionTypeKeys.strictABI(u.effectiveType(prepared, intrinsic, intrinsic.Signature, false)) if originalSignature != intrinsicSignature { return fmt.Errorf( "prepare emission universe: patched internal/abi.%s declaration and alternate intrinsic have different structural ABI signatures", localName, @@ -131,7 +131,7 @@ func (u *EmissionUniverse) aliasPatchedFuncPCABI0Declarations() error { candidateOpcode, opcodeOK := u.intrinsicOps[candidateOwnerKey] candidateSignature := "" if candidate.Signature != nil { - candidateSignature = structuralEmissionABITypeKey(u.effectiveType(prepared, candidate, candidate.Signature, false)) + candidateSignature = u.emissionTypeKeys.strictABI(u.effectiveType(prepared, candidate, candidate.Signature, false)) } if kindOK && candidateKind == llgoInstr && opcodeOK && candidateOpcode == llgoFuncPCABI0 && candidateSignature == originalSignature { diff --git a/cl/coro_implicit_fault.go b/cl/coro_implicit_fault.go index e50fb0829a..abfc0d242a 100644 --- a/cl/coro_implicit_fault.go +++ b/cl/coro_implicit_fault.go @@ -226,8 +226,8 @@ func (p *context) compileCoroPlannedWrapNilGuard( instruction ssa.Instruction, base, recvType, methodName llssa.Expr, ) llssa.Expr { - if !p.hasCoroPhysicalBody() || b == nil || b.Func != p.fn || instruction == nil { - panic("value-method nil guard escaped its physical coroutine body") + if !p.hasStructuredOutcomePhysicalBody() || b == nil || b.Func != p.fn || instruction == nil { + panic("value-method nil guard escaped its structured physical body") } if !p.coroEmissionExplicitStatus() { panic("value-method nil guard requires the PhysicalABIV1 explicit-status panic ABI") @@ -239,8 +239,7 @@ func (p *context) compileCoroPlannedWrapNilGuard( b.If(isNil, fault, normal) b.SetBlockEx(fault, llssa.AtEnd, false) - typeSlot := p.coroFrameAlloca(p.prog.VoidPtr()) - dataSlot := p.coroFrameAlloca(p.prog.VoidPtr()) + typeSlot, dataSlot := p.coroFaultPayloadSlots(b) b.Store(typeSlot, p.prog.Nil(p.prog.VoidPtr())) b.Store(dataSlot, p.prog.Nil(p.prog.VoidPtr())) payload := p.pkg.NewFunc( @@ -342,10 +341,9 @@ func (p *context) compileCoroIndexAddrPlanned( base, index llssa.Expr, plan coroPhysicalInstructionPlan, ) llssa.Expr { - body := p.coroBody() - if body == nil || operation == nil || operation.X == nil || + if !p.hasStructuredOutcomePhysicalBody() || operation == nil || operation.X == nil || b == nil || b.Func != p.fn { - panic("structured coroutine IndexAddr escaped its physical body") + panic("structured IndexAddr escaped its physical body") } if plan.recipe != coroPhysicalInstructionIndexAddr { panic("structured coroutine IndexAddr has the wrong physical recipe") @@ -396,9 +394,9 @@ func (p *context) compileCoroIndexPlanned( takeArrayAddr func() (addr llssa.Expr, zero bool), plan coroPhysicalInstructionPlan, ) llssa.Expr { - if p == nil || p.coroBody() == nil || operation == nil || operation.X == nil || + if p == nil || !p.hasStructuredOutcomePhysicalBody() || operation == nil || operation.X == nil || operation.Index == nil || b == nil || b.Func != p.fn { - panic("structured coroutine Index escaped its physical body") + panic("structured Index escaped its physical body") } if plan.recipe != coroPhysicalInstructionIndex { @@ -455,14 +453,13 @@ func (p *context) compileCoroSlicePlanned( base, low, high, max llssa.Expr, plan coroPhysicalInstructionPlan, ) llssa.Expr { - body := p.coroBody() - if body == nil || operation == nil || operation.X == nil || + if !p.hasStructuredOutcomePhysicalBody() || operation == nil || operation.X == nil || b == nil || b.Func != p.fn { - panic("structured coroutine Slice escaped its physical body") + panic("structured Slice escaped its physical body") } if (plan.nilGuard || plan.boundsGuard) && - (!p.coroEmissionExplicitStatus() || body.abi.version < coroPhysicalABIVersionV1) { - panic("structured coroutine Slice requires the PhysicalABIV1 explicit-status panic ABI") + !p.coroEmissionExplicitStatus() { + panic("structured Slice requires the explicit-status panic ABI") } if plan.recipe != coroPhysicalInstructionSlice || plan.boundsDisabled != b.Prog.BoundsChecksDisabled() || @@ -648,10 +645,12 @@ func (p *context) compileCoroTerminalFaultWithOperands( if b == nil || b.Func != p.fn { panic("outcome-plain terminal fault escaped its physical body") } - if kind != coroFaultNilV1 || operands != nil { - panic("outcome-plain v0 admitted a parameterized or non-nil language fault") + if kind == coroFaultNilV1 && operands == nil { + p.outcomePlainBody().publishNilFault(b) + return } - p.outcomePlainBody().publishNilFault(b) + typeWord, dataWord := p.materializeCoroFaultPayloadWithOperands(b, kind, operands) + p.compileOutcomePlainPanicPair(b, typeWord, dataWord) return } body := p.coroBody() @@ -725,12 +724,11 @@ func (p *context) materializeCoroFaultPayloadWithOperands( operands *coroFaultOperands, ) (typeWord, dataWord llssa.Expr) { body := p.coroBody() - if body == nil || b == nil || b.Func != p.fn || - !p.coroEmissionExplicitStatus() || body.abi.version < coroPhysicalABIVersionV1 { - panic("coroutine fault payload materialization requires an explicit-status PhysicalABIV1 body") + if !p.hasStructuredOutcomePhysicalBody() || b == nil || b.Func != p.fn || + !p.coroEmissionExplicitStatus() || body != nil && body.abi.version < coroPhysicalABIVersionV1 { + panic("fault payload materialization requires an explicit-status structured body") } - typeSlot := p.coroFrameAlloca(p.prog.VoidPtr()) - dataSlot := p.coroFrameAlloca(p.prog.VoidPtr()) + typeSlot, dataSlot := p.coroFaultPayloadSlots(b) b.Store(typeSlot, p.prog.Nil(p.prog.VoidPtr())) b.Store(dataSlot, p.prog.Nil(p.prog.VoidPtr())) args := []llssa.Expr{p.prog.IntVal(uint64(kind), p.prog.Uint32())} @@ -753,6 +751,20 @@ func (p *context) materializeCoroFaultPayloadWithOperands( return b.Load(typeSlot), b.Load(dataSlot) } +// coroFaultPayloadSlots keeps payload cells in the LLVM coroutine ramp when a +// body may suspend, and on the ordinary native stack for a synchronous outcome +// twin. Both lifetimes dominate the immediate payload helper call and terminal +// publication; the payload itself is runtime-owned persistent storage. +func (p *context) coroFaultPayloadSlots(b llssa.Builder) (typeSlot, dataSlot llssa.Expr) { + if p == nil || !p.hasStructuredOutcomePhysicalBody() || b == nil || b.Func != p.fn { + panic("fault payload slots escaped their structured physical body") + } + if p.hasCoroPhysicalBody() { + return p.coroFrameAlloca(p.prog.VoidPtr()), p.coroFrameAlloca(p.prog.VoidPtr()) + } + return b.AllocaT(p.prog.VoidPtr()), b.AllocaT(p.prog.VoidPtr()) +} + // enterCoroPropagatedNilFault transports the allocation-free nil-fault status // through synchronous outcome bodies. The first full coroutine parent uses // its existing V1 terminal-fault path, so certified outcome functions contain diff --git a/cl/coro_interface_dispatch.go b/cl/coro_interface_dispatch.go index 5de8af0517..904f06ea3b 100644 --- a/cl/coro_interface_dispatch.go +++ b/cl/coro_interface_dispatch.go @@ -331,7 +331,11 @@ func validateCoroInterfaceDispatchCandidate( if err != nil { return fail("derive effective source call signature: %v", err) } - if entrySignature == nil || !coroInterfaceDispatchSignaturesIdentical(effectiveSourceSignature, entrySignature) { + var typeKeys *emissionTypeKeyCache + if universe != nil { + typeKeys = &universe.emissionTypeKeys + } + if entrySignature == nil || !coroInterfaceDispatchSignaturesIdentical(effectiveSourceSignature, entrySignature, typeKeys.strictABI) { return fail("effective method entry signature %v does not match source call signature %v", entrySignature, effectiveSourceSignature) } @@ -339,7 +343,7 @@ func validateCoroInterfaceDispatchCandidate( if err != nil { return fail("derive effective target signature: %v", err) } - if targetSignature == nil || !coroInterfaceDispatchSignaturesIdentical(effectiveSourceSignature, coroInterfaceDispatchCanonicalSignature(targetSignature)) { + if targetSignature == nil || !coroInterfaceDispatchSignaturesIdentical(effectiveSourceSignature, coroInterfaceDispatchCanonicalSignature(targetSignature), typeKeys.strictABI) { return fail("effective source call signature %v does not match receiver-free target signature %v", effectiveSourceSignature, targetSignature) } if len(target.Params) != target.Signature.Params().Len()+1 || target.Params[0] == nil || !types.Identical(target.Params[0].Type(), recv.Type()) { @@ -387,10 +391,16 @@ func coroInterfaceDispatchEffectiveCallableSignature( return coroInterfaceDispatchCanonicalSignature(coroInterfaceDispatchCallableSignature(signature)), nil } -func coroInterfaceDispatchSignaturesIdentical(left, right *types.Signature) bool { +func coroInterfaceDispatchSignaturesIdentical( + left, right *types.Signature, + typeKeys ...func(types.Type) string, +) bool { if left == nil || right == nil { return left == right } + if len(typeKeys) != 0 && typeKeys[0] != nil { + return typeKeys[0](left) == typeKeys[0](right) + } return structuralEmissionABITypeKey(left) == structuralEmissionABITypeKey(right) } diff --git a/cl/coro_interface_dispatch_test.go b/cl/coro_interface_dispatch_test.go index 24385c6eb9..962228fdfd 100644 --- a/cl/coro_interface_dispatch_test.go +++ b/cl/coro_interface_dispatch_test.go @@ -309,7 +309,7 @@ func Root(value interface{ As(any) bool }, target any, flag bool) bool { if !strings.Contains(rootIR, "coro.dispatch.version.invalid") || !strings.Contains(rootIR, "coro.dispatch.flags.unknown") || !strings.Contains(rootIR, "call void @"+coroAwaitPrepareHookV1) || - !strings.Contains(rootIR, "call i1 @"+coroAwaitInlineHookV1) { + !strings.Contains(rootIR, "call i1 @"+coroAwaitInlineBeginHookV2) { t.Fatalf("open interface invoke did not enter validated descriptor child-await lowering:\n%s", rootIR) } if strings.Contains(rootIR, "call i1 %") && !strings.Contains(rootIR, "coro.dispatch") { diff --git a/cl/coro_library_effect_summary.go b/cl/coro_library_effect_summary.go index 9f88b87e48..0edd2005fa 100644 --- a/cl/coro_library_effect_summary.go +++ b/cl/coro_library_effect_summary.go @@ -29,7 +29,7 @@ import ( const ( coroLibraryFunctionABIDigestDomain = "llgo.coro.library-function-abi.v1" coroLibraryExportABIDigestDomain = "llgo.coro.library-export-abi.v1" - coroLibrarySummarySymbolPrefix = "__llgo_coro_library_effect_v6." + coroLibrarySummarySymbolPrefix = "__llgo_coro_library_effect_" + coro.LibraryEffectSummaryVersion + "." ) // CoroLibraryEffectView is the immutable archive-facing projection of a @@ -121,7 +121,7 @@ func (view CoroLibraryEffectView) FunctionABIHash( strconv.Itoa(metadata.PointerBits), metadata.Endianness, metadata.DataLayout, - structuralEmissionABITypeKey(signature), + u.emissionTypeKeys.strictABI(signature), )), nil } @@ -155,7 +155,7 @@ func (view CoroLibraryEffectView) ExportABIHash( strconv.Itoa(metadata.PointerBits), metadata.Endianness, metadata.DataLayout, - structuralEmissionABITypeKey(signature), + u.emissionTypeKeys.strictABI(signature), )), nil } @@ -202,6 +202,16 @@ func (view CoroLibraryEffectView) ValidateFunction( fact.ID, primary, fact.PrimarySymbol, ) } + outcome := "" + if fact.HasStaticOutcome() { + outcome = base + coroOutcomePlainPrimarySuffix + } + if fact.OutcomePlainSymbol != outcome { + return fmt.Errorf( + "coroutine library outcome-plain symbol for %q is %q, producer published %q", + fact.ID, outcome, fact.OutcomePlainSymbol, + ) + } if fact.RawPlainSymbol != "" && fact.RawPlainSymbol != base { return fmt.Errorf( "coroutine library raw-plain symbol for %q is %q, producer published %q", @@ -384,6 +394,7 @@ func (c *Compilation) validateCoroLibraryEffects() error { functionPlan.AtomicCost != fact.AtomicCost || functionPlan.AtomicCostProof != fact.AtomicCostProof || functionPlan.AtomicCostCertificate != fact.AtomicCostCertificate || + functionPlan.StaticOutcome != fact.StaticOutcome || functionPlan.Emission != coro.EmitNone && functionPlan.Emission != coro.EmitExternal { return fmt.Errorf( "coroutine library effect %q disagrees with final consumer plan: plan=%+v producer=%+v ignored=%t", @@ -391,7 +402,7 @@ func (c *Compilation) validateCoroLibraryEffects() error { ) } // RawPlainSymbol is retained in the producer record so a later lowering - // can bind exact legacy crossings without rediscovering symbols. The v5 + // can bind exact legacy crossings without rediscovering symbols. The v8 // managed-function consumer does not yet own an external raw-body capability, // however: mustRawPlainFunctionSymbol deliberately accepts only a // locally defined variant. Reject every imported raw demand here even @@ -399,18 +410,18 @@ func (c *Compilation) validateCoroLibraryEffects() error { // a later emitter cannot honor. if functionPlan.RawPlainDemand { return fmt.Errorf( - "coroutine library effect %q has consumer raw-plain demand, which library summary v5 does not lower", + "coroutine library effect %q has consumer raw-plain demand, which library summary v8 does not lower", fact.ID, ) } - // The v5 managed-function record publishes the primary entry only. - // Descriptor construction is an + // The v8 managed-function record publishes the primary and optional static + // outcome entry. Descriptor construction is an // independently versioned ABI and cannot be inferred from FuncRep width. // An undemanded declaration emits nothing and therefore needs no // descriptor in this consumer; reject only an active crossing. if fact.FuncRep == coro.Dispatch && functionPlan.Emission != coro.EmitNone { return fmt.Errorf( - "coroutine library effect %q requires an external Dispatch producer, which library summary v5 does not publish", + "coroutine library effect %q requires an external Dispatch producer, which library summary v8 does not publish", fact.ID, ) } @@ -701,8 +712,15 @@ func (p *context) emitCoroLibraryEffectSummary() error { AtomicCost: functionPlan.AtomicCost, AtomicCostProof: functionPlan.AtomicCostProof, AtomicCostCertificate: functionPlan.AtomicCostCertificate, + StaticOutcome: functionPlan.StaticOutcome, PrimarySymbol: entry.name, RawPlainSymbol: rawPlainSymbol, + OutcomePlainSymbol: func() string { + if functionPlan.HasStaticOutcome() { + return entry.baseName + coroOutcomePlainPrimarySuffix + } + return "" + }(), } if err := universe.CoroLibraryEffects().ValidateFunction(function, metadata, fact); err != nil { return fmt.Errorf("coroutine library summary: preflight %q: %w", functionPlan.ID, err) diff --git a/cl/coro_local_export_binding.go b/cl/coro_local_export_binding.go index 1b2f363984..dd529e703a 100644 --- a/cl/coro_local_export_binding.go +++ b/cl/coro_local_export_binding.go @@ -152,6 +152,7 @@ type coroLocalExportBindingFreezeInput struct { freezeCallableShape func(*ssa.Function) (coroCallableFrozenShape, error) entrySourceSignature func(*ssa.Function) (*types.Signature, error) finalFunctionIdentity func(*ssa.Function) string + cFunctionABITypeKey func(types.Type) string } // freezeCoroLocalExportBindings joins only compiler-visible source facts: @@ -170,7 +171,7 @@ func freezeCoroLocalExportBindings( bindings := make(map[*ssa.Function]coroLocalExportBinding) if input.canonicalAlias == nil || input.functionSortKey == nil || input.freezeCallableShape == nil || input.entrySourceSignature == nil || - input.finalFunctionIdentity == nil { + input.finalFunctionIdentity == nil || input.cFunctionABITypeKey == nil { return nil, fmt.Errorf("prepare emission universe: local export binding freezer has incomplete builder inputs") } @@ -210,7 +211,7 @@ func freezeCoroLocalExportBindings( if err != nil || effective == nil { continue } - abi := structuralCFunctionABITypeKey(effective) + abi := input.cFunctionABITypeKey(effective) if abi == "" { continue } diff --git a/cl/coro_lowered_call.go b/cl/coro_lowered_call.go index 408888fa2b..64f947071b 100644 --- a/cl/coro_lowered_call.go +++ b/cl/coro_lowered_call.go @@ -137,6 +137,12 @@ func (p *context) resolveCoroLoweredRuntimeCall(b llssa.Builder, helper string, } return b.Call(fn.Expr, args...), true } + if p.hasStructuredOutcomePhysicalBody() && targetPlan.HasStaticOutcome() { + nativeResult := !p.prog.LocalGoTypeExceedsNativeStack( + newOutcomePlainPhysicalABI(sourceSig).resultSlotType, + ) + return p.compileCoroStaticOutcomeTargetCall(b, target, args, nativeResult), true + } switch targetPlan.Emission { case coro.EmitPlain: diff --git a/cl/coro_lowering_facts.go b/cl/coro_lowering_facts.go index 4ac087253b..31e8815db6 100644 --- a/cl/coro_lowering_facts.go +++ b/cl/coro_lowering_facts.go @@ -193,6 +193,13 @@ func (u *EmissionUniverse) coroLoweringFunctionSites(plan *coro.SSAPlan, functio if _, unevaluated := ctx.unevaluatedSSA[instruction]; unevaluated { continue } + semantic, err := u.coroProgramIR.semanticInstructionPlan(function, owner, instruction) + if err != nil { + return nil, fmt.Errorf("coroutine lowering facts: function %q block %d semantic plan: %w", function.Name(), block.Index, err) + } + if !semantic.evaluated { + continue + } if _, debug := instruction.(*ssa.DebugRef); debug { continue } diff --git a/cl/coro_managed_heap_test.go b/cl/coro_managed_heap_test.go index 73ba4a4246..ae409b56c7 100644 --- a/cl/coro_managed_heap_test.go +++ b/cl/coro_managed_heap_test.go @@ -35,6 +35,8 @@ import ( const coroManagedHeapFixture = `package foo +import "unsafe" + type Node struct { Value uint32 Next *Node @@ -42,6 +44,13 @@ type Node struct { type Empty struct{} +type Published struct { Value uint32 } + +//llgo:link Compare llgo.atomicCmpXchg +func Compare(address *unsafe.Pointer, old, new unsafe.Pointer) (unsafe.Pointer, bool) { + return old, false +} + var ObservedWritten int64 var ObservedHandled bool @@ -57,6 +66,14 @@ func Root(value uint32) *Node { func Zero() *Empty { return &Empty{} } +func Publish(address *unsafe.Pointer, value uint32) { + candidate := &Published{Value: value} + for { + _, swapped := Compare(address, nil, unsafe.Pointer(candidate)) + if swapped { return } + } +} + func CapturedResults(value uint32) (written int64, err error, handled bool, node *Node) { defer func() { ObservedWritten = written @@ -197,8 +214,26 @@ func TestCoroManagedHeapAllocationNativeAndWasm32(t *testing.T) { defer prog.Dispose() root := ssaPkg.Func("Root") zero := ssaPkg.Func("Zero") + publishFn := ssaPkg.Func("Publish") captured := ssaPkg.Func("CapturedResults") + publishAudit, err := newCoroPhysicalPureSSAAudit(universe, plan, publishFn, "") + if err != nil { + t.Fatal(err) + } + publishAllocs := coroManagedHeapAllocs(publishFn) + if len(publishAllocs) != 1 { + t.Fatalf("Publish heap allocations = %d, want one", len(publishAllocs)) + } + if raw, borrowed := coro.ProveSSABorrowedAllocation(publishAllocs[0]); !borrowed { + t.Fatalf("Publish fixture no longer demonstrates the source-stub lifetime hazard: %+v", raw) + } + publishProof := publishAudit.currentFrameRetentionProof() + if len(publishProof.borrowedAllocations) != 0 || len(publishProof.managedHeapAllocations) != 1 { + t.Fatalf("Publish borrowed/managed allocations = %d/%d, want 0/1", + len(publishProof.borrowedAllocations), len(publishProof.managedHeapAllocations)) + } + audit, err := newCoroPhysicalPureSSAAudit(universe, plan, root, "") if err != nil { t.Fatal(err) @@ -328,6 +363,11 @@ func TestCoroManagedHeapAllocationNativeAndWasm32(t *testing.T) { if !strings.Contains(rootIR, "foo.Child$coro") { t.Fatalf("Root does not suspend through Child after its first allocation:\n%s", rootIR) } + publishPhysical := requireCoroPhysicalFunction(t, module, "foo.Publish") + publishIR := publishPhysical.String() + if strings.Count(publishIR, "runtime.AllocZ") != 1 || strings.Contains(publishIR, "alloca %foo.Published") { + t.Fatalf("Publish intrinsic boundary did not retain managed storage:\n%s", publishIR) + } capturedPhysical := requireCoroPhysicalFunction(t, module, "foo.CapturedResults") capturedIR := capturedPhysical.String() capturedHeapAllocs := coroManagedHeapAllocs(captured) @@ -345,7 +385,7 @@ func TestCoroManagedHeapAllocationNativeAndWasm32(t *testing.T) { var publishBlock llvm.BasicBlock for _, block := range capturedPhysical.BasicBlocks() { for instruction := block.FirstInstruction(); !instruction.IsNil(); instruction = llvm.NextInstruction(instruction) { - if instruction.InstructionOpcode() == llvm.Call && instruction.CalledValue().Name() == coroFramePublishHookV1 { + if instruction.InstructionOpcode() == llvm.Call && instruction.CalledValue().Name() == coroFramePublishHookV3 { publishBlock = instruction.InstructionParent() } } @@ -370,7 +410,7 @@ func TestCoroManagedHeapAllocationNativeAndWasm32(t *testing.T) { t.Fatalf("CapturedResults hoisted/ordinary AllocZ calls = %d/%d, want 3/%d:\n%s", hoistedHeapCalls, ordinaryHeapCalls, len(capturedHeapAllocs)-3, capturedIR) } - publish := strings.Index(capturedIR, "call void @"+coroFramePublishHookV1) + publish := strings.Index(capturedIR, "call void @"+coroFramePublishHookV3) alloc := strings.Index(capturedIR, "runtime.AllocZ") initialSuspend := strings.Index(capturedIR, "%coro.suspend = call i8 @llvm.coro.suspend") if publish < 0 || alloc < 0 || initialSuspend < 0 || publish >= alloc || alloc >= initialSuspend { @@ -402,6 +442,11 @@ func TestCoroManagedHeapAllocationNativeAndWasm32(t *testing.T) { t.Fatalf("CoroSplit moved ordinary Root AllocZ calls out of resume (resume AllocZ=%d):\nramp:\n%s\nresume:\n%s", got, rampIR, resumeIR) } + publishResume := module.NamedFunction("foo.Publish$coro.resume") + if publishResume.IsNil() || strings.Count(publishResume.String(), "runtime.AllocZ") != 1 || + strings.Contains(publishResume.String(), "alloca %foo.Published") { + t.Fatalf("CoroSplit lost Publish managed storage:\n%s", module.String()) + } capturedRamp := module.NamedFunction("foo.CapturedResults$coro") capturedResume := module.NamedFunction("foo.CapturedResults$coro.resume") if capturedRamp.IsNil() || capturedResume.IsNil() || @@ -490,7 +535,7 @@ func AllocU(size uintptr) unsafe.Pointer { functionIDs.CoroABI = coro.PhysicalABIV1 functionIDs.SchedulerABI = coro.SchedulerProgramBootstrapChannelClosedStaticSpawnABIV0 functionIDs.ArchiveReady = true - root, zero, child, captured := ssaPkg.Func("Root"), ssaPkg.Func("Zero"), ssaPkg.Func("Child"), ssaPkg.Func("CapturedResults") + root, zero, publish, child, captured := ssaPkg.Func("Root"), ssaPkg.Func("Zero"), ssaPkg.Func("Publish"), ssaPkg.Func("Child"), ssaPkg.Func("CapturedResults") var capturedCleanup *ssa.Function for _, block := range captured.Blocks { for _, instruction := range block.Instrs { @@ -513,6 +558,7 @@ func AllocU(size uintptr) unsafe.Pointer { plan, err := coro.AnalyzeSSA(ssaPkg.Prog, coro.Roots{ {Function: root, Demand: coro.AsyncDemand}, {Function: zero, Demand: coro.AsyncDemand}, + {Function: publish, Demand: coro.AsyncDemand}, {Function: captured, Demand: coro.AsyncDemand}, }, coro.SSAConfig{ EmissionUniverse: ssaUniverse, @@ -525,6 +571,10 @@ func AllocU(size uintptr) unsafe.Pointer { } return coro.SSAFunctionPolicy{}, nil }, + ClassifyElidedCall: func(_ *ssa.Function, call ssa.CallInstruction) (bool, error) { + semantics, intrinsic, err := universe.CoroIntrinsicCallSiteSemantics(call) + return intrinsic && semantics.ElidesManagedCall(), err + }, }) if err != nil { prog.Dispose() diff --git a/cl/coro_managed_interface.go b/cl/coro_managed_interface.go index b4234edd06..9cc6d98abd 100644 --- a/cl/coro_managed_interface.go +++ b/cl/coro_managed_interface.go @@ -182,7 +182,11 @@ func (p *coroManagedInterfaceDispatchPlan) acceptsTarget(fn *ssa.Function, plan return ok && target == fn } -func coroManagedInterfaceMethodKey(method *types.Func, signature *types.Signature) string { +func coroManagedInterfaceMethodKey( + method *types.Func, + signature *types.Signature, + typeKeys ...func(types.Type) string, +) string { if method == nil || signature == nil { return "" } @@ -190,6 +194,9 @@ func coroManagedInterfaceMethodKey(method *types.Func, signature *types.Signatur if callable == nil { return "" } + if len(typeKeys) != 0 && typeKeys[0] != nil { + return method.Id() + "\x00" + typeKeys[0](callable) + } return method.Id() + "\x00" + structuralEmissionABITypeKey(callable) } @@ -207,7 +214,11 @@ func coroManagedInterfaceInvokeMethodKey( if err != nil { return "", err } - key := coroManagedInterfaceMethodKey(common.Method, signature) + var typeKeys *emissionTypeKeyCache + if universe != nil { + typeKeys = &universe.emissionTypeKeys + } + key := coroManagedInterfaceMethodKey(common.Method, signature, typeKeys.strictABI) if key == "" { return "", fmt.Errorf("managed interface descriptor has no exact method/signature key") } @@ -667,7 +678,7 @@ func (p *context) resolveCoroRawMethodSymbol( } return p.mustRawPlainFunctionSymbol(target).name, true } - key := sha256.Sum256([]byte(string(entry.plan.ID) + "\x00" + structuralEmissionABITypeKey(patched))) + key := sha256.Sum256([]byte(string(entry.plan.ID) + "\x00" + p.cachedStrictEmissionABITypeKey(patched))) name := coroManagedInterfaceRawTrapPrefix + hex.EncodeToString(key[:16]) // ABI method tables for one concrete type may be materialized in several // package archives. The content-addressed trap is therefore a coalescible @@ -852,7 +863,7 @@ func validateCoroManagedInterfaceDescriptorTarget( targetLogical := coroInterfaceDispatchCanonicalSignature(types.NewSignatureType( nil, nil, nil, types.NewTuple(params...), effective.Results(), effective.Variadic(), )) - if !coroInterfaceDispatchSignaturesIdentical(logicalSignature, targetLogical) { + if !coroInterfaceDispatchSignaturesIdentical(logicalSignature, targetLogical, universe.emissionTypeKeys.strictABI) { return fail("logical signature %s does not match effective target signature %s", logicalSignature, targetLogical) } return nil diff --git a/cl/coro_outcome_plain.go b/cl/coro_outcome_plain.go index 6fce258ac5..b96c988767 100644 --- a/cl/coro_outcome_plain.go +++ b/cl/coro_outcome_plain.go @@ -129,17 +129,26 @@ func validateOutcomePlainFrozenPlan( if plan == nil || plan.function == nil { return fmt.Errorf("outcome-plain emission requires one frozen physical plan") } - if logical.Emission != coro.EmitOutcomePlain || - logical.ManagedEntry != coro.ManagedEntryOutcomePlain || - !logical.AtomicCostProof.ProvesOutcomePlain() { + primary := logical.Emission == coro.EmitOutcomePlain && + logical.ManagedEntry == coro.ManagedEntryOutcomePlain + twin := logical.Emission == coro.EmitCoroutine && + logical.ManagedEntry == coro.ManagedEntryCoroutine + if (!primary && !twin) || !logical.HasStaticOutcome() { return fmt.Errorf("outcome-plain physical plan disagrees with its logical capability") } if plan.atomicCost != logical.AtomicCost || plan.atomicCostProof != logical.AtomicCostProof || plan.atomicCertificate != logical.AtomicCostCertificate { return fmt.Errorf("outcome-plain physical plan lost its exact atomic-cost certificate") } - if plan.cleanup != nil || plan.critical != nil || plan.needsPreempt || plan.preempt != nil { - return fmt.Errorf("outcome-plain body acquired cleanup, critical, or preemption state") + if plan.staticOutcome != logical.StaticOutcome { + return fmt.Errorf("outcome-plain physical plan lost its exact unbounded-static capability") + } + if plan.cleanup != nil || plan.critical != nil || + !logical.StaticOutcome && (plan.needsPreempt || plan.preempt != nil) { + return fmt.Errorf("outcome-plain body acquired cleanup, critical, or incompatible preemption state") + } + if logical.StaticOutcome { + return validateStaticOutcomeFrozenPlan(plan, logical) } directCalls := 0 for instruction, physical := range plan.instructions { @@ -198,6 +207,56 @@ func validateOutcomePlainFrozenPlan( return nil } +func validateStaticOutcomeFrozenPlan(plan *coroPhysicalFunctionPlan, logical coro.FunctionPlan) error { + for instruction, physical := range plan.instructions { + if !physical.semantic.evaluated { + continue + } + operationSupported := physical.operation == coroPhysicalOperationNone || + physical.operation == coroPhysicalOperationControl && + !physical.operationControl.NativeActivationBound() + if !operationSupported || physical.controlFailureHard || + physical.operationFailure != "" || physical.outcomeFailure != "" { + return fmt.Errorf( + "static-outcome function %q instruction %T %q selected an invalid physical recipe (operation=%s control=%s hard=%t operation-failure=%q outcome-failure=%q)", + plan.function.String(), instruction, instruction.String(), physical.operation, physical.control, + physical.controlFailureHard, physical.operationFailure, physical.outcomeFailure, + ) + } + if _, runDefers := instruction.(*ssa.RunDefers); runDefers { + // ProgramIR proved that no evaluated Defer registration exists. The + // synthetic RunDefers is therefore an explicit no-op in this twin. + continue + } + if _, deferInstruction := instruction.(*ssa.Defer); deferInstruction { + return fmt.Errorf("static-outcome body contains an evaluated defer registration") + } + if call, ok := instruction.(*ssa.Call); ok && + physical.semantic.recipe == coro.RecipeID("cl.ssa.call.v1") { + if _, builtin := call.Common().Value.(*ssa.Builtin); builtin { + continue + } + if physical.control == coroPhysicalControlNone && !physical.controlFailureHard { + // Full physical ABI preflight independently proved this exact + // ordinary call to be a closed no-unwind plain target. + continue + } + if physical.control != coroPhysicalControlDirectOutcome || physical.controlTarget == nil || + physical.controlTargetID == "" || !physical.directOutcomeNativeResult { + return fmt.Errorf("static-outcome call %q lacks an exact synchronous target", call.String()) + } + } else if physical.control != coroPhysicalControlNone { + return fmt.Errorf("static-outcome instruction %T selected unsupported control %s", instruction, physical.control) + } + switch physical.outcome { + case coroPhysicalOutcomeNone, coroPhysicalOutcomeReturn, coroPhysicalOutcomePanic: + default: + return fmt.Errorf("static-outcome instruction %T selected unsupported outcome %s", instruction, physical.outcome) + } + } + return nil +} + func (p *context) compileOutcomePlainPhysicalBody( b llssa.Builder, fn *ssa.Function, @@ -246,10 +305,12 @@ func (p *context) compileOutcomePlainPhysicalBody( phi() } emission.completeManagedPhysicalBody(bodyCapability) - if err := p.pkg.EmitCoroAtomicCostCertificate( - p.fn.Name(), logical.AtomicCost, uint8(logical.AtomicCostProof), logical.AtomicCostCertificate, - ); err != nil { - panic(fmt.Errorf("publish outcome-plain atomic-cost certificate: %w", err)) + if logical.AtomicCostProof.ProvesOutcomePlain() { + if err := p.pkg.EmitCoroAtomicCostCertificate( + p.fn.Name(), logical.AtomicCost, uint8(logical.AtomicCostProof), logical.AtomicCostCertificate, + ); err != nil { + panic(fmt.Errorf("publish outcome-plain atomic-cost certificate: %w", err)) + } } } @@ -304,17 +365,58 @@ func (p *context) compileCoroStaticOutcomeCall( args := p.compileValues(b, call.Call.Args, p.funcKind(call.Call.Value)) source, _ := call.Call.Value.(*ssa.Function) args = p.compileManagedGoLinknameCallArguments(b, source, callee, args) - entry := p.mustFunctionSymbol(callee) + result := p.compileCoroStaticOutcomeTargetCallResult( + b, callee, args, instructionPlan.directOutcomeNativeResult, + ) + value, retagged := p.compileManagedGoLinknameCallResult(b, source, callee, result.value) + if !retagged { + if !result.address.IsNil() { + p.recordCoroValueAddress(call, result.address) + } + } + return value +} + +// compileCoroStaticOutcomeTargetCall is the single synchronous outcome-call +// transaction shared by exact source calls and compiler-inserted runtime +// helpers. The target and arguments have already crossed their respective +// source/marker ABI boundary; this layer owns only the hidden outcome ABI and +// terminal propagation. +func (p *context) compileCoroStaticOutcomeTargetCall( + b llssa.Builder, + callee *ssa.Function, + args []llssa.Expr, + nativeResult bool, +) llssa.Expr { + return p.compileCoroStaticOutcomeTargetCallResult(b, callee, args, nativeResult).value +} + +func (p *context) compileCoroStaticOutcomeTargetCallResult( + b llssa.Builder, + callee *ssa.Function, + args []llssa.Expr, + nativeResult bool, +) coroAwaitedValue { + if !p.hasStructuredOutcomePhysicalBody() || callee == nil || len(callee.FreeVars) != 0 { + panic("static outcome target call escaped its context-free structured body") + } + entry := p.mustOutcomePlainFunctionSymbol(callee) sourceSig, err := p.emissionUniverse.coroPhysicalSourceSignature(callee) if err != nil { panic(fmt.Errorf("derive outcome-plain target %q ABI: %w", entry.plan.ID, err)) } abi := newOutcomePlainPhysicalABI(sourceSig) - calleeFn, _, kind := p.compileFunctionEntry(entry) + calleeFn, _, kind := p.compileOutcomePlainFunction(callee) if kind != goFunc { panic(fmt.Sprintf("outcome-plain target %q did not resolve to a Go entry", entry.plan.ID)) } - if p.hasOutcomePlainPhysicalBody() && entry.plan.External == coro.ExternalKnown { + // Complete-program analysis keeps a producer Defined even while a different + // package module sees only its declaration. Certificate consumption is a + // physical owner fact, not the logical whole-program External dimension. + // compileOutcomePlainFunction has already materialized every locally owned + // body, so HasBody is the exact module-boundary test here. + if p.hasOutcomePlainPhysicalBody() && !calleeFn.HasBody() && + entry.plan.AtomicCostProof.ProvesOutcomePlain() { if err := p.pkg.EmitCoroAtomicCostDependency( calleeFn.Name(), entry.plan.AtomicCost, uint8(entry.plan.AtomicCostProof), entry.plan.AtomicCostCertificate, ); err != nil { @@ -330,14 +432,16 @@ func (p *context) compileCoroStaticOutcomeCall( if p.hasCoroPhysicalBody() { resultSlot = p.coroResultSlot(resultType) } else { - if !p.hasOutcomePlainPhysicalBody() || !instructionPlan.directOutcomeNativeResult { + if !p.hasOutcomePlainPhysicalBody() || !nativeResult { panic("outcome-plain DAG call result escaped its frozen native-stack bound") } - resultSlot = b.AllocaT(resultType) + resultSlot = p.structuredOutcomeAlloca(resultType, false) } // The completion record is three fixed words and is consumed before the next - // suspension, so this site-local alloca is independently bounded. - completion := b.AllocaZeroedT(outcomePlainCompletionType(p.prog)) + // suspension. Keep it in the physical function entry: loop iterations reuse + // the record after a fully synchronous call, avoiding dynamic stack growth and + // exposing its fields to ordinary SROA. + completion := p.structuredOutcomeAlloca(outcomePlainCompletionType(p.prog), true) physicalArgs := make([]llssa.Expr, 0, len(args)+3) physicalArgs = append(physicalArgs, p.managedPhysicalTask(), @@ -346,7 +450,20 @@ func (p *context) compileCoroStaticOutcomeCall( ) physicalArgs = append(physicalArgs, args...) b.Call(calleeFn.Expr, physicalArgs...) + p.dispatchOutcomePlainCompletion(b, completion) + return coroAwaitedValue{ + value: p.loadCoroAwaitResult(b, resultSlot, sourceSig.Results()), + address: p.coroAwaitResultAddress(b, resultSlot, sourceSig.Results()), + } +} +// dispatchOutcomePlainCompletion consumes one immediate synchronous child +// transaction and leaves b in the child's Return continuation. Every other +// terminal state is propagated into the current structured parent. +func (p *context) dispatchOutcomePlainCompletion(b llssa.Builder, completion llssa.Expr) { + if !p.hasStructuredOutcomePhysicalBody() || b == nil || b.Func != p.fn || completion.IsNil() { + panic("outcome-plain completion dispatch escaped its structured parent") + } status := b.Load(b.FieldAddr(completion, outcomePlainCompletionStatus)) returned := p.fn.MakeBlock() panicked := p.fn.MakeBlock() @@ -372,10 +489,4 @@ func (p *context) compileCoroStaticOutcomeCall( b.SetBlockEx(invalid, llssa.AtEnd, false) b.Unreachable() b.SetBlockContinuation(returned) - value := p.loadCoroAwaitResult(b, resultSlot, sourceSig.Results()) - value, retagged := p.compileManagedGoLinknameCallResult(b, source, callee, value) - if !retagged { - p.recordCoroValueAddress(call, p.coroAwaitResultAddress(b, resultSlot, sourceSig.Results())) - } - return value } diff --git a/cl/coro_outcome_plain_test.go b/cl/coro_outcome_plain_test.go index 51aa10bb82..9edbd701a5 100644 --- a/cl/coro_outcome_plain_test.go +++ b/cl/coro_outcome_plain_test.go @@ -31,6 +31,7 @@ import ( llssa "github.com/goplus/llgo/ssa" "github.com/xgo-dev/llvm" "golang.org/x/tools/go/ssa" + "golang.org/x/tools/go/ssa/ssautil" ) const coroOutcomePlainFixture = `package foo @@ -118,6 +119,100 @@ func Parent(value, divisor, shift int) int { } ` +const coroOutcomePlainAtomicIntrinsicFixture = `package foo + +type Word uint32 +type Cell struct { value Word } + +//llgo:link atomicLoad llgo.atomicLoad +func atomicLoad(ptr *Word) Word { return *ptr } + +func (cell *Cell) Load() Word { + return atomicLoad(&cell.value) +} + +func Root(cell *Cell) Word { + return cell.Load() +} +` + +const coroOutcomePlainGoLinknameAtomicIntrinsicFixture = `package foo + +import _ "unsafe" + +type Word uint64 +type Cell struct { value Word } + +//go:linkname atomicAdd llgo.atomicAddReturnNew +func atomicAdd(ptr *Word, delta Word) Word + +func (cell *Cell) Add(delta Word) Word { + return atomicAdd(&cell.value, delta) +} + +func Root(cell *Cell, delta Word) Word { + return cell.Add(delta) +} +` + +const coroOutcomePlainStaticTwinFixture = `package foo + +func Leaf(value uint32, payload any, fail bool) uint32 { + if fail { panic(payload) } + return value + 1 +} + +func Static(value uint32, payload any, fail bool) uint32 { + return Leaf(value, payload, fail) +} + + +func Publish() func(uint32, any, bool) uint32 { + return Leaf +} + +func Root(value uint32, payload any, fail bool) uint32 { + _ = Publish() + return Static(value, payload, fail) +} +` + +func TestCoroOutcomePlainStaticTwinKeepsDynamicCoroutineEntry(t *testing.T) { + prog, pkg, plan, ssaPkg := compileCoroOutcomePlainSource( + t, nil, coroOutcomePlainStaticTwinFixture, "Root", 64, + ) + defer prog.Dispose() + module := pkg.Module() + defer module.Dispose() + + leaf := ssaPkg.Func("Leaf") + leafPlan, found := plan.FunctionPlan(leaf) + if !found || leafPlan.Emission != coro.EmitCoroutine || + leafPlan.ManagedEntry != coro.ManagedEntryCoroutine || leafPlan.FuncRep != coro.Dispatch || + !leafPlan.AtomicCostProof.ProvesOutcomePlain() || leafPlan.AtomicCost == 0 { + t.Fatalf("static-twin Leaf plan = %+v, present=%t", leafPlan, found) + } + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify static outcome twin module: %v\n%s", err, module.String()) + } + text := module.String() + base := "foo.Leaf" + if module.NamedFunction(base+coroPrimarySuffix).IsNil() || + module.NamedFunction(base+coroOutcomePlainPrimarySuffix).IsNil() { + t.Fatalf("Leaf did not emit both coroutine and outcome entries:\n%s", text) + } + staticBody := module.NamedFunction("foo.Static" + coroOutcomePlainPrimarySuffix).String() + if !strings.Contains(staticBody, base+coroOutcomePlainPrimarySuffix) || + strings.Contains(staticBody, base+coroPrimarySuffix) { + t.Fatalf("static caller did not select only the outcome twin:\n%s", staticBody) + } + if !strings.Contains(text, coroCoroDispatchThunkPrefix) || + !strings.Contains(text, base+coroPrimarySuffix) { + t.Fatalf("dynamic descriptor did not retain the coroutine primary:\n%s", text) + } + runCoroABITestPipeline(t, prog, module) +} + func TestCoroOutcomePlainLeafNativeAndWasm32(t *testing.T) { llssa.Initialize(llssa.InitAll) for _, test := range []struct { @@ -244,6 +339,10 @@ func Root(header *producer.Header, code producer.Code) producer.Code { OutcomeMode: coro.OutcomeExplicitStatus, ClassifyLocalBody: universe.CoroLocalBodyFacts, ClassifyLoweredCalls: universe.CoroLoweredCalls, + ClassifyElidedCall: func(_ *ssa.Function, call ssa.CallInstruction) (bool, error) { + callPlan, found, err := universe.CoroCallSitePlan(call) + return found && callPlan.ElidesCall(), err + }, }) if err != nil { t.Fatal(err) @@ -359,6 +458,82 @@ func TestCoroOutcomePlainUnprovenFaultRecipesFailClosed(t *testing.T) { } } +func TestCoroOutcomePlainAdmitsExactInlineAtomicIntrinsic(t *testing.T) { + prog, pkg, plan, ssaPkg := compileCoroOutcomePlainSource( + t, nil, coroOutcomePlainAtomicIntrinsicFixture, "Root", 64, + ) + defer prog.Dispose() + module := pkg.Module() + defer module.Dispose() + + var load *ssa.Function + for _, member := range ssaPkg.Members { + function, ok := member.(*ssa.Function) + if ok && function.Name() == "Load" && function.Signature.Recv() != nil { + load = function + break + } + } + if load == nil { + for function := range ssautil.AllFunctions(ssaPkg.Prog) { + if function != nil && function.Pkg == ssaPkg && function.Name() == "Load" && + function.Signature != nil && function.Signature.Recv() != nil { + load = function + break + } + } + } + if load == nil { + t.Fatal("atomic fixture method Load is absent") + } + loadPlan, found := plan.FunctionPlan(load) + if !found || loadPlan.Emission != coro.EmitOutcomePlain || + loadPlan.AtomicCostProof != coro.AtomicCostLeaf || loadPlan.AtomicCost == 0 { + t.Fatalf("atomic Load plan = %+v, present=%t; want outcome-plain leaf", loadPlan, found) + } + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify outcome-plain atomic module before CoroSplit: %v\n%s", err, module.String()) + } + text := module.String() + if !strings.Contains(text, "load atomic") { + t.Fatalf("outcome-plain atomic wrapper lost its inline atomic load:\n%s", text) + } + runCoroABITestPipeline(t, prog, module) +} + +func TestCoroOutcomePlainAdmitsBodylessGoLinknameAtomicIntrinsic(t *testing.T) { + prog, pkg, plan, ssaPkg := compileCoroOutcomePlainSource( + t, nil, coroOutcomePlainGoLinknameAtomicIntrinsicFixture, "Root", 64, + ) + defer prog.Dispose() + module := pkg.Module() + defer module.Dispose() + + var add *ssa.Function + for function := range ssautil.AllFunctions(ssaPkg.Prog) { + if function != nil && function.Pkg == ssaPkg && function.Name() == "Add" && + function.Signature != nil && function.Signature.Recv() != nil { + add = function + break + } + } + if add == nil { + t.Fatal("go:linkname atomic fixture method Add is absent") + } + addPlan, found := plan.FunctionPlan(add) + if !found || addPlan.Emission != coro.EmitOutcomePlain || + addPlan.AtomicCostProof != coro.AtomicCostLeaf || addPlan.AtomicCost == 0 { + t.Fatalf("go:linkname atomic Add plan = %+v, present=%t; want outcome-plain leaf", addPlan, found) + } + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify go:linkname outcome-plain atomic module before CoroSplit: %v\n%s", err, module.String()) + } + if text := module.String(); !strings.Contains(text, "atomicrmw add") { + t.Fatalf("go:linkname outcome-plain atomic wrapper lost its inline atomic add:\n%s", text) + } + runCoroABITestPipeline(t, prog, module) +} + func TestCoroOutcomePlainDAGNativeAndWasm32(t *testing.T) { llssa.Initialize(llssa.InitAll) for _, test := range []struct { @@ -487,8 +662,9 @@ func TestCoroOutcomePlainPhysicalCostAgainstCoroutineBaseline(t *testing.T) { baselineLeafPlan, baselineFound := baselinePlan.FunctionPlan(baselineLeaf) optimizedLeafPlan, optimizedFound := optimizedPlan.FunctionPlan(optimizedLeaf) - if !baselineFound || baselineLeafPlan.Emission != coro.EmitCoroutine { - t.Fatalf("baseline Leaf plan = %+v, present=%t; want coroutine", baselineLeafPlan, baselineFound) + if !baselineFound || baselineLeafPlan.Emission != coro.EmitCoroutine || + !baselineLeafPlan.HasStaticOutcome() { + t.Fatalf("baseline Leaf plan = %+v, present=%t; want coroutine primary plus static outcome twin", baselineLeafPlan, baselineFound) } if !optimizedFound || optimizedLeafPlan.Emission != coro.EmitOutcomePlain { t.Fatalf("optimized Leaf plan = %+v, present=%t; want outcome-plain", optimizedLeafPlan, optimizedFound) @@ -524,9 +700,10 @@ func TestCoroOutcomePlainPhysicalCostAgainstCoroutineBaseline(t *testing.T) { strings.Count(baselineParent, coroAwaitConsumeHookV1) optimizedAwaitCalls := strings.Count(optimizedParent, coroAwaitPrepareHookV1) + strings.Count(optimizedParent, coroAwaitConsumeHookV1) - if baselineAwaitCalls == 0 || optimizedAwaitCalls != 0 || + if baselineAwaitCalls != 0 || optimizedAwaitCalls != 0 || + !strings.Contains(baselineParent, "foo.Leaf$outcome") || !strings.Contains(optimizedParent, "foo.Leaf$outcome") { - t.Fatalf("Leaf scheduling boundary baseline calls=%d optimized calls=%d", baselineAwaitCalls, optimizedAwaitCalls) + t.Fatalf("static Leaf call retained a scheduling boundary: baseline calls=%d optimized calls=%d", baselineAwaitCalls, optimizedAwaitCalls) } optimizeCoroOutcomeCostModule(t, baselineProg, baselineModule) @@ -552,8 +729,8 @@ func TestCoroOutcomePlainPhysicalCostAgainstCoroutineBaseline(t *testing.T) { ) } t.Logf( - "post-split fixture: IR baseline=%d optimized=%d; O2 object baseline=%d optimized=%d; eliminated Leaf frame=1 resume=1 destroy=1 await-hook-refs=%d", - len(baselineIR), len(optimizedIR), baselineBytes, optimizedBytes, baselineAwaitCalls, + "post-split fixture: IR baseline=%d optimized=%d; O2 object baseline=%d optimized=%d; static call is flat in both, optimized primary eliminates Leaf frame/resume/destroy=1/1/1", + len(baselineIR), len(optimizedIR), baselineBytes, optimizedBytes, ) }) } @@ -735,6 +912,7 @@ func Caller(value uint32, payload any, fail bool) uint32 { AtomicCostProof: coro.AtomicCostLeaf, AtomicCostCertificate: strings.Repeat("a", 64), PrimarySymbol: baseSymbol + coroOutcomePlainPrimarySuffix, + OutcomePlainSymbol: baseSymbol + coroOutcomePlainPrimarySuffix, } plan, err := coro.AnalyzeSSA( ssaPkg.Prog, @@ -912,6 +1090,10 @@ func compileCoroOutcomePlainSource( OutcomeMode: coro.OutcomeExplicitStatus, ClassifyLocalBody: universe.CoroLocalBodyFacts, ClassifyLoweredCalls: universe.CoroLoweredCalls, + ClassifyElidedCall: func(_ *ssa.Function, call ssa.CallInstruction) (bool, error) { + callPlan, found, err := universe.CoroCallSitePlan(call) + return found && callPlan.ElidesCall(), err + }, }) if err != nil { prog.Dispose() diff --git a/cl/coro_owner_emission.go b/cl/coro_owner_emission.go index ecf356b12b..71f6fb39ec 100644 --- a/cl/coro_owner_emission.go +++ b/cl/coro_owner_emission.go @@ -132,6 +132,17 @@ func (p *context) emitCoroFrozenOwnerBodies(pkg llssa.Package) error { p.coroOwnerBodySymbols[function.Name()] = none{} functionPlan, _ := plan.FunctionPlan(body.function) + if functionPlan.Emission == coro.EmitCoroutine && + functionPlan.HasStaticOutcome() { + outcome := p.outcomePlainFuncs[body.function] + if outcome == nil || !outcome.HasBody() { + return fmt.Errorf( + "coroutine owner emission: frozen Go body %q did not materialize its outcome/plain twin", + body.function.String(), + ) + } + p.coroOwnerBodySymbols[outcome.Name()] = none{} + } if functionPlan.Emission != coro.EmitCoroutine || !plan.HasRawPlainVariant(body.function) { continue diff --git a/cl/coro_panic_test.go b/cl/coro_panic_test.go index f0013a8da7..7c704122ae 100644 --- a/cl/coro_panic_test.go +++ b/cl/coro_panic_test.go @@ -252,7 +252,7 @@ func compileCoroExplicitStatusPanicFixture(t *testing.T, target *llssa.Target) ( return prog, pkg, plan, root } -func TestCoroExplicitStatusPanicPreflightRemainsFailClosed(t *testing.T) { +func TestCoroExplicitStatusPanicPreflightCapabilities(t *testing.T) { for _, test := range []struct { name string source string @@ -268,9 +268,15 @@ func Root(address uintptr, trigger bool) { if trigger { panic(*(*any)(unsafe.Poi want: "uintptr-to-pointer conversion has no traceable exact pointer provenance", }, { - name: "boxed scalar", + name: "constant backed scalar", source: `package foo func Root(trigger bool) { if trigger { panic(uint32(7)) } } +`, + }, + { + name: "dynamic boxed scalar", + source: `package foo +func Root(value uint32, trigger bool) { if trigger { panic(value) } } `, want: "structured runtime helper validation requires a frozen emission universe", }, @@ -320,6 +326,12 @@ func Root(trigger bool) { defer cleanup(); if trigger { panic(&Payload) } } Exec: coro.MayUnwind | test.exec, } err = validateCoroPhysicalABIWithUniverseCapabilities(root, plan, nil, universe, true, false, false, true) + if test.want == "" { + if err != nil { + t.Fatalf("preflight error = %v, want constant-backed payload acceptance", err) + } + return + } if err == nil || !strings.Contains(err.Error(), test.want) { t.Fatalf("preflight error = %v, want %q", err, test.want) } @@ -476,6 +488,7 @@ func Selected(first, second <-chan any) { EmissionUniverse: ssaUniverse, FunctionIDs: functionIDs, MaxPlainInstructions: -1, + ClassifyLocalBody: universe.CoroLocalBodyFacts, ClassifyLoweredCalls: universe.CoroLoweredCalls, }) if err != nil { diff --git a/cl/coro_park_emitter.go b/cl/coro_park_emitter.go index 4a8fec174b..4d03064ae6 100644 --- a/cl/coro_park_emitter.go +++ b/cl/coro_park_emitter.go @@ -52,13 +52,19 @@ type coroParkFaultRoute struct { // WaitSet transaction inside these hooks and is never represented as several // independent parks. type coroParkOperation struct { - shouldSuspend llssa.Expr - park func(llssa.Builder) - resume func(llssa.Builder) llssa.Expr - normal []uint64 - faults []coroParkFaultRoute - abort uint64 - shutdown uint64 + // prepare runs in the active block after the emitter has allocated the + // static state identity. It returns the runtime suspend predicate. Most + // operations only return an already-computed predicate and bind park below; + // a fused try-or-park hook may instead publish its complete state here and + // leave park nil. In both cases, state allocation and the stack-cut protocol + // remain compiler-owned rather than leaking into a feature lowerer. + prepare func(llssa.Builder, uint32, uint32) llssa.Expr + park func(llssa.Builder) + resume func(llssa.Builder) llssa.Expr + normal []uint64 + faults []coroParkFaultRoute + abort uint64 + shutdown uint64 } // suspendCoroCurrentBlockIf is the single compiler-owned conditional @@ -121,7 +127,7 @@ func validateCoroParkOperationStatuses( func (c *coroBodyContext) emitCoroParkOperation(p *context, b llssa.Builder, operation coroParkOperation) { if c == nil || p == nil || b == nil || b.Func != p.fn || c.coro == nil || c.unsupportedRunDecision == nil || - operation.shouldSuspend.IsNil() || operation.park == nil || operation.resume == nil { + operation.prepare == nil || operation.resume == nil { panic("coroutine park operation requires a complete physical emitter and protocol") } if err := validateCoroParkOperationStatuses( @@ -132,7 +138,12 @@ func (c *coroBodyContext) emitCoroParkOperation(p *context, b llssa.Builder, ope ); err != nil { panic(err) } - if operation.shouldSuspend.Type != b.Prog.Bool() { + stateID := c.nextState + c.nextState++ + c.instructions = 0 + sourceLine := p.coroCurrentSourceLine() + shouldSuspend := operation.prepare(b, stateID, sourceLine) + if shouldSuspend.IsNil() || shouldSuspend.Type != b.Prog.Bool() { panic("coroutine park suspend predicate must be bool") } faultTargets := make([]llssa.BasicBlock, len(operation.faults)) @@ -140,17 +151,17 @@ func (c *coroBodyContext) emitCoroParkOperation(p *context, b llssa.Builder, ope faultTargets[index] = b.Func.MakeBlock() } join := c.suspendCoroCurrentBlockIf( - operation.shouldSuspend, + shouldSuspend, func(suspend llssa.Builder) { - stateID := c.nextState - c.nextState++ - c.instructions = 0 + if operation.park == nil { + return + } c.publishState( suspend, coroSuspendPark, coroLifecycleSuspended, stateID, - p.coroCurrentSourceLine(), + sourceLine, ) operation.park(suspend) }, diff --git a/cl/coro_patch_init.go b/cl/coro_patch_init.go index d16f8e7d98..9897cabf62 100644 --- a/cl/coro_patch_init.go +++ b/cl/coro_patch_init.go @@ -95,8 +95,17 @@ func (p *context) tryCompileCoroPatchInitRedirect(b llssa.Builder, call *ssa.Cal } b.Call(fn.Expr) case coro.EmitCoroutine: + if p.hasOutcomePlainPhysicalBody() { + if !targetPlan.HasStaticOutcome() { + panic("outcome-plain patch initializer replacement target has no synchronous twin") + } + if result := p.compileCoroStaticOutcomeTargetCall(b, target, nil, true); !result.IsNil() { + panic("outcome-plain patch initializer replacement returned a value") + } + break + } if p.coroBody() == nil { - panic("coroutine patch initializer replacement escaped into a plain owner") + panic("coroutine patch initializer replacement escaped into a structured owner") } if result := p.compileCoroTargetAwait(b, target, nil); !result.IsNil() { panic("coroutine patch initializer replacement returned a value") diff --git a/cl/coro_physical_plan.go b/cl/coro_physical_plan.go index 55e3187ca6..b68e255c66 100644 --- a/cl/coro_physical_plan.go +++ b/cl/coro_physical_plan.go @@ -54,6 +54,7 @@ const ( coroPhysicalInstructionIntegerDivideByZeroGuard coroPhysicalInstructionTerminalResultAllocation coroPhysicalInstructionFrameAllocation + coroPhysicalInstructionBorrowedAllocation coroPhysicalInstructionFrameBitcastAllocation coroPhysicalInstructionFrameAllocaBytes coroPhysicalInstructionHeapCStr @@ -97,6 +98,8 @@ func (recipe coroPhysicalInstructionRecipe) String() string { return "terminal-result-allocation" case coroPhysicalInstructionFrameAllocation: return "frame-allocation" + case coroPhysicalInstructionBorrowedAllocation: + return "borrowed-allocation" case coroPhysicalInstructionFrameBitcastAllocation: return "frame-bitcast-allocation" case coroPhysicalInstructionFrameAllocaBytes: @@ -400,7 +403,7 @@ func (plan coroPhysicalInstructionPlan) elidesRuntimeHelper(helper string) bool if helper == "AssertDivideByZero" { return true } - case coroPhysicalInstructionFrameAllocation: + case coroPhysicalInstructionFrameAllocation, coroPhysicalInstructionBorrowedAllocation: if helper == "AllocZ" { return true } @@ -431,6 +434,8 @@ type coroPhysicalFunctionPlan struct { atomicCost uint64 atomicCostProof coro.AtomicCostProof atomicCertificate string + staticOutcome bool + reachableBlocks map[*ssa.BasicBlock]bool instructions map[ssa.Instruction]coroPhysicalInstructionPlan } @@ -453,8 +458,12 @@ func prepareCoroPhysicalFunctionPlan( critical: critical, cleanup: cleanup, frameRetentionABI: audit.frameRetentionABI, + reachableBlocks: make(map[*ssa.BasicBlock]bool, len(audit.reachableBlocks)), instructions: make(map[ssa.Instruction]coroPhysicalInstructionPlan), } + for block, reachable := range audit.reachableBlocks { + plan.reachableBlocks[block] = reachable + } var logical coro.FunctionPlan if whole != nil { function, planned := whole.FunctionPlan(audit.fn) @@ -463,6 +472,7 @@ func prepareCoroPhysicalFunctionPlan( } logical = function plan.needsPreempt = function.Exec.Contains(coro.NeedsPreempt) + plan.staticOutcome = function.StaticOutcome } preempt, err := planCoroPhysicalPreemption( audit, critical, plan.needsPreempt, @@ -507,6 +517,21 @@ func prepareCoroPhysicalFunctionPlan( plan.instructions[instruction] = instructionPlan } } + if whole != nil { + for instruction, instructionPlan := range plan.instructions { + if instruction == nil || instruction.Block() == nil || + !plan.reachableBlocks[instruction.Block()] || + !coroPhysicalInstructionNeedsRuntimeContext(instructionPlan) { + continue + } + if !logical.Exec.Contains(coro.NeedsRuntimeContext) { + return nil, fmt.Errorf( + "physical %s operation at %q requires runtime context but logical exec is %s", + instructionPlan.control, instruction.String(), logical.Exec, + ) + } + } + } if logical.AtomicCostProof.ProvesOutcomePlain() { if audit.universe == nil || audit.universe.coroProgramIR == nil { return nil, fmt.Errorf("atomic-cost physical proof requires one frozen ProgramIR") @@ -526,7 +551,6 @@ func prepareCoroPhysicalFunctionPlan( } targetPlan, planned := whole.FunctionPlan(instructionPlan.controlTarget) if !planned || targetPlan.ID != instructionPlan.controlTargetID || - targetPlan.ManagedEntry != coro.ManagedEntryOutcomePlain || !targetPlan.AtomicCostProof.ProvesOutcomePlain() { return nil, fmt.Errorf("atomic-cost physical proof target %q has no exact outcome capability", instructionPlan.controlTargetID) } @@ -547,6 +571,20 @@ func prepareCoroPhysicalFunctionPlan( return plan, nil } +// coroPhysicalInstructionNeedsRuntimeContext is the emission-side closure of +// compiler-injected helpers whose requirement is not represented by an +// ordinary Go call edge. ProgramIR seeds the corresponding source operation; +// this independent gate prevents a stale/custom analyzer from issuing an +// unsafe frame descriptor when those two projections disagree. +func coroPhysicalInstructionNeedsRuntimeContext(plan coroPhysicalInstructionPlan) bool { + switch plan.operation { + case coroPhysicalOperationChannelSelectPark, coroPhysicalOperationChannelSelectTry: + return true + default: + return false + } +} + func (plan *coroPhysicalFunctionPlan) instructionPlan(instruction ssa.Instruction) (coroPhysicalInstructionPlan, error) { if plan == nil || plan.function == nil || plan.owner == nil || instruction == nil || instruction.Parent() != plan.function { return coroPhysicalInstructionPlan{}, fmt.Errorf("physical instruction plan requires one exact frozen function owner and source instruction") @@ -638,6 +676,44 @@ func (ir *coroProgramIR) physicalFunctionPlan(function *ssa.Function, owner *pre return plan, nil } +// programCapabilities projects optional target-service demand only after the +// complete physical plan transaction has committed. Logical WaitForeign or a +// target's ability to host workers is insufficient: same-M episodes and dead +// source blocks must not start a worker pool. Conversely every reachable +// worker transaction below is the exact recipe codegen will emit. +func (ir *coroProgramIR) programCapabilities() (coro.ProgramCapabilities, error) { + if ir == nil || !ir.physicalPlansSealed { + return 0, fmt.Errorf("coroutine program capabilities require sealed physical plans") + } + worker := false + for key, function := range ir.physicalPlans { + if key.function == nil || key.owner == nil || function == nil || + function.function != key.function || function.owner != key.owner { + return 0, fmt.Errorf("coroutine program capabilities found an incomplete physical owner") + } + for instruction, plan := range function.instructions { + if instruction == nil || instruction.Parent() != function.function || instruction.Block() == nil { + return 0, fmt.Errorf("coroutine program capabilities found an incomplete physical instruction") + } + if !function.reachableBlocks[instruction.Block()] { + continue + } + switch plan.operation { + case coroPhysicalOperationWorkerSyscall, + coroPhysicalOperationWorkerForeign, + coroPhysicalOperationWorkerCgo, + coroPhysicalOperationWorkerCgoErrno: + worker = true + } + } + } + capabilities := coro.NewProgramCapabilities(worker) + if !capabilities.Valid() { + return 0, fmt.Errorf("coroutine program capabilities are invalid") + } + return capabilities, nil +} + // physicalFunctionPlanForEmission resolves the frozen definition projection // used by one physical emission. Ordinary functions require the exact current // package owner. A syntax-free generated wrapper is the sole exception: ABI @@ -714,6 +790,9 @@ func planCoroPhysicalInstruction( return result, fmt.Errorf("load semantic instruction recipe: %w", err) } result.semantic = semantic + if !semantic.evaluated { + return result, nil + } planCoroPhysicalControlInstruction(audit, whole, instruction, capabilities, &result) planCoroPhysicalOperationInstruction(audit, whole, instruction, capabilities, &result) planCoroPhysicalOutcomeInstruction(audit, cleanup, instruction, capabilities, &result) @@ -727,6 +806,8 @@ func planCoroPhysicalInstruction( case audit.frameRetainsManagedHeapAllocation(instruction): // Preserve AllocZ for semantic escapes and target-layout promotion // of oversized locals. CoroSplit retains the resulting pointer. + case audit.frameRetainsBorrowedAllocation(instruction): + result.recipe = coroPhysicalInstructionBorrowedAllocation case !instruction.Heap || audit.frameRetainsAllocation(instruction): result.recipe = coroPhysicalInstructionFrameAllocation } @@ -1050,6 +1131,15 @@ func planCoroPhysicalOutcomeInstruction( result.outcome = coroPhysicalOutcomeDeferRegister case *ssa.RunDefers: if cleanup == nil || len(cleanup.sites) == 0 { + if audit != nil && audit.plan != nil { + if logical, planned := audit.plan.FunctionPlan(audit.fn); planned && + !logical.Exec.Contains(coro.NeedsCleanupFrame) { + // ProgramIR removed NeedsCleanupFrame only after proving that no + // reachable Defer registration remains. Leave this synthetic + // RunDefers as an explicit no-op physical recipe. + return + } + } result.outcomeFailure = "RunDefers has no frozen cleanup plan" return } @@ -1549,7 +1639,7 @@ func planCoroPhysicalControlInstruction( result.controlFailureHard = true return } - if targetPlan.ManagedEntry == coro.ManagedEntryOutcomePlain { + if targetPlan.HasStaticOutcome() { result.control = coroPhysicalControlDirectOutcome if audit.ctx != nil && audit.ctx.prog != nil { result.directOutcomeNativeResult = !audit.ctx.prog.LocalGoTypeExceedsNativeStack( diff --git a/cl/coro_physical_plan_test.go b/cl/coro_physical_plan_test.go index 4e3bcc412f..bcbae11662 100644 --- a/cl/coro_physical_plan_test.go +++ b/cl/coro_physical_plan_test.go @@ -50,6 +50,8 @@ func TestCoroPhysicalPlanRuntimeHelperElisionIsRecipeOwned(t *testing.T) { {name: "integer divide keeps unrelated", plan: coroPhysicalInstructionPlan{recipe: coroPhysicalInstructionIntegerDivideByZeroGuard}, helper: "AssertNegativeShift"}, {name: "frame allocation", plan: coroPhysicalInstructionPlan{recipe: coroPhysicalInstructionFrameAllocation}, helper: "AllocZ", want: true}, {name: "frame allocation keeps unrelated", plan: coroPhysicalInstructionPlan{recipe: coroPhysicalInstructionFrameAllocation}, helper: "AllocU"}, + {name: "borrowed allocation", plan: coroPhysicalInstructionPlan{recipe: coroPhysicalInstructionBorrowedAllocation}, helper: "AllocZ", want: true}, + {name: "borrowed allocation keeps unrelated", plan: coroPhysicalInstructionPlan{recipe: coroPhysicalInstructionBorrowedAllocation}, helper: "AllocU"}, {name: "panic outcome", plan: coroPhysicalInstructionPlan{outcome: coroPhysicalOutcomePanic}, helper: "Panic", want: true}, {name: "recover outcome", plan: coroPhysicalInstructionPlan{outcome: coroPhysicalOutcomeRecover}, helper: "Recover", want: true}, } @@ -112,6 +114,68 @@ func Root(value int) int { return value + 1 } } } +func TestCoroProgramCapabilitiesUseOnlyReachablePhysicalWorkerRecipes(t *testing.T) { + ssaPkg, _, _ := buildGoSSAPkg(t, `package foo +func Worker(enabled bool) { + if enabled { + println("worker") + } +} +`) + function := ssaPkg.Func("Worker") + owner := &preparedEmissionPackage{identity: "foo"} + physical := &coroPhysicalFunctionPlan{ + function: function, + owner: owner, + reachableBlocks: make(map[*ssa.BasicBlock]bool), + instructions: make(map[ssa.Instruction]coroPhysicalInstructionPlan), + } + var reachable, unreachable ssa.Instruction + for _, block := range function.Blocks { + physical.reachableBlocks[block] = block.Index != 2 + for _, instruction := range block.Instrs { + physical.instructions[instruction] = coroPhysicalInstructionPlan{} + if block.Index == 2 && unreachable == nil { + unreachable = instruction + } else if block.Index != 2 && reachable == nil { + reachable = instruction + } + } + } + if reachable == nil || unreachable == nil { + t.Fatal("worker capability fixture has no reachable/unreachable instructions") + } + key := emissionFunctionOwnerKey{function: function, owner: owner} + commit := func() *coroProgramIR { + stage := newCoroPhysicalPlanStage() + if err := stage.freezePhysicalFunctionPlan(physical); err != nil { + t.Fatal(err) + } + ir := newCoroProgramIR() + ir.callsFrozen = true + if err := ir.commitPhysicalFunctionPlans(stage, map[emissionFunctionOwnerKey]none{key: {}}); err != nil { + t.Fatal(err) + } + return ir + } + + plan := physical.instructions[unreachable] + plan.operation = coroPhysicalOperationWorkerSyscall + physical.instructions[unreachable] = plan + capabilities, err := commit().programCapabilities() + if err != nil || capabilities.Worker() { + t.Fatalf("unreachable worker capability = (%v, %v), want no worker", capabilities, err) + } + + plan = physical.instructions[reachable] + plan.operation = coroPhysicalOperationWorkerCgo + physical.instructions[reachable] = plan + capabilities, err = commit().programCapabilities() + if err != nil || !capabilities.Worker() { + t.Fatalf("reachable worker capability = (%v, %v), want worker", capabilities, err) + } +} + func TestCoroPhysicalPlanOwnsManagedToRawPlainCall(t *testing.T) { ssaPkg, _, files := buildGoSSAPkg(t, `package foo func Critical() {} diff --git a/cl/coro_poll_wait.go b/cl/coro_poll_wait.go index a9d0a1ff25..a8cddbe1e1 100644 --- a/cl/coro_poll_wait.go +++ b/cl/coro_poll_wait.go @@ -85,7 +85,9 @@ func (p *context) compileCoroPollWait(b llssa.Builder, args []ssa.Value) llssa.E result := b.Alloc(p.prog.Uint32(), false) body.emitCoroParkOperation(p, b, coroParkOperation{ - shouldSuspend: b.Prog.BoolVal(true), + prepare: func(active llssa.Builder, _, _ uint32) llssa.Expr { + return active.Prog.BoolVal(true) + }, park: func(suspend llssa.Builder) { park := p.pkg.NewFunc(coroPollParkHookV2, coroPollParkSignatureV2(), llssa.InC) suspend.Call( diff --git a/cl/coro_program_ir.go b/cl/coro_program_ir.go index 8d40fdf5e8..5d34fabf56 100644 --- a/cl/coro_program_ir.go +++ b/cl/coro_program_ir.go @@ -201,6 +201,7 @@ func (ir *coroProgramIR) freezeSite(function *ssa.Function, owner *preparedEmiss return nil } if len(plan.managedRuntimeHelpers) != 0 || len(plan.plainRuntimeHelpers) != 0 || + plan.plainAllocation.borrowed() || len(plan.localityDispatchers) != 0 { byInstruction[instruction] = plan } @@ -223,15 +224,51 @@ func (ir *coroProgramIR) freezeSiteOwner( return fmt.Errorf("coroutine site plan owner %q has no frozen function preamble", function.Name()) } semantic := ir.semanticPlans[key] + preamble := ir.functionPreambles[key] + facts, err := deriveCoroLocalBodyFacts(prog, function, semantic, preamble.emitsGoBody) + if err != nil { + return err + } + if previous, exists := ir.localBodyFacts[function]; exists && !previous.Same(facts) { + return fmt.Errorf("function %q acquired owner-dependent local semantic facts", function.Name()) + } + ir.localBodyFacts[function] = facts + ir.siteOwners[key] = none{} + return nil +} + +// deriveCoroLocalBodyFacts projects one already-frozen semantic instruction +// table into the analyzer-owned local body summary. Call-site finalization may +// invoke it a second time after replacing an exact compiler-elided intrinsic +// call with its narrower semantic recipe. Keeping the projection here avoids +// an analysis-time raw SSA rescan and keeps ProgramIR as the sole authority. +func deriveCoroLocalBodyFacts( + prog llssa.Program, + function *ssa.Function, + semantic map[ssa.Instruction]coroSemanticInstructionPlan, + outcomePlainEligible bool, +) (coro.SSAFunctionBodyFacts, error) { + if prog == nil || function == nil { + return coro.SSAFunctionBodyFacts{}, fmt.Errorf("local body facts require one exact program and function") + } facts := coro.SSAFunctionBodyFacts{ - Effect: coro.NoSuspend, - OutcomePlainLeaf: function.Blocks != nil, - OutcomePlainDAG: function.Blocks != nil, + Effect: coro.NoSuspend, + OutcomePlainLeaf: function.Blocks != nil, + OutcomePlainDAG: function.Blocks != nil, + StaticOutcomeLocal: function.Blocks != nil, } if function.Blocks != nil { facts.Exec = coro.MayUnwind } + if coroRuntimeContextPrimitive(function) { + // These are the only bottom-level runtime operations which observe or + // replace the ambient logical G. All ordinary Go wrappers acquire this + // bit through the normal call fixed point, including across imported + // library summaries; source annotations are neither required nor read. + facts.Exec = facts.Exec.Join(coro.NeedsRuntimeContext) + } atomicBlocks := make([]coro.SSAAtomicBlockFacts, len(function.Blocks)) + hasEvaluatedDefer := false for _, block := range function.Blocks { blockFacts := coro.SSAAtomicBlockFacts{Index: block.Index} for _, successor := range block.Succs { @@ -241,13 +278,19 @@ func (ir *coroProgramIR) freezeSiteOwner( for instructionIndex, instruction := range block.Instrs { plan, ok := semantic[instruction] if !ok { - return fmt.Errorf("coroutine semantic SitePlan owner %q omitted source instruction %q", function.Name(), instruction.String()) + return coro.SSAFunctionBodyFacts{}, fmt.Errorf("coroutine semantic SitePlan owner %q omitted source instruction %q", function.Name(), instruction.String()) } if !plan.evaluated { facts.OutcomePlainLeaf = false facts.OutcomePlainDAG = false continue } + if _, deferInstruction := instruction.(*ssa.Defer); deferInstruction { + hasEvaluatedDefer = true + } + if !plan.staticOutcome { + facts.StaticOutcomeLocal = false + } if !coroOutcomePlainLeafSemanticRecipe(plan) { facts.OutcomePlainLeaf = false } @@ -258,7 +301,7 @@ func (ir *coroProgramIR) freezeSiteOwner( facts.OutcomePlainCallCount++ call, ok := instruction.(*ssa.Call) if !ok { - return fmt.Errorf("coroutine call recipe in %q is attached to %T", function.Name(), instruction) + return coro.SSAFunctionBodyFacts{}, fmt.Errorf("coroutine call recipe in %q is attached to %T", function.Name(), instruction) } blockFacts.Calls = append(blockFacts.Calls, coro.SSAAtomicCallSiteFacts{ Instruction: call, @@ -273,6 +316,11 @@ func (ir *coroProgramIR) freezeSiteOwner( // that hidden allocation, so reject the optimization while the // ordinary coroutine fallback is still available. facts.OutcomePlainDAG = false + // An unbounded static-outcome body has the same synchronous + // caller-owned result-slot constraint. Its full coroutine primary + // may use a managed frame slot, but the native twin must not place + // an oversized child result on a fixed stack. + facts.StaticOutcomeLocal = false } } facts.Effect = facts.Effect.Join(plan.effect) @@ -285,23 +333,51 @@ func (ir *coroProgramIR) freezeSiteOwner( atomicBlocks[block.Index] = blockFacts } facts.Effect = facts.Effect.Normalize() + if !hasEvaluatedDefer { + // x/tools retains RunDefers on normal returns whenever the function has + // any syntactic defer, including one behind a constant-dead branch. With + // no evaluated registration site it is semantically a no-op and does not + // require a cleanup frame. + facts.Exec &^= coro.NeedsCleanupFrame + for instruction, plan := range semantic { + if _, runDefers := instruction.(*ssa.RunDefers); !runDefers || !plan.evaluated { + continue + } + plan.exec &^= coro.NeedsCleanupFrame + plan.materialized = false + semantic[instruction] = plan + } + } facts.HasCycle = coroSemanticEvaluatedCFGHasCycle(function.Blocks, semantic) if len(function.Blocks) != 0 { atomicPath, err := coro.NewSSAAtomicPathFacts(function, atomicBlocks) if err != nil { - return fmt.Errorf("function %q atomic path projection: %w", function.Name(), err) + return coro.SSAFunctionBodyFacts{}, fmt.Errorf("function %q atomic path projection: %w", function.Name(), err) } facts.AtomicPath = atomicPath } else { facts.OutcomePlainLeaf = false facts.OutcomePlainDAG = false + facts.StaticOutcomeLocal = false } - if previous, exists := ir.localBodyFacts[function]; exists && !previous.Same(facts) { - return fmt.Errorf("function %q acquired owner-dependent local semantic facts", function.Name()) + if !outcomePlainEligible { + facts.OutcomePlainLeaf = false + facts.OutcomePlainDAG = false + } + return facts, nil +} + +func coroRuntimeContextPrimitive(function *ssa.Function) bool { + if function == nil || function.Pkg == nil || function.Pkg.Pkg == nil || + llssa.PathOf(function.Pkg.Pkg) != "github.com/goplus/llgo/runtime/internal/runtime" { + return false + } + switch function.Name() { + case "getg", "getgIfPresent", "setg", "setgRaw": + return true + default: + return false } - ir.localBodyFacts[function] = facts - ir.siteOwners[key] = none{} - return nil } // coroOutcomePlainLeafSemanticRecipe consumes the capability already frozen by @@ -329,6 +405,103 @@ func coroOutcomePlainDAGSemanticRecipe(plan coroSemanticInstructionPlan) bool { } } +// finalizeOutcomePlainIntrinsicSemantics narrows exact compiler-elided source +// calls after the call-site table is complete. Raw SSA alone cannot know that +// a declaration call will become one helper-free LLVM operation, so the first +// semantic pass deliberately classifies it as an ordinary call. This final +// ProgramIR builder step admits only operations whose frozen call recipe is +// independently shape-checked and allocation-free. +// +// Atomic intrinsics acquire their bounded leaf proof here. A real inline yield +// is also recorded here as an evaluated local effect: this distinguishes an +// explicit scheduler handoff from the synthetic YieldOnly/NeedsPreempt seed +// added later for an otherwise synchronous CFG loop. Static outcome twins may +// omit the latter under the current compute-blocking policy, but must never +// erase the former. Other InlineNoSuspend intrinsics include asm, dynamic +// alloca, control transfer, and target-specific operations; the broad enum is +// therefore not by itself an outcome-plain proof. +func (ir *coroProgramIR) finalizeOutcomePlainIntrinsicSemantics( + prog llssa.Program, + functions []*ssa.Function, + sortedUseOwners func(*ssa.Function) []*preparedEmissionPackage, +) error { + if ir == nil || prog == nil || sortedUseOwners == nil { + return fmt.Errorf("outcome-plain intrinsic finalization requires one exact ProgramIR input") + } + if ir.callsFrozen { + return fmt.Errorf("outcome-plain intrinsic finalization occurred after call SitePlan freeze") + } + for _, function := range functions { + if function == nil || len(function.Blocks) == 0 { + continue + } + refined := false + for _, block := range function.Blocks { + for _, instruction := range block.Instrs { + call, ordinary := instruction.(*ssa.Call) + if !ordinary { + continue + } + frozen, found := ir.callPlans[call] + if !found || frozen.failure != "" || !frozen.plan.Intrinsic || + frozen.plan.Elision != CoroCallElidedIntrinsic { + continue + } + atomic := frozen.plan.IntrinsicSemantics == CoroIntrinsicCallInlineNoSuspend && + isCoroAtomicIntrinsic(frozen.opcode) + realYield := frozen.plan.IntrinsicSemantics == CoroIntrinsicCallInlineYield + if !atomic && !realYield { + continue + } + for _, owner := range sortedUseOwners(function) { + key := emissionFunctionOwnerKey{function: function, owner: owner} + if _, sealed := ir.siteOwners[key]; !sealed { + return fmt.Errorf("atomic intrinsic in %q has no frozen semantic owner %q", function.Name(), owner.identity) + } + semantic, present := ir.semanticPlans[key][call] + if !present || !semantic.evaluated || semantic.recipe != coro.RecipeID("cl.ssa.call.v1") || + semantic.effect != coro.NoSuspend || semantic.exec != 0 { + return fmt.Errorf("atomic intrinsic call %q has an incompatible preliminary semantic recipe", call.String()) + } + if atomic { + semantic.recipe = coro.RecipeID("cl.intrinsic.atomic.inline-nosuspend.v1") + semantic.outcomePlainLeaf = true + semantic.staticOutcome = true + } else { + semantic.recipe, semantic.effect = coroIntrinsicLoweringRecipe(CoroIntrinsicCallInlineYield) + semantic.materialized = true + semantic.outcomePlainLeaf = false + semantic.staticOutcome = false + } + ir.semanticPlans[key][call] = semantic + } + refined = true + } + } + if !refined { + continue + } + var final coro.SSAFunctionBodyFacts + for index, owner := range sortedUseOwners(function) { + key := emissionFunctionOwnerKey{function: function, owner: owner} + preamble, present := ir.functionPreambles[key] + if !present { + return fmt.Errorf("finalize atomic intrinsic body %q: owner %q has no function preamble", function.Name(), owner.identity) + } + facts, err := deriveCoroLocalBodyFacts(prog, function, ir.semanticPlans[key], preamble.emitsGoBody) + if err != nil { + return fmt.Errorf("finalize atomic intrinsic body %q: %w", function.Name(), err) + } + if index != 0 && !final.Same(facts) { + return fmt.Errorf("function %q acquired owner-dependent finalized local semantic facts", function.Name()) + } + final = facts + } + ir.localBodyFacts[function] = final + } + return nil +} + func coroSemanticEvaluatedCFGHasCycle( blocks []*ssa.BasicBlock, semantic map[ssa.Instruction]coroSemanticInstructionPlan, @@ -388,7 +561,7 @@ func (ir *coroProgramIR) semanticInstructionPlan( } func (ir *coroProgramIR) functionLocalBodyFacts(function *ssa.Function) (coro.SSAFunctionBodyFacts, error) { - if ir == nil || function == nil { + if ir == nil || !ir.callsFrozen || function == nil { return coro.SSAFunctionBodyFacts{}, fmt.Errorf("local body facts require one exact function") } facts, ok := ir.localBodyFacts[function] @@ -563,6 +736,7 @@ func cloneCoroEmissionSitePlan(plan coroEmissionSitePlan) coroEmissionSitePlan { func sameCoroEmissionSitePlan(first, second coroEmissionSitePlan) bool { return slices.Equal(first.managedRuntimeHelpers, second.managedRuntimeHelpers) && slices.Equal(first.plainRuntimeHelpers, second.plainRuntimeHelpers) && + first.plainAllocation == second.plainAllocation && slices.Equal(first.localityDispatchers, second.localityDispatchers) && first.hasCallPlan == second.hasCallPlan && sameCoroFrozenCallSitePlan(first.callPlan, second.callPlan) } diff --git a/cl/coro_pure_ssa.go b/cl/coro_pure_ssa.go index 4008ff511b..e472c06040 100644 --- a/cl/coro_pure_ssa.go +++ b/cl/coro_pure_ssa.go @@ -471,6 +471,13 @@ func (a *coroPhysicalPureSSAAudit) validateFieldAddr(field *ssa.FieldAddr) strin if err := validateCoroPhysicalSSAValueType(a.typeOf(field.Type())); err != nil { return "field address has unsupported type: " + err.Error() } + if a.allowImplicitNilFault { + // ExplicitStatus lowering emits the nil branch before forming the GEP and + // publishes the fault directly. This covers address-taking consumers such + // as an inline atomic intrinsic; unlike a dominated load there is no later + // dereference instruction that can own the guard. + return a.requireOnlyCompilerElidedRuntimeHelpers(field, "AssertNilDeref") + } return a.requireNoRuntimeHelpersExcept(field, "AssertNilDeref") } @@ -904,7 +911,8 @@ func (a *coroPhysicalPureSSAAudit) validateMakeInterface(box *ssa.MakeInterface) // lowerings. This admits ordinary `return errno` error paths without // granting a symbol-name exception to syscall or to error itself. physical := a.ctx.type_(box.X.Type(), llssa.InGo) - needsAlloc := !emissionDirectIfaceType(physical.RawType()) + needsAlloc := !emissionDirectIfaceType(physical.RawType()) && + !makeInterfaceUsesConstantBacking(box) needsNilCheck := false needsTypedMove := false if unop, ok := box.X.(*ssa.UnOp); ok && unop.Op == token.MUL && @@ -924,7 +932,7 @@ func (a *coroPhysicalPureSSAAudit) validateMakeInterface(box *ssa.MakeInterface) if needsNilCheck { expected = append(expected, "AssertNilDeref") } - if !target.Empty() { + if !target.Empty() && !llssa.CanBuildStaticItab(target, physical.RawType()) { expected = append(expected, "NewItab") } if needsTypedMove { @@ -2957,7 +2965,7 @@ func (a *coroPhysicalPureSSAAudit) validateCloseBuiltin(call *ssa.Call) string { if !a.allowImplicitNilFault { return "close builtin requires the explicit-status panic ABI" } - return a.requireFrozenExactRuntimeHelper(call, "CoroChanTryClose") + return a.requireFrozenExactRuntimeHelper(call, "CoroChanTryCloseTask") } // validateUnsafeDataBuiltin accepts only the two header projection intrinsics. diff --git a/cl/coro_python.go b/cl/coro_python.go index c685683528..09aade79dc 100644 --- a/cl/coro_python.go +++ b/cl/coro_python.go @@ -161,7 +161,7 @@ func (p *context) resolveCoroPythonCall( keyFields = append(keyFields, "cl-coro-python-call-thunk-v1", fn.Name(), - structuralEmissionABITypeKey(signature), + p.cachedStrictEmissionABITypeKey(signature), strconv.Itoa(p.prog.PointerSize()), ) for _, argument := range args { @@ -169,14 +169,14 @@ func (p *context) resolveCoroPythonCall( panic("coroutine Python call has an untyped physical argument") } fields = append(fields, argument.Type) - keyFields = append(keyFields, structuralEmissionABITypeKey(argument.RawType())) + keyFields = append(keyFields, p.cachedStrictEmissionABITypeKey(argument.RawType())) } resultField := -1 if results != nil && results.Len() == 1 { resultField = len(fields) resultType := p.prog.Type(results.At(0).Type(), llssa.InC) fields = append(fields, resultType) - keyFields = append(keyFields, structuralEmissionABITypeKey(resultType.RawType())) + keyFields = append(keyFields, p.cachedStrictEmissionABITypeKey(resultType.RawType())) } recordType := p.prog.Struct(fields...) name := coroPythonCallThunkPrefixV1 + emissionDigest(framedEmissionKey(keyFields...)) diff --git a/cl/coro_raw_plain_entry_test.go b/cl/coro_raw_plain_entry_test.go index db9898909e..60609c5fd4 100644 --- a/cl/coro_raw_plain_entry_test.go +++ b/cl/coro_raw_plain_entry_test.go @@ -456,7 +456,12 @@ func implementation(value uint32) uint32 { return value + 1 } defer prog.Dispose() definitionModule := definitionLL.Module() declarationModule := declarationLL.Module() - defer definitionModule.Dispose() + definitionConsumed := false + defer func() { + if !definitionConsumed { + definitionModule.Dispose() + } + }() defer declarationModule.Dispose() for name, module := range map[string]llvm.Module{"definition": definitionModule, "declaration": declarationModule} { if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { @@ -479,6 +484,52 @@ func implementation(value uint32) uint32 { return value + 1 } if !strings.Contains(rootBody, "runtimeHook$coro") || strings.Contains(rootBody, "runtimeHook\"(") { t.Fatalf("managed root did not select the canonical coroutine alias:\n%s", rootBody) } + + // LLVM 22's annotation-elision protocol is only useful if the exact proof + // survives the bodyless managed-linkname facade and the package-module + // boundary. Link the two frontend modules just as FullLTO does, then require + // the child ramp and its allocator path to disappear from Root.resume. + elideKind := llvm.AttributeKindID("coro_elide_safe") + if elideKind == 0 { + return + } + foundElideSafeCall := false + rootEntry := declarationModule.NamedFunction("example.com/coro/linkdecl.Root" + coroPrimarySuffix) + for block := rootEntry.FirstBasicBlock(); !block.IsNil(); block = llvm.NextBasicBlock(block) { + for instruction := block.FirstInstruction(); !instruction.IsNil(); instruction = llvm.NextInstruction(instruction) { + if instruction.InstructionOpcode() != llvm.Call || + instruction.CalledValue().Name() != baseName+coroPrimarySuffix { + continue + } + foundElideSafeCall = instruction.GetCallSiteEnumAttribute(-1, elideKind).IsEnum() + } + } + if !foundElideSafeCall { + t.Fatalf("managed-linkname static await lost coro_elide_safe before FullLTO:\n%s", rootBody) + } + if err := llvm.LinkModules(declarationModule, definitionModule); err != nil { + t.Fatalf("link managed-linkname coroutine modules: %v", err) + } + definitionConsumed = true + options := llvm.NewPassBuilderOptions() + defer options.Dispose() + options.SetVerifyEach(true) + if err := declarationModule.RunPasses("lto", prog.TargetMachine(), options); err != nil { + t.Fatalf("run managed-linkname coroutine FullLTO gate: %v\n%s", err, declarationModule.String()) + } + if err := llvm.VerifyModule(declarationModule, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify managed-linkname coroutine FullLTO gate: %v\n%s", err, declarationModule.String()) + } + rootResume := declarationModule.NamedFunction("example.com/coro/linkdecl.Root" + coroPrimarySuffix + ".resume") + if rootResume.IsNil() { + t.Fatalf("managed-linkname FullLTO gate has no Root coroutine resume:\n%s", declarationModule.String()) + } + resumeBody := rootResume.String() + if strings.Contains(resumeBody, "call ptr @\""+baseName+coroPrimarySuffix+"\"(") || + strings.Contains(resumeBody, "call ptr @"+baseName+coroPrimarySuffix+"(") || + strings.Contains(resumeBody, coroFrameAllocHookV1) { + t.Fatalf("managed-linkname static await retained its ramp/allocation path after FullLTO:\n%s", resumeBody) + } } func TestCoroUnpairedGoLinknameDefinitionRemainsRawBoundary(t *testing.T) { diff --git a/cl/coro_semantic_plan.go b/cl/coro_semantic_plan.go index a20b620fef..e33fc86842 100644 --- a/cl/coro_semantic_plan.go +++ b/cl/coro_semantic_plan.go @@ -45,6 +45,10 @@ type coroSemanticInstructionPlan struct { // only here, while classifying raw SSA, and is consumed from the frozen // ProgramIR by analysis and physical emission. outcomePlainLeaf bool + // staticOutcome records that this local instruction can participate in an + // unbounded synchronous outcome twin. Calls still require a whole-program + // exact-target proof; this bit only closes the local lowering vocabulary. + staticOutcome bool } func coroOutcomePlainBasicInfo(typ types.Type) types.BasicInfo { @@ -181,14 +185,22 @@ func planCoroSemanticInstruction(instruction ssa.Instruction) (plan coroSemantic }() ordinary := func(recipe string) (coroSemanticInstructionPlan, error) { return coroSemanticInstructionPlan{ - class: coro.OpPure, - recipe: coro.RecipeID(recipe), - effect: coro.NoSuspend, + class: coro.OpPure, + recipe: coro.RecipeID(recipe), + effect: coro.NoSuspend, + staticOutcome: true, }, nil } leaf := func(recipe string, safe bool) (coroSemanticInstructionPlan, error) { plan, err := ordinary(recipe) plan.outcomePlainLeaf = safe + // The bounded leaf vocabulary is intentionally narrower than the + // synchronous outcome vocabulary. Operations such as string arithmetic, + // interface comparison, and checked integer arithmetic may lower through + // exact runtime helpers or explicit fault edges, but neither fact makes the + // source operation suspend. Static-outcome closure accounts for those helper + // calls independently, so do not erase the ordinary synchronous capability + // merely because this instruction is not an allocation-free atomic leaf. return plan, err } control := func(recipe string, exec coro.ExecFlags) (coroSemanticInstructionPlan, error) { @@ -210,6 +222,7 @@ func planCoroSemanticInstruction(instruction ssa.Instruction) (plan coroSemantic return leaf("cl.ssa.phi.v1", true) case *ssa.Call: plan, err := ordinary("cl.ssa.call.v1") + plan.staticOutcome = true if err != nil { return plan, err } @@ -219,6 +232,7 @@ func planCoroSemanticInstruction(instruction ssa.Instruction) (plan coroSemantic plan.exec = coro.MayUnwind plan.materialized = true plan.recipe = coro.RecipeID("cl.ssa.builtin-panic.v0") + plan.staticOutcome = true } } return plan, nil @@ -245,7 +259,12 @@ func planCoroSemanticInstruction(instruction ssa.Instruction) (plan coroSemantic case *ssa.SliceToArrayPointer: return ordinary("cl.ssa.slice-to-array-pointer.v1") case *ssa.MakeInterface: - return ordinary("cl.ssa.make-interface.v1") + plan, err := ordinary("cl.ssa.make-interface.v1") + // The source operation itself is synchronous. Any backing allocation or + // itab construction remains an explicit owner-scoped lowered-call edge; + // static-outcome planning closes those helpers independently. + plan.staticOutcome = true + return plan, err case *ssa.MakeClosure: return ordinary("cl.ssa.make-closure.v1") case *ssa.MakeMap: @@ -273,9 +292,13 @@ func planCoroSemanticInstruction(instruction ssa.Instruction) (plan coroSemantic effect = coro.MayPark } return coroSemanticInstructionPlan{ - class: coro.OpSelect, - recipe: coro.RecipeID(recipe), - effect: effect, + class: coro.OpSelect, + recipe: coro.RecipeID(recipe), + effect: effect, + // The current select helper ABI still samples owner-local completion + // through the installed runtime G. Single-channel send/receive and + // close already carry an explicit task and need no such seed. + exec: coro.NeedsRuntimeContext, materialized: true, }, nil case *ssa.Range: @@ -294,12 +317,19 @@ func planCoroSemanticInstruction(instruction ssa.Instruction) (plan coroSemantic plan, err := control("cl.ssa.return.v1", 0) plan.materialized = false plan.outcomePlainLeaf = true + plan.staticOutcome = true return plan, err case *ssa.RunDefers: - return control("cl.ssa.run-defers.v0", coro.NeedsCleanupFrame) + plan, err := control("cl.ssa.run-defers.v0", coro.NeedsCleanupFrame) + // Whole-function projection admits this only when no evaluated Defer site + // remains. In that case x/tools' synthetic RunDefers is a no-op and the + // physical static-outcome recipe removes it explicitly. + plan.staticOutcome = true + return plan, err case *ssa.Panic: plan, err := control("cl.ssa.panic.v0", coro.MayUnwind) plan.outcomePlainLeaf = true + plan.staticOutcome = true return plan, err case *ssa.Go: return coroSemanticInstructionPlan{ diff --git a/cl/coro_site_plan.go b/cl/coro_site_plan.go index dd35d24151..c797ebd708 100644 --- a/cl/coro_site_plan.go +++ b/cl/coro_site_plan.go @@ -45,6 +45,8 @@ type coroSiteEmissionObserver struct { expectedIntrinsicSemantics CoroIntrinsicCallSemantics expectedElision CoroCallElisionKind seenElision bool + expectedPlainAllocation coroPlainAllocationPlan + seenPlainAllocation bool expectedPhysical coroPhysicalInstructionPlan hasExpectedPhysical bool seenSemantic bool @@ -233,6 +235,9 @@ func (p *context) beginCoroSiteEmissionMode( seenLocalityDispatchers: make(map[*ssa.Function]none, len(localityDispatchers)), observeFrozenSite: p.emissionUniverse.CompleteRuntimeABI(), } + if placement == coroRuntimeHelperAtSource && !hasPhysical { + observer.expectedPlainAllocation = plan.plainAllocation + } if hasPhysical { observer.expectedPhysical = physical observer.hasExpectedPhysical = true @@ -305,6 +310,12 @@ func (p *context) beginCoroSiteEmissionMode( instruction.String(), observer.expectedElision, )) } + if observer.expectedPlainAllocation.borrowed() && !observer.seenPlainAllocation { + panic(fmt.Errorf( + "coroutine emission site %q omitted frozen borrowed-allocation recipe", + instruction.String(), + )) + } if observer.hasExpectedPhysical && !observer.seenSemantic { panic(fmt.Errorf( "coroutine emission site %q omitted frozen semantic recipe %s", @@ -350,6 +361,39 @@ func (p *context) beginCoroSiteEmissionMode( } } +// selectCoroPlainBorrowedAllocation consumes the exact source SitePlan storage +// recipe for a synchronous Go body. Physical coro/outcome bodies use their +// separate instruction plan and never enter this selector. +func (p *context) selectCoroPlainBorrowedAllocation(allocation *ssa.Alloc) bool { + if p == nil || allocation == nil || p.hasCoroPhysicalEmission() { + return false + } + if p.rawPlainBody && p.emissionUniverse != nil && + p.emissionUniverse.coroProgramIR != nil { + // Raw-plain compiler/runtime islands deliberately suppress the ordinary + // helper observer because their complete call inventory is validated by + // the separate raw closure. Storage is orthogonal: consume the same exact + // owner-scoped frozen allocation recipe without re-running escape proof. + plan, err := p.emissionUniverse.coroProgramIR.sitePlan(p, allocation) + if err != nil { + panic(fmt.Errorf("raw-plain allocation %q: %w", allocation.String(), err)) + } + return plan.plainAllocation.borrowed() + } + observer := p.coroEmissionSite() + if observer == nil || observer.instruction != allocation || observer.hasExpectedPhysical { + return false + } + if !observer.expectedPlainAllocation.borrowed() { + return false + } + if observer.seenPlainAllocation { + panic(fmt.Errorf("source allocation %q consumed its borrowed storage recipe more than once", allocation.String())) + } + observer.seenPlainAllocation = true + return true +} + func (p *context) observeCoroSemanticInstruction(instruction ssa.Instruction) { observer := p.coroEmissionSite() if observer == nil { diff --git a/cl/coro_slice_to_array.go b/cl/coro_slice_to_array.go index f9755ecc71..cf99ed8155 100644 --- a/cl/coro_slice_to_array.go +++ b/cl/coro_slice_to_array.go @@ -96,12 +96,12 @@ func (p *context) compileCoroSliceToArrayPointer( plan coroPhysicalInstructionPlan, ) llssa.Expr { body := p.coroBody() - if body == nil || conversion == nil || b == nil || b.Func != p.fn { - panic("structured slice-to-array-pointer conversion escaped its physical coroutine body") + if !p.hasStructuredOutcomePhysicalBody() || conversion == nil || b == nil || b.Func != p.fn { + panic("structured slice-to-array-pointer conversion escaped its physical body") } if !p.coroEmissionExplicitStatus() || - body.abi.version < coroPhysicalABIVersionV1 { - panic("slice-to-array-pointer fault requires the PhysicalABIV1 explicit-status panic ABI") + body != nil && body.abi.version < coroPhysicalABIVersionV1 { + panic("slice-to-array-pointer fault requires an explicit-status physical ABI") } if plan.recipe != coroPhysicalInstructionSliceToArrayPointer || plan.bound < 0 || plan.boundsDisabled != b.Prog.BoundsChecksDisabled() || diff --git a/cl/coro_timer_sleep.go b/cl/coro_timer_sleep.go index 1b74eac3e1..7034a6850b 100644 --- a/cl/coro_timer_sleep.go +++ b/cl/coro_timer_sleep.go @@ -92,7 +92,9 @@ func (p *context) compileCoroTimerSleep(b llssa.Builder, args []ssa.Value) { state := b.Alloc(p.prog.RuntimeType("CoroTimerParkV2"), false) body.emitCoroParkOperation(p, b, coroParkOperation{ - shouldSuspend: b.Prog.BoolVal(true), + prepare: func(active llssa.Builder, _, _ uint32) llssa.Expr { + return active.Prog.BoolVal(true) + }, park: func(suspend llssa.Builder) { park := p.pkg.NewFunc(coroTimerParkHookV2, coroTimerParkSignatureV2(), llssa.InC) suspend.Call( @@ -137,7 +139,9 @@ func (p *context) compileCoroControlledTimerWait(b llssa.Builder, args []ssa.Val result := b.Alloc(p.prog.Uint32(), false) body.emitCoroParkOperation(p, b, coroParkOperation{ - shouldSuspend: b.Prog.BoolVal(true), + prepare: func(active llssa.Builder, _, _ uint32) llssa.Expr { + return active.Prog.BoolVal(true) + }, park: func(suspend llssa.Builder) { park := p.pkg.NewFunc(coroControlledTimerParkHookV2, coroControlledTimerParkSignatureV2(), llssa.InC) suspend.Call( diff --git a/cl/coro_unsafe_slice.go b/cl/coro_unsafe_slice.go index efe908ab34..0101288a19 100644 --- a/cl/coro_unsafe_slice.go +++ b/cl/coro_unsafe_slice.go @@ -81,10 +81,10 @@ func (p *context) compileCoroUnsafeSlice( pointerValue, lengthValue llssa.Expr, ) llssa.Expr { body := p.coroBody() - if body == nil || b == nil || b.Func != p.fn || call == nil || + if !p.hasStructuredOutcomePhysicalBody() || b == nil || b.Func != p.fn || call == nil || !p.coroEmissionExplicitStatus() || - body.abi.version < coroPhysicalABIVersionV1 { - panic("unsafe.Slice coroutine lowering requires the PhysicalABIV1 explicit-status ABI") + body != nil && body.abi.version < coroPhysicalABIVersionV1 { + panic("unsafe.Slice structured lowering requires an explicit-status physical ABI") } results := call.Signature().Results() if results == nil || results.Len() != 1 || len(call.Args) != 2 || diff --git a/cl/coro_unsafe_string.go b/cl/coro_unsafe_string.go index 4e55617e79..f6ff691e8b 100644 --- a/cl/coro_unsafe_string.go +++ b/cl/coro_unsafe_string.go @@ -76,10 +76,10 @@ func (p *context) compileCoroUnsafeString( pointerValue, lengthValue llssa.Expr, ) llssa.Expr { body := p.coroBody() - if body == nil || b == nil || b.Func != p.fn || call == nil || + if !p.hasStructuredOutcomePhysicalBody() || b == nil || b.Func != p.fn || call == nil || !p.coroEmissionExplicitStatus() || - body.abi.version < coroPhysicalABIVersionV1 { - panic("unsafe.String coroutine lowering requires the PhysicalABIV1 explicit-status ABI") + body != nil && body.abi.version < coroPhysicalABIVersionV1 { + panic("unsafe.String structured lowering requires an explicit-status physical ABI") } results := call.Signature().Results() if results == nil || results.Len() != 1 || len(call.Args) != 2 || pointerValue.IsNil() || lengthValue.IsNil() { diff --git a/cl/coro_worker.go b/cl/coro_worker.go index d10847218f..6a18b17830 100644 --- a/cl/coro_worker.go +++ b/cl/coro_worker.go @@ -330,7 +330,9 @@ func (p *context) compileCoroWorkerWordCall( emitPark := func(worker llssa.Builder) { body.emitCoroParkOperation(p, worker, coroParkOperation{ - shouldSuspend: worker.Prog.BoolVal(true), + prepare: func(active llssa.Builder, _, _ uint32) llssa.Expr { + return active.Prog.BoolVal(true) + }, park: func(suspend llssa.Builder) { park := p.pkg.NewFunc(parkHook, parkSignature, llssa.InC) suspend.Call(park.Expr, physicalArgs...) diff --git a/cl/coro_worker_cgo.go b/cl/coro_worker_cgo.go index caf91e4c65..c40f9ad457 100644 --- a/cl/coro_worker_cgo.go +++ b/cl/coro_worker_cgo.go @@ -347,7 +347,7 @@ func (ctx *context) coroCgoErrnoWorkerCallShape( strconv.Itoa(block.Index), strconv.Itoa(semantic), ) - abi := structuralCFunctionABITypeKey(directSignature) + abi := u.emissionTypeKeys.cFunctionABI(directSignature) targetSpec := u.prog.TargetSpec() certificate := CoroCgoWorkerCallCertificate{ ID: emissionDigest(framedEmissionKey( @@ -599,7 +599,7 @@ func (u *EmissionUniverse) freezeCoroCgoWorkerCallCertificate( if linkIdentity == "" || targetIdentity == "" || targetIdentity == "" || targetIdentity == "" { return certificate, nil, false, fmt.Errorf("generated cgo worker target %q has no frozen identity", target.Name()) } - abi := structuralEmissionABITypeKey(signature) + abi := u.emissionTypeKeys.strictABI(signature) targetSpec := u.prog.TargetSpec() certificate = CoroCgoWorkerCallCertificate{ ID: emissionDigest(framedEmissionKey( @@ -659,7 +659,7 @@ func validateCoroWorkerCgoCall( if signatureErr != nil { return shape, true, signatureErr } - if structuralEmissionABITypeKey(signature) != frozen.cgoWorker.ABISignature { + if universe.emissionTypeKeys.strictABI(signature) != frozen.cgoWorker.ABISignature { return shape, true, fmt.Errorf("generated cgo worker ABI differs from its frozen certificate") } result := types.Type(nil) @@ -729,7 +729,7 @@ func validateCoroWorkerCgoErrnoCall( return shape, true, fmt.Errorf("frozen C2 cgo call no longer has its generated worker shape") } if derived.certificate != frozen.cgoWorker || - structuralCFunctionABITypeKey(derived.signature) != frozen.cgoWorker.ABISignature { + universe.emissionTypeKeys.cFunctionABI(derived.signature) != frozen.cgoWorker.ABISignature { return shape, true, fmt.Errorf("generated C2 cgo call differs from its frozen worker certificate") } return derived, true, nil @@ -774,7 +774,7 @@ func (p *context) coroWorkerCgoThunk(shape coroWorkerCgoCallShape, target llssa. "cl-coro-worker-cgo-thunk-v1", shape.certificate.ID, target.Name(), - structuralEmissionABITypeKey(shape.signature), + p.cachedStrictEmissionABITypeKey(shape.signature), )) thunk := p.pkg.NewFuncEx(name, coroWorkerForeignThunkSignature(), llssa.InC, false, true) if thunk.HasBody() { @@ -811,7 +811,7 @@ func (p *context) coroWorkerCgoErrnoThunk(shape coroWorkerCgoErrnoCallShape) lls name := coroWorkerCgoThunkPrefixV1 + emissionDigest(framedEmissionKey( "cl-coro-worker-cgo-errno-thunk-v1", shape.certificate.ID, - structuralCFunctionABITypeKey(shape.signature), + p.cachedCFunctionABITypeKey(shape.signature), )) thunk := p.pkg.NewFuncEx(name, coroWorkerForeignThunkSignature(), llssa.InC, false, true) if thunk.HasBody() { diff --git a/cl/coro_worker_foreign.go b/cl/coro_worker_foreign.go index be73dfeea8..e4af115507 100644 --- a/cl/coro_worker_foreign.go +++ b/cl/coro_worker_foreign.go @@ -478,6 +478,7 @@ type coroStaticForeignCallAuthority struct { type coroStaticForeignCallAuthorization struct { mode coroForeignCallMode reentry coro.ReentryClass + exec coro.ExecFlags } // authorize accepts exactly one of the legacy worker certificate and the @@ -562,6 +563,7 @@ func (a coroStaticForeignCallAuthority) authorize( } return coroStaticForeignCallAuthorization{ mode: coroForeignCallModeWorker, reentry: coro.ReentryNone, + exec: coro.BlockForeign | coro.IRQUnsafe, }, nil } @@ -607,6 +609,8 @@ func (a coroStaticForeignCallAuthority) authorize( } } contract := universeCallable.Contract + exec := coro.BlockForeign | coro.IRQUnsafe | + coro.CallableContractExecConstraints(contract) if contract.Progress != coro.ProgressMayBlock { return reject, fmt.Errorf( "callable progress %q does not authorize synchronous blocking foreign lowering; require %q", @@ -628,6 +632,7 @@ func (a coroStaticForeignCallAuthority) authorize( case coro.ReentryNone: return coroStaticForeignCallAuthorization{ mode: coroForeignCallModeWorker, reentry: contract.Reentry, + exec: exec, }, nil case coro.ReentryManagedCallback: // A synchronous managed callback needs the parent M's native @@ -635,6 +640,7 @@ func (a coroStaticForeignCallAuthority) authorize( // itself is otherwise thread-independent. return coroStaticForeignCallAuthorization{ mode: coroForeignCallModeSameM, reentry: contract.Reentry, + exec: exec, }, nil } case coro.AffinityCallerThread: @@ -642,6 +648,7 @@ func (a coroStaticForeignCallAuthority) authorize( case coro.ReentryNone, coro.ReentryManagedCallback: return coroStaticForeignCallAuthorization{ mode: coroForeignCallModeSameM, reentry: contract.Reentry, + exec: exec, }, nil } default: @@ -881,7 +888,7 @@ func validateCoroWorkerForeignCallWithAuthority( } mode := authorization.mode if targetPlan.External != coro.ExternalUnknownForeign || targetPlan.Emission != coro.EmitExternal || - targetPlan.Effect != coro.NoSuspend || targetPlan.Exec != coro.BlockForeign|coro.IRQUnsafe { + targetPlan.Effect != coro.NoSuspend || targetPlan.Exec != authorization.exec { return shape, true, fmt.Errorf( "target %q is not an exact blocking foreign declaration (external=%s emission=%s effect=%s exec=%s)", targetPlan.ID, targetPlan.External, targetPlan.Emission, targetPlan.Effect, targetPlan.Exec, @@ -1140,8 +1147,8 @@ func (p *context) coroWorkerForeignThunk(shape coroWorkerForeignCallShape, targe key := framedEmissionKey( "cl-coro-worker-foreign-thunk-v1", targetName, - structuralEmissionABITypeKey(shape.calleeType), - structuralEmissionABITypeKey(shape.signature), + p.cachedStrictEmissionABITypeKey(shape.calleeType), + p.cachedStrictEmissionABITypeKey(shape.signature), strconv.Itoa(p.prog.PointerSize()), ) name := coroWorkerForeignThunkPrefixV1 + emissionDigest(key) diff --git a/cl/coro_worker_result_projection.go b/cl/coro_worker_result_projection.go index 975421ec4e..d577bf5200 100644 --- a/cl/coro_worker_result_projection.go +++ b/cl/coro_worker_result_projection.go @@ -413,7 +413,7 @@ func (u *EmissionUniverse) freezeCoroWorkerResultProjectionCertificates() error certificate.id = framedEmissionKey( "llgo-coro-worker-result-projection-v1", identity, - structuralGoLinknameABITypeKey(fn.Signature), + u.emissionTypeKeys.goLinknameABI(fn.Signature), projection.canonical, ) u.workerResultProjections[fn] = certificate diff --git a/cl/coro_worker_syscall_capability.go b/cl/coro_worker_syscall_capability.go index 055df793d9..1b1bffb3d5 100644 --- a/cl/coro_worker_syscall_capability.go +++ b/cl/coro_worker_syscall_capability.go @@ -228,7 +228,7 @@ func (u *EmissionUniverse) aliasPatchedWorkerAddressTrampolines() error { name, ) } - if structuralGoLinknameABITypeKey(original.Signature) != structuralGoLinknameABITypeKey(alternate.Signature) { + if u.emissionTypeKeys.goLinknameABI(original.Signature) != u.emissionTypeKeys.goLinknameABI(alternate.Signature) { return fmt.Errorf("prepare emission universe: patched workeraddr target %q changes the upstream trampoline ABI", name) } if canonical := u.canonicalAlias(original); canonical == nil || canonical != original { @@ -334,7 +334,11 @@ func coroWorkerAddressFunctionIdentity(universe *EmissionUniverse, fn *ssa.Funct } signature := "" if fn.Signature != nil { - signature = structuralGoLinknameABITypeKey(fn.Signature) + if universe != nil { + signature = universe.emissionTypeKeys.goLinknameABI(fn.Signature) + } else { + signature = structuralGoLinknameABITypeKey(fn.Signature) + } } return framedEmissionKey( "llgo-coro-worker-address-function-v0", diff --git a/cl/coro_zero_sized_channel_test.go b/cl/coro_zero_sized_channel_test.go index 7f40e25fc9..36d7ce4281 100644 --- a/cl/coro_zero_sized_channel_test.go +++ b/cl/coro_zero_sized_channel_test.go @@ -55,8 +55,7 @@ func TestCoroZeroSizedChannelResultsUseKnownNonNilStorage(t *testing.T) { t.Fatalf("verify zero-sized channel lowering before CoroSplit: %v\n%s", err, module.String()) } recv := requireCoroPhysicalFunction(t, module, "foo.Recv").String() - if !strings.Contains(recv, "github.com/goplus/llgo/runtime/internal/runtime.CoroChanTryRecv") || - !strings.Contains(recv, "@"+coroChanRecvParkHookV1) { + if !strings.Contains(recv, "@"+coroChanRecvTryParkHookV2) { t.Fatalf("direct zero-sized receive did not use coroutine channel lowering:\n%s", recv) } selected := requireCoroPhysicalFunction(t, module, "foo.Select").String() @@ -117,6 +116,7 @@ func compileCoroZeroSizedChannelFixture(t *testing.T) (llssa.Program, llssa.Pack EmissionUniverse: ssaUniverse, FunctionIDs: functionIDs, MaxPlainInstructions: -1, + ClassifyLocalBody: universe.CoroLocalBodyFacts, }) if err != nil { program.Dispose() diff --git a/cl/emission_abi_demand_test.go b/cl/emission_abi_demand_test.go index ee1c8186a5..fea72dffa5 100644 --- a/cl/emission_abi_demand_test.go +++ b/cl/emission_abi_demand_test.go @@ -1354,7 +1354,7 @@ func TestEmissionIntrinsicOperandPolicyCoversRegistry(t *testing.T) { add(emissionIntrinsicFirstValue, "alloca", "allocCStr", "allocaCStr", "allocaCStrs", "string", "stringData", "_Cfunc_CString", "_Cfunc_CBytes", "_Cfunc_GoString", - "_Cfunc__CMalloc", "_cgo_runtime_cgocall") + "_Cfunc__CMalloc", "_cgo_runtime_cgocall", "cgoUse", "cgoKeepAlive") add(emissionIntrinsicFirstTwoValues, "advance", "index", "sigsetjmp", "siglongjmp", "_Cfunc_GoStringN", "_Cfunc_GoBytes") diff --git a/cl/emission_call_roots.go b/cl/emission_call_roots.go index d38e7fe69b..5cc7ffceb7 100644 --- a/cl/emission_call_roots.go +++ b/cl/emission_call_roots.go @@ -73,7 +73,7 @@ func emissionIntrinsicPolicy(instruction int) (emissionIntrinsicOperandPolicy, e case llgoAlloca, llgoAllocaCStr, llgoAllocCStr, llgoAllocaCStrs, llgoString, llgoStringData, llgoCgoCString, llgoCgoCBytes, llgoCgoGoString, llgoCgoCMalloc, - llgoCgoCgocall: + llgoCgoCgocall, llgoCgoUse, llgoCgoKeepAlive: return emissionIntrinsicFirstValue, nil case llgoPyList, llgoPyTuple: diff --git a/cl/emission_lowered_call_test.go b/cl/emission_lowered_call_test.go index 33840b0401..38bf20776c 100644 --- a/cl/emission_lowered_call_test.go +++ b/cl/emission_lowered_call_test.go @@ -21,6 +21,7 @@ package cl import ( "go/ast" "go/token" + "slices" "strings" "testing" @@ -188,7 +189,6 @@ func Use(m map[int]int, key int, value I) { delete(m, key) value.M() } - `) testProg.ssa.Build() universe, owner := newEmissionABIDemandTestUniverse(testProg, pkg) @@ -212,6 +212,36 @@ func Use(m map[int]int, key int, value I) { } } +func TestLoweredRuntimeHelpersCgoC2UsesStaticErrnoItab(t *testing.T) { + testProg := newEmissionTestProgram() + pkg := testProg.addPackage(t, "example.com/emission/cgoc2helpers", `package cgoc2helpers +var _cgo_demo uintptr +func _cgo_runtime_cgocall(fn uintptr, arg uintptr) int32 +func _C2func_demo() (int, error) { + _cgo_runtime_cgocall(_cgo_demo, 0) + return 0, nil +} +`) + testProg.ssa.Build() + prog := newLLSSAProg(t) + defer prog.Dispose() + universe, owner := newEmissionABIDemandTestUniverse(testProg, pkg) + universe.prog = prog + fn := pkg.ssa.Func("_C2func_demo") + ctx, err := universe.functionABIContext(fn, owner) + if err != nil { + t.Fatal(err) + } + call := findStaticCall(t, fn, "_cgo_runtime_cgocall") + helpers := universe.loweredRuntimeHelpers(ctx, call) + if !slices.Contains(helpers, "AllocU") { + t.Fatalf("C2 errno return helpers = %v, want AllocU for the nil error slot", helpers) + } + if slices.Contains(helpers, "NewItab") { + t.Fatalf("C2 errno return helpers = %v, static errno itab must not retain NewItab", helpers) + } +} + func TestLoweredRuntimeHelpersIncludeDynamicFunctionNilEdge(t *testing.T) { testProg := newEmissionTestProgram() pkg := testProg.addPackage(t, "example.com/emission/dynamicnil", `package dynamicnil diff --git a/cl/emission_runtime_abi_test.go b/cl/emission_runtime_abi_test.go index be9baa61ff..3c65c14d03 100644 --- a/cl/emission_runtime_abi_test.go +++ b/cl/emission_runtime_abi_test.go @@ -226,9 +226,10 @@ func Use() {} func TestEmissionUniverseCoroChannelRetainsPlainAndPhysicalHelpers(t *testing.T) { testProg := newEmissionTestProgram() runtimePkg := testProg.addPackage(t, llssa.PkgRuntime, `package runtime -func CoroChanTrySend(ch chan int, value *int, size int) bool { return false } -func CoroChanTryRecv(ch chan int, value *int, size int) (bool, bool) { return false, false } +func CoroChanTrySend(task *byte, ch chan int, value *int, size int) bool { return false } +func CoroChanTryRecv(task *byte, ch chan int, value *int, size int) (bool, bool) { return false, false } func CoroChanTryClose(ch chan int) uint32 { return 0 } +func CoroChanTryCloseTask(task *byte, ch chan int) uint32 { return 0 } type ChanOp struct{} func CoroChanSelectTry(ops ...ChanOp) (int, bool, bool, bool) { return 0, false, false, false } func CoroChanSelectPark(ops ...ChanOp) {} @@ -270,7 +271,7 @@ func Close(ch chan int) { close(ch) } required[fn] = true } for _, helper := range []string{ - "CoroChanTrySend", "CoroChanTryRecv", "CoroChanTryClose", "CoroChanSelectTry", "CoroChanSelectPark", "CoroChanSelectResume", + "CoroChanTrySend", "CoroChanTryRecv", "CoroChanTryClose", "CoroChanTryCloseTask", "CoroChanSelectTry", "CoroChanSelectPark", "CoroChanSelectResume", "ChanSend", "ChanRecv", "ChanClose", "Select", "TrySelect", } { if fn := runtimePkg.ssa.Func(helper); fn == nil || !required[fn] { @@ -281,11 +282,14 @@ func Close(ch chan int) { close(ch) } owner string want []string }{ - {owner: "Send", want: []string{"CoroChanTrySend"}}, - {owner: "Recv", want: []string{"CoroChanTryRecv"}}, + // One-case send/receive use the frozen raw try-or-park V2 ABI. Unlike + // select and close, no separately emitted Go runtime helper remains at + // the source instruction. + {owner: "Send"}, + {owner: "Recv"}, {owner: "BlockingSelect", want: []string{"CoroChanSelectPark", "CoroChanSelectResume", "CoroChanSelectTry"}}, {owner: "NonblockingSelect", want: []string{"CoroChanSelectTry"}}, - {owner: "Close", want: []string{"CoroChanTryClose"}}, + {owner: "Close", want: []string{"CoroChanTryCloseTask"}}, } { lowered, err := universe.CoroLoweredCalls(callerPkg.ssa.Func(test.owner)) if err != nil { diff --git a/cl/emission_runtime_helpers.go b/cl/emission_runtime_helpers.go index 74e9206c36..ed64275af3 100644 --- a/cl/emission_runtime_helpers.go +++ b/cl/emission_runtime_helpers.go @@ -51,6 +51,92 @@ type coroLogicalLocalitySite struct { dispatchers []*ssa.Function } +type coroPlainAllocationRecipe uint8 + +const ( + coroPlainAllocationOrdinary coroPlainAllocationRecipe = iota + coroPlainAllocationBorrowed +) + +// coroPlainAllocationPlan is frozen with the exact source SitePlan before +// whole-program effect analysis. It prevents final code generation from +// rediscovering escape/lifetime facts and records enough proof shape to make +// owner conflicts observable during universe construction. +type coroPlainAllocationPlan struct { + recipe coroPlainAllocationRecipe + functionsVisited uint32 + parametersProven uint32 +} + +func (plan coroPlainAllocationPlan) borrowed() bool { + return plan.recipe == coroPlainAllocationBorrowed && + plan.functionsVisited != 0 +} + +func planCoroPlainAllocation(ctx *context, allocation *ssa.Alloc) coroPlainAllocationPlan { + if ctx == nil || ctx.prog == nil || ctx.emissionUniverse == nil || allocation == nil || !allocation.Heap || + allocation.Type() == nil || ctx.skipSyntheticMakeSliceAlloc(allocation) || + isEmissionVargsAlloc(ctx, allocation) { + return coroPlainAllocationPlan{} + } + if bitcast, exact := coro.ProveSSAExactScalarBitcast(allocation.Parent()); exact && + bitcast.Allocation == allocation { + return coroPlainAllocationPlan{} + } + pointer, ok := types.Unalias(ctx.patchType(allocation.Type())).Underlying().(*types.Pointer) + if !ok || ctx.prog.LocalGoTypeExceedsNativeStack(ctx.patchType(pointer.Elem())) { + return coroPlainAllocationPlan{} + } + if ctx.prog.PhysicalSizeOfGoType(ctx.patchType(pointer.Elem())) == 0 { + // Preserve the existing module-sentinel identity for a zero-sized heap + // allocation. It already has no AllocZ cost and needs no stack slot. + return coroPlainAllocationPlan{} + } + proof, exact := proveCoroBorrowedAllocation(ctx.emissionUniverse, allocation) + if !exact || proof.FunctionsVisited == 0 { + return coroPlainAllocationPlan{} + } + return coroPlainAllocationPlan{ + recipe: coroPlainAllocationBorrowed, + functionsVisited: proof.FunctionsVisited, + parametersProven: proof.ParametersProven, + } +} + +// coroBorrowedAllocationUniverse is the complete read-only authority needed by +// the interprocedural lifetime proof. Keeping this as a two-method view avoids +// turning a local allocation classifier into another whole-program plan owner. +type coroBorrowedAllocationUniverse interface { + Resolve(*ssa.Function) (*ssa.Function, bool) + FunctionBackground(*ssa.Function) (llssa.Background, bool, error) +} + +// proveCoroBorrowedAllocation admits only bodies which the frozen emission +// universe will actually compile as managed Go. Source bodies attached to C, +// Python, or LLVM intrinsic declarations are type-checking stubs rather than +// memory-semantics evidence and must never justify local storage. +func proveCoroBorrowedAllocation( + universe coroBorrowedAllocationUniverse, + allocation *ssa.Alloc, +) (coro.SSABorrowedAllocationProof, bool) { + if universe == nil { + return coro.SSABorrowedAllocationProof{Allocation: allocation}, false + } + return coro.ProveSSABorrowedAllocationWithConfig(allocation, coro.SSABorrowedAllocationConfig{ + ResolveCalleeBody: func(function *ssa.Function) (*ssa.Function, bool) { + canonical, resolved := universe.Resolve(function) + if !resolved || canonical == nil || len(canonical.Blocks) == 0 { + return nil, false + } + background, classified, err := universe.FunctionBackground(canonical) + if err != nil || !classified || background != llssa.InGo { + return nil, false + } + return canonical, true + }, + }) +} + // coroFunctionPreamblePlan freezes compiler-owned operations which have no // source SSA instruction anchor. Package-init locality guard setup is the first // such operation: it resolves one logical-G package block, then marks the @@ -65,6 +151,10 @@ type coroFunctionPreamblePlan struct { localityGuards []locality.Kind localContextEntry bool logicalCallerEntry bool + // emitsGoBody distinguishes a real frontend-emitted Go definition from a + // declaration/intrinsic stub whose SSA body exists only for policy and type + // analysis. Only the former may acquire a physical outcome-plain entry. + emitsGoBody bool } func cloneCoroFunctionPreamblePlan(plan coroFunctionPreamblePlan) coroFunctionPreamblePlan { @@ -79,7 +169,8 @@ func sameCoroFunctionPreamblePlan(first, second coroFunctionPreamblePlan) bool { slices.Equal(first.plainRuntimeHelpers, second.plainRuntimeHelpers) && slices.Equal(first.localityGuards, second.localityGuards) && first.localContextEntry == second.localContextEntry && - first.logicalCallerEntry == second.logicalCallerEntry + first.logicalCallerEntry == second.logicalCallerEntry && + first.emitsGoBody == second.emitsGoBody } func (plan coroFunctionPreamblePlan) validate() error { @@ -199,7 +290,7 @@ func (builder coroProgramIRBuilder) materializeFunctionPreamble( if u == nil || u.coroProgramIR == nil || ctx == nil || ownerFn == nil || ownerPkg == nil { return fmt.Errorf("prepare emission universe: function preamble requires one exact program IR, builder, owner, and function") } - plan := coroFunctionPreamblePlan{} + plan := coroFunctionPreamblePlan{emitsGoBody: ftype == goFunc} needsLocalContext := u.prog != nil && u.prog.NeedsLocalContext() if u.prog != nil && u.logicalLocality { needsLocalContext = u.prog.NeedsLogicalLocalContext() @@ -297,6 +388,9 @@ func (builder coroProgramIRBuilder) materializeLoweredRuntimeHelpers(ctx *contex } sitePlan := coroEmissionSitePlan{} if u.prog != nil { + if allocation, ok := instr.(*ssa.Alloc); ok { + sitePlan.plainAllocation = planCoroPlainAllocation(ctx, allocation) + } managed, err := u.classifyCoroRuntimeHelpers(ctx, shape, instr) if err != nil { return fmt.Errorf("prepare emission universe: function %q logical locality site: %w", ownerFn.Name(), err) @@ -308,6 +402,12 @@ func (builder coroProgramIRBuilder) materializeLoweredRuntimeHelpers(ctx *contex }) } sitePlan.plainRuntimeHelpers = u.classifyPlainRuntimeHelpers(ctx, instr, managed) + if sitePlan.plainAllocation.borrowed() { + sitePlan.plainRuntimeHelpers = slices.DeleteFunc( + sitePlan.plainRuntimeHelpers, + func(helper string) bool { return helper == "AllocZ" }, + ) + } preamble, err := u.coroProgramIR.functionPreambleDuringFreeze(ownerFn, ownerPkg) if err != nil { return fmt.Errorf("prepare emission universe: function %q preamble lookup: %w", ownerFn.Name(), err) @@ -447,7 +547,7 @@ func coroCompilerRawPlainLoweredRuntimeHelper(u *EmissionUniverse, helper string return false } switch helper { - case "CoroChanTrySend", "CoroChanTryRecv", "CoroChanTryClose", + case "CoroChanTrySend", "CoroChanTryRecv", "CoroChanTryCloseTask", "CoroChanSelectTry", "CoroChanSelectPark", "CoroChanSelectResume": return true default: @@ -485,6 +585,8 @@ func coroLoweredCallExplicitStatusElided(instr ssa.Instruction, helper string) b // helper name emitted by any other lowering remains an ordinary managed edge. func coroCompilerElidesImplicitFaultRuntimeHelper(instr ssa.Instruction, helper string) bool { switch instr.(type) { + case *ssa.FieldAddr: + return helper == "AssertNilDeref" case *ssa.Index, *ssa.IndexAddr: return helper == "CheckIndexRange" || helper == "AssertNilDeref" default: @@ -712,7 +814,9 @@ func (u *EmissionUniverse) classifyCoroRuntimeHelpers( case *ssa.UnOp: switch v.Op { case token.ARROW: - add("CoroChanTryRecv") + // Physical coroutine lowering owns the raw try-or-park V2 hook + // through the frozen bootstrap ABI. No Go runtime helper call remains + // at this source instruction. case token.MUL: free, _ := v.X.(*ssa.FreeVar) elidedZeroSizedFreeVar := u.closureEnvironments.elidesZeroSizedFreeVar(v.Parent(), free) @@ -912,7 +1016,8 @@ func (u *EmissionUniverse) classifyCoroRuntimeHelpers( add("Panic") } case *ssa.Send: - add("CoroChanTrySend") + // See receive above: the raw V2 hook, not a separately emitted Go + // nonblocking helper, owns the complete one-case transaction. case *ssa.Call: if emissionCallNeedsManagedCoroResultSlot(ctx, v) { add(coroManagedFrameSlotAllocZCall) @@ -937,11 +1042,13 @@ func (u *EmissionUniverse) classifyCoroRuntimeHelpers( case intrinsic && opcode == llgoCgoCgocall && v.Parent() != nil && isCgoC2func(v.Parent().Name()): // The exact generated C2 worker transaction resumes in the Go - // wrapper and constructs its (result, error) pair there. Attach - // that synthetic interface construction to the cgocall source - // site so helper closure, physical emission, and observation - // share one immutable recipe. - add("AllocU", "NewItab") + // wrapper and constructs its (result, error) pair there. The nil + // error slot owns AllocU. The non-nil syscall.Errno conversion has + // one statically known concrete type, so Builder.MakeInterface now + // emits an immutable itab and no runtime NewItab call. Attach only + // the helper that physical emission actually performs to this + // cgocall source site. + add("AllocU") case intrinsic && opcode == llgoDeferData: // Builder.DeferData replaces the compiler declaration with an // ordinary runtime.GetThreadDefer call. @@ -1563,9 +1670,6 @@ func (u *EmissionUniverse) makeInterfaceRuntimeHelpers(ctx *context, makeInterfa if !u.makeInterfaceEmitsABIType(makeInterface, ctx) { return } - if interfaceIsNonEmpty(ctx.patchType(makeInterface.Type())) { - add("NewItab") - } // Helper planning must remain valid for report/identity universes whose // LLSSA program intentionally has no runtime package. The interface data // representation and the large/zero dereference rules depend only on the @@ -1577,7 +1681,11 @@ func (u *EmissionUniverse) makeInterfaceRuntimeHelpers(ctx *context, makeInterfa // only at the source *types.Signature would incorrectly classify it as one // direct pointer and omit the AllocU call that code generation emits. physical := u.physicalFunctionABIType(ctx, makeInterface.X.Type()) - if !emissionDirectIfaceType(physical) { + if target := ctx.patchType(makeInterface.Type()); interfaceIsNonEmpty(target) && + !llssa.CanBuildStaticItab(target, physical) { + add("NewItab") + } + if !emissionDirectIfaceType(physical) && !makeInterfaceUsesConstantBacking(makeInterface) { add("AllocU") } if unop, ok := makeInterface.X.(*ssa.UnOp); ok && unop.Op == token.MUL && @@ -1591,6 +1699,14 @@ func (u *EmissionUniverse) makeInterfaceRuntimeHelpers(ctx *context, makeInterfa } } +func makeInterfaceUsesConstantBacking(makeInterface *ssa.MakeInterface) bool { + if makeInterface == nil || isUntypedNilConst(makeInterface.X) { + return false + } + _, ok := makeInterface.X.(*ssa.Const) + return ok +} + func emissionLargeOrZeroInterfaceDeref(typ types.Type, pointerSize int) bool { raw := types.Unalias(typ) if raw == nil { @@ -1758,7 +1874,7 @@ func (u *EmissionUniverse) builtinRuntimeHelpers(ctx *context, call *ssa.CallCom case "copy": add("SliceCopy") case "close": - add("CoroChanTryClose") + add("CoroChanTryCloseTask") case "recover": add("Recover") case "panic": diff --git a/cl/emission_type_graph.go b/cl/emission_type_graph.go index 68a87bb86d..4626839c4f 100644 --- a/cl/emission_type_graph.go +++ b/cl/emission_type_graph.go @@ -18,6 +18,7 @@ package cl import ( "strconv" + "sync" llssa "github.com/goplus/llgo/ssa" "go/types" @@ -26,8 +27,136 @@ import ( const ( emissionGoLinknameTypeGraphSchema = "llgo.emission.go-linkname-type-graph.v1" emissionGoLinknameTypeGraphKeyPrefix = "graph-sha256-v1:" + emissionStrictTypeGraphSchema = "llgo.emission.strict-type-graph.v1" + emissionStrictTypeGraphKeyPrefix = "strict-graph-sha256-v1:" + emissionStrictABITypeGraphSchema = "llgo.emission.strict-abi-type-graph.v1" + emissionStrictABITypeGraphKeyPrefix = "strict-abi-graph-sha256-v1:" + emissionIdentityFreeTypeGraphSchema = "llgo.emission.identity-free-abi-type-graph.v1" + emissionIdentityFreeTypeGraphPrefix = "identity-free-abi-graph-sha256-v1:" ) +type emissionTypeGraphOptions struct { + omitTupleNames bool + expandNamed bool + omitStructFieldMetadata bool +} + +type emissionTypeKeyMode uint8 + +const ( + emissionTypeKeyStrict emissionTypeKeyMode = iota + emissionTypeKeyStrictABI + emissionTypeKeyGoLinknameABI + emissionTypeKeyIdentityFreeABI +) + +type emissionTypeKeyCacheKey struct { + mode emissionTypeKeyMode + typ types.Type +} + +// emissionTypeKeyCache is a self-contained, compilation-lifetime cache. It is +// deliberately narrower than EmissionUniverse: structural type identity is a +// reusable service, not another consumer of the whole-program plan authority. +type emissionTypeKeyCache struct { + mu sync.RWMutex + values map[emissionTypeKeyCacheKey]string +} + +func structuralEmissionTypeKeyForMode(mode emissionTypeKeyMode, typ types.Type) string { + switch mode { + case emissionTypeKeyStrict: + return structuralEmissionTypeKey(typ) + case emissionTypeKeyStrictABI: + return structuralEmissionABITypeKey(typ) + case emissionTypeKeyGoLinknameABI: + return structuralGoLinknameABITypeKey(typ) + case emissionTypeKeyIdentityFreeABI: + return structuralNamedIdentityFreeABITypeKey(typ) + default: + panic("cl: unknown structural emission type-key mode") + } +} + +// key retains a root digest only inside the cache which owns its go/types +// graph. This avoids both repeated full-graph walks and a process-global cache +// that would keep every prior compiler request alive. The computation remains +// outside the mutex so independent package emitters do not serialize on a +// large first-use graph. +func (c *emissionTypeKeyCache) key( + mode emissionTypeKeyMode, + typ types.Type, +) string { + if c == nil { + return structuralEmissionTypeKeyForMode(mode, typ) + } + if typ != nil { + typ = types.Unalias(typ) + } + key := emissionTypeKeyCacheKey{mode: mode, typ: typ} + c.mu.RLock() + value, ok := c.values[key] + c.mu.RUnlock() + if ok { + return value + } + value = structuralEmissionTypeKeyForMode(mode, typ) + c.mu.Lock() + if c.values == nil { + c.values = make(map[emissionTypeKeyCacheKey]string) + } + if existing, exists := c.values[key]; exists { + value = existing + } else { + c.values[key] = value + } + c.mu.Unlock() + return value +} + +func (c *emissionTypeKeyCache) strict(typ types.Type) string { + return c.key(emissionTypeKeyStrict, typ) +} + +func (c *emissionTypeKeyCache) strictABI(typ types.Type) string { + return c.key(emissionTypeKeyStrictABI, typ) +} + +func (c *emissionTypeKeyCache) goLinknameABI(typ types.Type) string { + return c.key(emissionTypeKeyGoLinknameABI, typ) +} + +func (c *emissionTypeKeyCache) cFunctionABI(typ types.Type) string { + return c.key(emissionTypeKeyIdentityFreeABI, typ) +} + +func (p *context) cachedStrictEmissionTypeKey(typ types.Type) string { + if p != nil { + if universe := p.immutableEmissionUniverse(); universe != nil { + return universe.emissionTypeKeys.strict(typ) + } + } + return structuralEmissionTypeKey(typ) +} + +func (p *context) cachedStrictEmissionABITypeKey(typ types.Type) string { + if p != nil { + if universe := p.immutableEmissionUniverse(); universe != nil { + return universe.emissionTypeKeys.strictABI(typ) + } + } + return structuralEmissionABITypeKey(typ) +} + +func (p *context) cachedCFunctionABITypeKey(typ types.Type) string { + if p != nil { + if universe := p.immutableEmissionUniverse(); universe != nil { + return universe.emissionTypeKeys.cFunctionABI(typ) + } + } + return structuralCFunctionABITypeKey(typ) +} + // emissionTypeGraphToken retains the exact ordered scalar/child sequence of // one structural ABI node. Child indexes never enter the final digest // directly: acyclic children contribute their Merkle digest, while edges @@ -43,23 +172,42 @@ type emissionTypeGraphNode struct { } // emissionTypeGraphBuilder turns the reachable go/types value graph into a -// compact intermediate graph. Named identity is deliberately transparent for -// go:linkname ABI pairing: a named value and its physical anonymous +// compact intermediate graph. Its options preserve the distinct identity +// policies used by ordinary emission, managed ABI matching, and C/go:linkname +// boundaries. In identity-free modes a named value and its physical anonymous // underlying type share one node. Pointer sharing outside recursive SCCs is -// not semantic and is removed later by Merkle hashing. +// never semantic and is removed later by Merkle hashing. type emissionTypeGraphBuilder struct { - ids map[types.Type]int - nodes []emissionTypeGraphNode + ids map[types.Type]int + nodes []emissionTypeGraphNode + options emissionTypeGraphOptions } func compactStructuralGoLinknameABITypeKey(typ types.Type) string { if signature, ok := types.Unalias(typ).(*types.Signature); ok { typ = normalizeGoLinknameABISignature(signature) } - builder := emissionTypeGraphBuilder{ids: make(map[types.Type]int)} + return compactStructuralTypeGraphKey( + typ, + emissionGoLinknameTypeGraphSchema, + emissionGoLinknameTypeGraphKeyPrefix, + emissionTypeGraphOptions{ + omitTupleNames: true, + expandNamed: true, + omitStructFieldMetadata: true, + }, + ) +} + +func compactStructuralTypeGraphKey( + typ types.Type, + schema, prefix string, + options emissionTypeGraphOptions, +) string { + builder := emissionTypeGraphBuilder{ids: make(map[types.Type]int), options: options} root := builder.node(typ) - digester := newEmissionTypeGraphDigester(builder.nodes) - return emissionGoLinknameTypeGraphKeyPrefix + digester.digestNode(root) + digester := newEmissionTypeGraphDigester(builder.nodes, schema) + return prefix + digester.digestNode(root) } func (b *emissionTypeGraphBuilder) node(typ types.Type) int { @@ -70,7 +218,7 @@ func (b *emissionTypeGraphBuilder) node(typ types.Type) int { return b.reserveAndFill(nil, nil) } typ = types.Unalias(typ) - if named, ok := typ.(*types.Named); ok { + if named, ok := typ.(*types.Named); ok && b.options.expandNamed { if index, exists := b.ids[named]; exists { return index } @@ -91,7 +239,7 @@ func (b *emissionTypeGraphBuilder) reserveAndFill(identity, body types.Type) int index := len(b.nodes) b.nodes = append(b.nodes, emissionTypeGraphNode{}) b.ids[identity] = index - if named, ok := identity.(*types.Named); ok { + if named, ok := identity.(*types.Named); ok && b.options.expandNamed { b.ids[types.Unalias(named.Underlying())] = index } b.nodes[index] = b.describe(body) @@ -138,20 +286,52 @@ func (b *emissionTypeGraphBuilder) describe(typ types.Type) emissionTypeGraphNod text("chan", strconv.Itoa(int(typ.Dir()))) edge(typ.Elem()) case *types.Named: - // node removes named wrappers before describe. Keep this defensive case - // fail-closed and deterministic if go/types ever exposes a named - // underlying node through a new representation. - text("named-underlying") - edge(typ.Underlying()) + if b.options.expandNamed { + // node removes named wrappers before describe. Keep this defensive + // case fail-closed and deterministic if go/types ever exposes a + // named underlying node through a new representation. + text("named-underlying") + edge(typ.Underlying()) + break + } + object := typ.Obj() + packageLevel := false + text("named") + if object != nil { + text(pkgKey(object.Pkg()), object.Name()) + packageLevel = object.Pkg() != nil && object.Parent() == object.Pkg().Scope() + } + if arguments := typ.TypeArgs(); arguments != nil { + for index := 0; index < arguments.Len(); index++ { + edge(arguments.At(index)) + } + } + if !packageLevel { + text("local-underlying") + edge(typ.Underlying()) + } case *types.Struct: text("struct", strconv.Itoa(typ.NumFields())) for index := 0; index < typ.NumFields(); index++ { - edge(typ.Field(index).Type()) + field := typ.Field(index) + if !b.options.omitStructFieldMetadata { + text( + pkgKey(field.Pkg()), + field.Name(), + strconv.FormatBool(field.Embedded()), + typ.Tag(index), + ) + } + edge(field.Type()) } case *types.Tuple: text("tuple", strconv.Itoa(typ.Len())) for index := 0; index < typ.Len(); index++ { - edge(typ.At(index).Type()) + variable := typ.At(index) + if !b.options.omitTupleNames { + text(pkgKey(variable.Pkg()), variable.Name()) + } + edge(variable.Type()) } case *types.Signature: text("signature", strconv.FormatBool(typ.Variadic())) @@ -203,6 +383,7 @@ func (b *emissionTypeGraphBuilder) describe(typ types.Type) emissionTypeGraphNod type emissionTypeGraphDigester struct { nodes []emissionTypeGraphNode + schema string component []int components [][]int cyclic []bool @@ -210,7 +391,7 @@ type emissionTypeGraphDigester struct { active map[int]bool } -func newEmissionTypeGraphDigester(nodes []emissionTypeGraphNode) *emissionTypeGraphDigester { +func newEmissionTypeGraphDigester(nodes []emissionTypeGraphNode, schema string) *emissionTypeGraphDigester { component, components := emissionTypeGraphComponents(nodes) cyclic := make([]bool, len(components)) for index, members := range components { @@ -227,7 +408,7 @@ func newEmissionTypeGraphDigester(nodes []emissionTypeGraphNode) *emissionTypeGr } } return &emissionTypeGraphDigester{ - nodes: nodes, component: component, components: components, cyclic: cyclic, + nodes: nodes, schema: schema, component: component, components: components, cyclic: cyclic, digests: make(map[int]string), active: make(map[int]bool), } } @@ -250,7 +431,7 @@ func (d *emissionTypeGraphDigester) digestNode(node int) string { d.appendCyclicComponent(&fields, component, node) } delete(d.active, component) - digest := emissionDigest(framedEmissionKey(emissionGoLinknameTypeGraphSchema, framedEmissionKey(fields...))) + digest := emissionDigest(framedEmissionKey(d.schema, framedEmissionKey(fields...))) d.digests[node] = digest return digest } diff --git a/cl/emission_type_graph_test.go b/cl/emission_type_graph_test.go index 817b4e3830..4ccc32d24f 100644 --- a/cl/emission_type_graph_test.go +++ b/cl/emission_type_graph_test.go @@ -42,6 +42,106 @@ func TestStructuralGoLinknameABITypeKeyCompactsSharedTypeDAG(t *testing.T) { } } +func TestStructuralEmissionTypeGraphsCompactSharedDAGAndPreserveMetadataModes(t *testing.T) { + shared := testSharedAnonymousEmissionType(14) + strict := structuralEmissionTypeKey(shared) + strictABI := structuralEmissionABITypeKey(shared) + identityFree := structuralNamedIdentityFreeABITypeKey(shared) + for name, test := range map[string]struct { + key string + prefix string + }{ + "strict": {key: strict, prefix: emissionStrictTypeGraphKeyPrefix}, + "strict ABI": {key: strictABI, prefix: emissionStrictABITypeGraphKeyPrefix}, + "identity-free": {key: identityFree, prefix: emissionIdentityFreeTypeGraphPrefix}, + } { + if want := len(test.prefix) + 64; len(test.key) != want { + t.Errorf("%s compact key length = %d, want %d", name, len(test.key), want) + } + } + + paramsX := types.NewTuple(types.NewParam(token.NoPos, nil, "x", shared)) + paramsY := types.NewTuple(types.NewParam(token.NoPos, nil, "y", shared)) + if structuralEmissionTypeKey(paramsX) == structuralEmissionTypeKey(paramsY) { + t.Fatal("strict emission type key erased tuple variable names") + } + if structuralEmissionABITypeKey(paramsX) != structuralEmissionABITypeKey(paramsY) { + t.Fatal("strict ABI type key retained tuple variable names") + } + + fieldX := types.NewStruct([]*types.Var{ + types.NewField(token.NoPos, nil, "x", types.Typ[types.Uintptr], false), + }, []string{"json:\"x\""}) + fieldY := types.NewStruct([]*types.Var{ + types.NewField(token.NoPos, nil, "y", types.Typ[types.Uintptr], false), + }, []string{"json:\"y\""}) + if structuralEmissionABITypeKey(fieldX) == structuralEmissionABITypeKey(fieldY) { + t.Fatal("strict ABI type key erased struct field metadata") + } + if structuralNamedIdentityFreeABITypeKey(fieldX) != structuralNamedIdentityFreeABITypeKey(fieldY) { + t.Fatal("identity-free ABI type key retained struct field metadata") + } +} + +func TestStructuralEmissionABITypeGraphIgnoresIncidentalPointerSharing(t *testing.T) { + child := func() types.Type { + return types.NewStruct([]*types.Var{ + types.NewField(token.NoPos, nil, "value", types.Typ[types.Uintptr], false), + }, nil) + } + sharedChild := child() + shared := types.NewStruct([]*types.Var{ + types.NewField(token.NoPos, nil, "left", sharedChild, false), + types.NewField(token.NoPos, nil, "right", sharedChild, false), + }, nil) + duplicated := types.NewStruct([]*types.Var{ + types.NewField(token.NoPos, nil, "left", child(), false), + types.NewField(token.NoPos, nil, "right", child(), false), + }, nil) + if structuralEmissionABITypeKey(shared) != structuralEmissionABITypeKey(duplicated) { + t.Fatal("incidental go/types pointer sharing changed the strict ABI key") + } +} + +func TestEmissionUniverseStructuralTypeKeyCacheIsSessionLocalAndModeSeparated(t *testing.T) { + typ := testSharedAnonymousEmissionType(12) + alias := types.NewAlias(types.NewTypeName(token.NoPos, nil, "Alias", nil), typ) + universe := new(EmissionUniverse) + tests := []struct { + mode emissionTypeKeyMode + want string + }{ + {mode: emissionTypeKeyStrict, want: structuralEmissionTypeKey(typ)}, + {mode: emissionTypeKeyStrictABI, want: structuralEmissionABITypeKey(typ)}, + {mode: emissionTypeKeyGoLinknameABI, want: structuralGoLinknameABITypeKey(typ)}, + {mode: emissionTypeKeyIdentityFreeABI, want: structuralNamedIdentityFreeABITypeKey(typ)}, + } + for _, test := range tests { + if got := universe.emissionTypeKeys.key(test.mode, typ); got != test.want { + t.Fatalf("cached mode %d key = %q, want %q", test.mode, got, test.want) + } + if got := universe.emissionTypeKeys.key(test.mode, alias); got != test.want { + t.Fatalf("cached alias mode %d key = %q, want %q", test.mode, got, test.want) + } + } + universe.emissionTypeKeys.mu.RLock() + entries := len(universe.emissionTypeKeys.values) + universe.emissionTypeKeys.mu.RUnlock() + if entries != len(tests) { + t.Fatalf("session cache entries = %d, want one canonical entry per mode (%d)", entries, len(tests)) + } + other := new(EmissionUniverse) + if got := other.emissionTypeKeys.strictABI(typ); got != tests[1].want { + t.Fatalf("independent session key = %q, want %q", got, tests[1].want) + } + other.emissionTypeKeys.mu.RLock() + otherEntries := len(other.emissionTypeKeys.values) + other.emissionTypeKeys.mu.RUnlock() + if otherEntries != 1 { + t.Fatalf("independent session cache entries = %d, want 1", otherEntries) + } +} + func TestStructuralGoLinknameABITypeKeyIgnoresIncidentalPointerSharing(t *testing.T) { child := func() types.Type { return types.NewStruct([]*types.Var{ @@ -121,6 +221,37 @@ func BenchmarkStructuralGoLinknameABITypeKeySharedDAG(b *testing.B) { } } +func BenchmarkStructuralEmissionABITypeKeySharedDAG(b *testing.B) { + typ := testSharedAnonymousEmissionType(18) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + _ = structuralEmissionABITypeKey(typ) + } +} + +func BenchmarkEmissionUniverseCachedStructuralEmissionABITypeKeySharedDAG(b *testing.B) { + typ := testSharedAnonymousEmissionType(18) + universe := new(EmissionUniverse) + _ = universe.emissionTypeKeys.strictABI(typ) + b.ReportAllocs() + b.ResetTimer() + for range b.N { + _ = universe.emissionTypeKeys.strictABI(typ) + } +} + +func testSharedAnonymousEmissionType(depth int) types.Type { + current := types.Type(types.Typ[types.Uintptr]) + for range depth { + current = types.NewStruct([]*types.Var{ + types.NewField(token.NoPos, nil, "left", current, false), + types.NewField(token.NoPos, nil, "right", current, false), + }, nil) + } + return current +} + func testSharedGoLinknameType(path string, depth int, leaf types.Type) types.Type { pkg := types.NewPackage(path, "shared") current := leaf diff --git a/cl/emission_universe.go b/cl/emission_universe.go index 3ec5acc1e3..f4a1cd289f 100644 --- a/cl/emission_universe.go +++ b/cl/emission_universe.go @@ -243,6 +243,13 @@ type EmissionUniverse struct { globalPhysicalGroups map[string]CoroGlobalPhysicalIdentity globalPhysicalSeen map[*ssa.Global]none + // emissionTypeKeys memoizes structural digests only for the lifetime of + // this frozen compilation universe. A package-level cache would retain + // complete go/types graphs across compiler requests; rebuilding at every + // use instead made managed-interface and cgo emission repeat the same graph + // walk many times. + emissionTypeKeys emissionTypeKeyCache + effectiveTypeMu sync.Mutex effectiveTypeCacheReady bool effectiveTypes map[emissionEffectiveTypeKey]types.Type @@ -627,6 +634,11 @@ type coroFrozenCallSitePlan struct { type coroEmissionSitePlan struct { managedRuntimeHelpers []coroPlannedRuntimeHelper plainRuntimeHelpers []string + // plainAllocation freezes the stronger interprocedural storage decision for + // a synchronous Go body. Managed coroutine emission keeps its independent + // physical plan because the same allocation then belongs in a CoroSplit + // frame rather than a native entry stack. + plainAllocation coroPlainAllocationPlan // localityDispatchers are compiler-materialized func() values passed to // EnsureLogicalLocalInitializer at this source instruction. They are // demand references rather than owner-local calls; the runtime helper owns @@ -2260,6 +2272,11 @@ func (u *EmissionUniverse) classifyCoroIntrinsicCallSite( return CoroIntrinsicCallUnsupported, true, err } return CoroIntrinsicCallInlineNoSuspend, true, nil + case llgoCgoUse, llgoCgoKeepAlive: + if err := verifyCoroGeneratedTouchCall(u.prog, opcode, direct); err != nil { + return CoroIntrinsicCallUnsupported, true, err + } + return CoroIntrinsicCallInlineNoSuspend, true, nil case llgoCoroYield: if err := verifyCoroExactVoidIntrinsicCallSite(direct, "llgo.coroYield"); err != nil { return CoroIntrinsicCallUnsupported, true, err @@ -2469,6 +2486,61 @@ func verifyCoroCgoCheckPointerCall(call *ssa.Call) error { return nil } +func verifyCoroGeneratedTouchCall(prog llssa.Program, opcode int, call *ssa.Call) error { + name := "_Cgo_use" + if opcode == llgoCgoKeepAlive { + name = "_Cgo_keepalive" + } else if opcode != llgoCgoUse { + return fmt.Errorf("generated cgo liveness call has unknown opcode %d", opcode) + } + const shape = "func(any) guarded by runtime.cgoAlwaysFalse" + if prog == nil || call == nil || call.Common() == nil || + call.Common().IsInvoke() || call.Common().Method != nil || + len(call.Common().Args) != 1 { + return fmt.Errorf("%s requires one exact direct %s call", name, shape) + } + signature := call.Common().Signature() + if signature == nil || signature.Recv() != nil || signature.Variadic() || + signature.Params() == nil || signature.Params().Len() != 1 || + signature.Results() != nil && signature.Results().Len() != 0 || + !types.Identical(signature.Params().At(0).Type(), call.Common().Args[0].Type()) { + return fmt.Errorf("%s call %q requires the exact %s shape", name, call.String(), shape) + } + parameter, ok := types.Unalias(signature.Params().At(0).Type()).Underlying().(*types.Interface) + if !ok || !parameter.Empty() { + return fmt.Errorf("%s call %q requires the exact %s shape", name, call.String(), shape) + } + block := call.Block() + if block == nil || len(block.Preds) != 1 { + return fmt.Errorf("%s call %q is not reached through one exact false-global guard", name, call.String()) + } + pred := block.Preds[0] + if pred == nil || len(pred.Succs) != 2 || pred.Succs[0] != block || len(pred.Instrs) == 0 { + return fmt.Errorf("%s call %q is not the true edge of one exact false-global guard", name, call.String()) + } + branch, ok := pred.Instrs[len(pred.Instrs)-1].(*ssa.If) + if !ok { + return fmt.Errorf("%s call %q is not controlled by one exact false-global guard", name, call.String()) + } + load, ok := branch.Cond.(*ssa.UnOp) + if !ok || load.Op != token.MUL { + return fmt.Errorf("%s call %q guard is not an exact global load", name, call.String()) + } + global, ok := load.X.(*ssa.Global) + if !ok || global.Pkg == nil || global.Pkg.Pkg == nil || global.Name() != "_Cgo_always_false" { + return fmt.Errorf("%s call %q guard is not cmd/cgo's exact global", name, call.String()) + } + condition, ok := types.Unalias(load.Type()).Underlying().(*types.Basic) + if !ok || condition.Kind() != types.Bool { + return fmt.Errorf("%s call %q guard is not a bool global", name, call.String()) + } + linkname, linked := prog.Linkname(llssa.FullName(global.Pkg.Pkg, global.Name())) + if !linked || linkname != "runtime.cgoAlwaysFalse" { + return fmt.Errorf("%s call %q guard does not resolve to runtime.cgoAlwaysFalse", name, call.String()) + } + return nil +} + const coroWorkerMaxArgsV1 = 9 func planCoroHostOperationCallShape(call *ssa.Call) (coroHostOperationCallShape, error) { @@ -3019,6 +3091,12 @@ func coroIntrinsicCallSemantics(opcode int) CoroIntrinsicCallSemantics { // compatibility operation. It consumes its already-evaluated operands // and emits no call or suspension. return CoroIntrinsicCallInlineNoSuspend + case llgoCgoUse, llgoCgoKeepAlive: + // cmd/cgo emits these liveness calls only on the true edge of the exact + // runtime.cgoAlwaysFalse global. ProgramIR verifies that generated shape; + // lowering retains the argument through the preceding worker suspension + // with llvm.fake.use and emits no callable edge. + return CoroIntrinsicCallInlineNoSuspend case llgoCoroPark: return CoroIntrinsicCallInlineSuspend case llgoCoroYield: @@ -3644,7 +3722,7 @@ func (u *EmissionUniverse) physicalName(ownerSSA *ssa.Package, fn *ssa.Function, } return "", fmt.Errorf( "coroutine entry resolution: generated wrapper %q (%q, %s) has no frozen physical symbol for owner %q; frozen owners: %v", - fn.Name(), fn.Synthetic, structuralEmissionTypeKey(fn.Signature), ownerName, available, + fn.Name(), fn.Synthetic, u.emissionTypeKeys.strict(fn.Signature), ownerName, available, ) } return legacy, nil @@ -4527,9 +4605,9 @@ func (u *EmissionUniverse) samePromotedWrapperLinkIdentity(owner *preparedEmissi } func (u *EmissionUniverse) structuralWrapperABIKey(owner *preparedEmissionPackage, fn *ssa.Function) string { - fields := []string{"wrapper-abi-v1", structuralEmissionTypeKey(u.effectiveType(owner, fn, fn.Signature, false))} + fields := []string{"wrapper-abi-v1", u.emissionTypeKeys.strict(u.effectiveType(owner, fn, fn.Signature, false))} for _, free := range fn.FreeVars { - fields = append(fields, structuralEmissionTypeKey(u.effectiveType(owner, fn, free.Type(), false))) + fields = append(fields, u.emissionTypeKeys.strict(u.effectiveType(owner, fn, free.Type(), false))) } return framedEmissionKey(fields...) } @@ -4605,13 +4683,21 @@ func deterministicSSABody(fn *ssa.Function) string { // used only by the prepared active universe; it does not change global // funcName or report-only IR naming. func structuralEmissionTypeKey(typ types.Type) string { - builder := emissionTypeKeyBuilder{active: make(map[types.Type]int)} - return builder.key(typ) + return compactStructuralTypeGraphKey( + typ, + emissionStrictTypeGraphSchema, + emissionStrictTypeGraphKeyPrefix, + emissionTypeGraphOptions{}, + ) } func structuralEmissionABITypeKey(typ types.Type) string { - builder := emissionTypeKeyBuilder{active: make(map[types.Type]int), omitTupleNames: true} - return builder.key(typ) + return compactStructuralTypeGraphKey( + typ, + emissionStrictABITypeGraphSchema, + emissionStrictABITypeGraphKeyPrefix, + emissionTypeGraphOptions{omitTupleNames: true}, + ) } // structuralGoLinknameABITypeKey models the source-level ABI promise made by @@ -4636,16 +4722,19 @@ func structuralCFunctionABITypeKey(typ types.Type) string { } func structuralNamedIdentityFreeABITypeKey(typ types.Type) string { - builder := emissionTypeKeyBuilder{ - active: make(map[types.Type]int), - omitTupleNames: true, - expandNamed: true, - omitStructFieldMetadata: true, - } if signature, ok := types.Unalias(typ).(*types.Signature); ok { typ = normalizeGoLinknameABISignature(signature) } - return builder.key(typ) + return compactStructuralTypeGraphKey( + typ, + emissionIdentityFreeTypeGraphSchema, + emissionIdentityFreeTypeGraphPrefix, + emissionTypeGraphOptions{ + omitTupleNames: true, + expandNamed: true, + omitStructFieldMetadata: true, + }, + ) } // normalizeGoLinknameABISignature models the symbol ABI used by a linkname to @@ -4690,167 +4779,6 @@ func normalizeGoLinknameABISignature(signature *types.Signature) *types.Signatur ) } -type emissionTypeKeyBuilder struct { - active map[types.Type]int - next int - omitTupleNames bool - expandNamed bool - omitStructFieldMetadata bool -} - -func (b *emissionTypeKeyBuilder) key(typ types.Type) string { - if typ == nil { - return framedEmissionKey("nil-type") - } - typ = types.Unalias(typ) - if named, ok := typ.(*types.Named); ok && b.expandNamed { - return b.expandedNamedKey(named) - } - return b.keyNode(typ) -} - -// expandedNamedKey assigns one cycle identity to a named type and its -// underlying node. Linkname ABI comparison deliberately erases named -// identity; a patched signature may therefore contain the named source type -// on one side and its anonymous physical RawType on the other. Recursive -// layouts must hash identically in both representations. -func (b *emissionTypeKeyBuilder) expandedNamedKey(named *types.Named) string { - if id, ok := b.active[named]; ok { - return framedEmissionKey("type-cycle", strconv.Itoa(id)) - } - underlying := types.Unalias(named.Underlying()) - if id, ok := b.active[underlying]; ok { - return framedEmissionKey("type-cycle", strconv.Itoa(id)) - } - id := b.next - b.next++ - b.active[named] = id - b.active[underlying] = id - defer delete(b.active, named) - defer delete(b.active, underlying) - return b.keyBody(underlying) -} - -func (b *emissionTypeKeyBuilder) keyNode(typ types.Type) string { - if id, ok := b.active[typ]; ok { - return framedEmissionKey("type-cycle", strconv.Itoa(id)) - } - id := b.next - b.next++ - b.active[typ] = id - defer delete(b.active, typ) - return b.keyBody(typ) -} - -func (b *emissionTypeKeyBuilder) keyBody(typ types.Type) string { - pkgKey := func(pkg *types.Package) string { - if pkg == nil { - return "" - } - return llssa.PathOf(pkg) - } - switch typ := typ.(type) { - case *types.Basic: - return framedEmissionKey("basic", strconv.Itoa(int(typ.Kind())), typ.Name()) - case *types.Pointer: - return framedEmissionKey("pointer", b.key(typ.Elem())) - case *types.Array: - return framedEmissionKey("array", strconv.FormatInt(typ.Len(), 10), b.key(typ.Elem())) - case *types.Slice: - return framedEmissionKey("slice", b.key(typ.Elem())) - case *types.Map: - return framedEmissionKey("map", b.key(typ.Key()), b.key(typ.Elem())) - case *types.Chan: - return framedEmissionKey("chan", strconv.Itoa(int(typ.Dir())), b.key(typ.Elem())) - case *types.Named: - obj := typ.Obj() - fields := []string{"named"} - packageLevel := false - if obj != nil { - fields = append(fields, pkgKey(obj.Pkg()), obj.Name()) - packageLevel = obj.Pkg() != nil && obj.Parent() == obj.Pkg().Scope() - } - if args := typ.TypeArgs(); args != nil { - for i := 0; i < args.Len(); i++ { - fields = append(fields, b.key(args.At(i))) - } - } - if !packageLevel { - fields = append(fields, "local-underlying", b.key(typ.Underlying())) - } - return framedEmissionKey(fields...) - case *types.Struct: - fields := []string{"struct", strconv.Itoa(typ.NumFields())} - for i := 0; i < typ.NumFields(); i++ { - field := typ.Field(i) - if b.omitStructFieldMetadata { - fields = append(fields, b.key(field.Type())) - continue - } - fields = append(fields, - pkgKey(field.Pkg()), - field.Name(), - strconv.FormatBool(field.Embedded()), - typ.Tag(i), - b.key(field.Type()), - ) - } - return framedEmissionKey(fields...) - case *types.Tuple: - fields := []string{"tuple", strconv.Itoa(typ.Len())} - for i := 0; i < typ.Len(); i++ { - variable := typ.At(i) - if !b.omitTupleNames { - fields = append(fields, pkgKey(variable.Pkg()), variable.Name()) - } - fields = append(fields, b.key(variable.Type())) - } - return framedEmissionKey(fields...) - case *types.Signature: - fields := []string{"signature", strconv.FormatBool(typ.Variadic())} - if typ.Recv() != nil { - fields = append(fields, "recv", b.key(typ.Recv().Type())) - } - for _, params := range []*types.TypeParamList{typ.RecvTypeParams(), typ.TypeParams()} { - fields = append(fields, "type-params") - if params != nil { - for i := 0; i < params.Len(); i++ { - fields = append(fields, b.key(params.At(i))) - } - } - } - fields = append(fields, b.key(typ.Params()), b.key(typ.Results())) - return framedEmissionKey(fields...) - case *types.Interface: - typ.Complete() - fields := []string{"interface", strconv.Itoa(typ.NumMethods()), strconv.Itoa(typ.NumEmbeddeds())} - for i := 0; i < typ.NumMethods(); i++ { - method := typ.Method(i) - fields = append(fields, pkgKey(method.Pkg()), method.Name(), b.key(method.Type())) - } - for i := 0; i < typ.NumEmbeddeds(); i++ { - fields = append(fields, b.key(typ.EmbeddedType(i))) - } - return framedEmissionKey(fields...) - case *types.TypeParam: - obj := typ.Obj() - name, pkg := "", "" - if obj != nil { - name, pkg = obj.Name(), pkgKey(obj.Pkg()) - } - return framedEmissionKey("type-param", pkg, name, b.key(typ.Constraint())) - case *types.Union: - fields := []string{"union", strconv.Itoa(typ.Len())} - for i := 0; i < typ.Len(); i++ { - term := typ.Term(i) - fields = append(fields, strconv.FormatBool(term.Tilde()), b.key(term.Type())) - } - return framedEmissionKey(fields...) - default: - return framedEmissionKey("other-type", types.TypeString(typ, func(pkg *types.Package) string { return pkgKey(pkg) })) - } -} - func isLocallyMergedPromotedWrapper(fn *ssa.Function) bool { if fn == nil || !strings.HasPrefix(fn.Synthetic, "wrapper for ") { return false @@ -5297,12 +5225,12 @@ func (index emissionCanonicalIndex) managedGoLinknamePairKeyWithPointerFacade( if signature := physicalSignature.(*types.Signature); forcePointerFacade || managedGoLinknameHasDirectUnsafePointer(signature) { signatureType = managedGoLinknameDirectPointerFacade(signature) } - signature := structuralGoLinknameABITypeKey(signatureType) + signature := u.emissionTypeKeys.goLinknameABI(signatureType) if typeArgs := function.TypeArgs(); len(typeArgs) != 0 { fields := make([]string, 0, len(typeArgs)+2) fields = append(fields, "go-linkname-callable-instance-v1", signature) for _, argument := range typeArgs { - fields = append(fields, structuralGoLinknameABITypeKey(u.effectiveType(owner, function, argument, false))) + fields = append(fields, u.emissionTypeKeys.goLinknameABI(u.effectiveType(owner, function, argument, false))) } signature = framedEmissionKey(fields...) } @@ -5895,9 +5823,9 @@ func (u *EmissionUniverse) classifiedManagedSymbol(prepared *preparedEmissionPac // Parameter and result names are source/debug metadata, not callable ABI. // Patch replacements may legitimately omit or rename them. if ftype == cFunc { - sig = structuralCFunctionABITypeKey(patchedSignature) + sig = u.emissionTypeKeys.cFunctionABI(patchedSignature) } else { - sig = structuralEmissionABITypeKey(patchedSignature) + sig = u.emissionTypeKeys.strictABI(patchedSignature) } if typeArgs := fn.TypeArgs(); len(typeArgs) != 0 { // A generic argument is not necessarily observable in the callable @@ -5908,7 +5836,7 @@ func (u *EmissionUniverse) classifiedManagedSymbol(prepared *preparedEmissionPac fields := make([]string, 0, len(typeArgs)+2) fields = append(fields, "callable-instance-v1", sig) for _, argument := range typeArgs { - fields = append(fields, structuralEmissionTypeKey(ctx.patchType(argument))) + fields = append(fields, u.emissionTypeKeys.strict(ctx.patchType(argument))) } sig = framedEmissionKey(fields...) } @@ -6031,8 +5959,8 @@ func (u *EmissionUniverse) wrapperCallIdentity(prepared *preparedEmissionPackage "invoke-method-v1", pkgPath, method.Name(), - structuralEmissionTypeKey(u.effectiveType(prepared, fn, method.Type(), false)), - structuralEmissionTypeKey(u.effectiveType(prepared, fn, common.Value.Type(), false)), + u.emissionTypeKeys.strict(u.effectiveType(prepared, fn, method.Type(), false)), + u.emissionTypeKeys.strict(u.effectiveType(prepared, fn, common.Value.Type(), false)), ), false, nil } return "", false, nil @@ -6167,11 +6095,13 @@ func (u *EmissionUniverse) materializeFunctionForOwner(fn *ssa.Function, owner * // though codegen does not emit those instructions. ProgramIR must therefore // own their local facts as well; falling back to an analysis-time raw scan // would reopen a second semantic authority. + constantReachable := coroPhysicalConstantReachableBlocks(fn) for _, block := range fn.Blocks { for _, instruction := range block.Instrs { - evaluated := true + evaluated := constantReachable[block] if isCgo { - _, evaluated = cgoPlan.evaluated[instruction] + _, cgoEvaluated := cgoPlan.evaluated[instruction] + evaluated = evaluated && cgoEvaluated } if err := u.coroProgramIR.freezeSemanticInstruction(fn, owner, instruction, evaluated); err != nil { return fmt.Errorf("prepare emission universe: function %q semantic SitePlan: %w", fn.Name(), err) @@ -6276,7 +6206,7 @@ func (u *EmissionUniverse) materializeFunctionForOwner(fn *ssa.Function, owner * } effectiveSignature := u.effectiveType(owner, spawn.Parent(), signature, false) structuralKey, err := builtinSpawnWrapperStructuralKey( - key, owner, u.finalIdentity(spawn.Parent()), effectiveSignature, + u.emissionTypeKeys.strict, key, owner, u.finalIdentity(spawn.Parent()), effectiveSignature, ) if err != nil { return err @@ -6340,6 +6270,9 @@ func (u *EmissionUniverse) materializeFunctionForOwner(fn *ssa.Function, owner * return fmt.Errorf("prepare emission universe: function %q SitePlan shape: %w", fn.Name(), err) } for _, block := range fn.Blocks { + if !constantReachable[block] { + continue + } for _, instr := range block.Instrs { if _, unevaluated := ctx.unevaluatedSSA[instr]; unevaluated { continue @@ -7690,6 +7623,7 @@ func (u *EmissionUniverse) intrinsicWrapperStructuralKey(info intrinsicWrapperKe } func builtinSpawnWrapperStructuralKey( + typeKey func(types.Type) string, info builtinSpawnWrapperKey, owner *preparedEmissionPackage, parentIdentity string, @@ -7699,7 +7633,7 @@ func builtinSpawnWrapperStructuralKey( info.spawn.Parent() == nil || info.spawn.Block() == nil { return "", fmt.Errorf("builtin spawn wrapper has no exact owner and source call") } - if owner == nil || owner.ssa != info.owner || parentIdentity == "" || effectiveSignature == nil { + if typeKey == nil || owner == nil || owner.ssa != info.owner || parentIdentity == "" || effectiveSignature == nil { return "", fmt.Errorf("builtin spawn wrapper owner is absent from the emission universe") } builtin, ok := info.spawn.Common().Value.(*ssa.Builtin) @@ -7717,7 +7651,7 @@ func builtinSpawnWrapperStructuralKey( strconv.Itoa(info.spawn.Block().Index), strconv.Itoa(ordinal), builtin.Name(), - structuralEmissionTypeKey(effectiveSignature), + typeKey(effectiveSignature), ), nil } @@ -7790,7 +7724,7 @@ func (u *EmissionUniverse) freezeCoroGlobalPhysicalIdentities() error { global: global, owner: prepared, physicalSymbol: physicalSymbol, - structuralType: structuralEmissionTypeKey(patchedType), + structuralType: u.emissionTypeKeys.strict(patchedType), background: llssa.Background(variableType), define: define, linknamed: linknamed, @@ -8032,6 +7966,7 @@ func (u *EmissionUniverse) freezeFunctionIdentities() error { return err } key, err := builtinSpawnWrapperStructuralKey( + u.emissionTypeKeys.strict, info, owner, u.finalIdentity(info.spawn.Parent()), @@ -8702,6 +8637,7 @@ func (u *EmissionUniverse) finalIdentity(fn *ssa.Function) string { owner := u.packages[info.owner] if signature, err := builtinSpawnCarrierSignature(info.spawn.Common()); err == nil { if key, err := builtinSpawnWrapperStructuralKey( + u.emissionTypeKeys.strict, info, owner, u.finalIdentity(info.spawn.Parent()), diff --git a/cl/import.go b/cl/import.go index bbdd7d34ad..a1fa32a5c4 100644 --- a/cl/import.go +++ b/cl/import.go @@ -874,6 +874,13 @@ const ( llgoCgoCMalloc = llgoCgoBase + 0x5 llgoCgoCheckPointer = llgoCgoBase + 0x6 llgoCgoCgocall = llgoCgoBase + 0x7 + // llgoCgoUse and llgoCgoKeepAlive are private compiler identities for + // cmd/cgo's generated, false-guarded liveness declarations. They are not + // recognized by source name alone: funcName requires the exact bodyless + // signature and runtime.cgoUse/runtime.cgoKeepAlive linkname first, while + // ProgramIR separately verifies the exact runtime.cgoAlwaysFalse guard. + llgoCgoUse = llgoCgoBase + 0x8 + llgoCgoKeepAlive = llgoCgoBase + 0x9 llgoAsm = llgoInstrBase + 0x40 llgoStackSave = llgoInstrBase + 0x41 @@ -1141,6 +1148,9 @@ func (p *context) funcName(fn *ssa.Function) (*types.Package, string, int) { p.ensureLoaded(pkg) orgName = funcName(pkg, fn, false) } + if intrinsicName, ok := p.cgoGeneratedTouchIntrinsic(fn, orgName); ok { + return nil, intrinsicName, llgoInstr + } if v, ok := p.prog.Linkname(orgName); ok { if strings.HasPrefix(v, "C.") { return nil, v[2:], cFunc @@ -1173,6 +1183,36 @@ func (p *context) funcName(fn *ssa.Function) (*types.Package, string, int) { return pkg, funcName(pkg, fn, false), goFunc } +func (p *context) cgoGeneratedTouchIntrinsic(fn *ssa.Function, orgName string) (string, bool) { + if p == nil || p.prog == nil || fn == nil || orgName == "" || len(fn.Blocks) != 0 { + return "", false + } + var intrinsicName, target string + switch fn.Name() { + case "_Cgo_use": + intrinsicName, target = "cgoUse", "runtime.cgoUse" + case "_Cgo_keepalive": + intrinsicName, target = "cgoKeepAlive", "runtime.cgoKeepAlive" + default: + return "", false + } + linkname, linked := p.prog.Linkname(orgName) + if !linked || linkname != target { + return "", false + } + signature := fn.Signature + if signature == nil || signature.Recv() != nil || signature.Variadic() || + signature.Params() == nil || signature.Params().Len() != 1 || + signature.Results() != nil && signature.Results().Len() != 0 { + return "", false + } + iface, ok := types.Unalias(signature.Params().At(0).Type()).Underlying().(*types.Interface) + if !ok || !iface.Empty() { + return "", false + } + return intrinsicName, true +} + const ( ignoredVar = iota goVar = int(llssa.InGo) diff --git a/cl/instr.go b/cl/instr.go index 273b26d3e0..96c7e65574 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -861,6 +861,8 @@ var llgoInstrs = map[string]int{ "_Cfunc__CMalloc": llgoCgoCMalloc, "_cgoCheckPointer": llgoCgoCheckPointer, "_cgo_runtime_cgocall": llgoCgoCgocall, + "cgoUse": llgoCgoUse, + "cgoKeepAlive": llgoCgoKeepAlive, "asm": llgoAsm, "stackSave": llgoStackSave, @@ -1983,7 +1985,7 @@ func (p *context) emitPCLineLabel(b llssa.Builder, pos token.Pos) { pushSection = ".pushsection __DATA,__llgo_pcl,regular,live_support" recordSymbol = "l_llgo_pcline_rec_${:uid}:\n" } - b.InlineAsm( + b.CoroAtomicDataAnchor( asmLabel + ":\n" + pushSection + "\n" + ".p2align " + align + "\n" + @@ -2632,6 +2634,12 @@ func (p *context) callEx( ret = p.cgoCMalloc(b, args) case llgoCgoCheckPointer: p.cgoCheckPointer(b, args) + case llgoCgoUse, llgoCgoKeepAlive: + if act != llssa.Call || ds != nil || len(args) != 1 { + panic("generated cgo liveness intrinsic requires one exact direct call") + } + compiled := p.compileValues(b, args, kind) + b.KeepAlive(compiled[0]) case llgoCgoCgocall: ret = p.cgoCgocall(b, args) case llgoAdvance: diff --git a/cmd/llgo/llvm_path.go b/cmd/llgo/llvm_path.go index 7a804540ff..8e25b6b234 100644 --- a/cmd/llgo/llvm_path.go +++ b/cmd/llgo/llvm_path.go @@ -6,5 +6,5 @@ import "github.com/goplus/llgo/xtool/env/llvm" // dispatch so build requests and their workers only need an environment // snapshot; they do not carry or reselect an LLVM installation. func init() { - llvm.SetupPath() + llvm.SetupPathOrExit() } diff --git a/dev/Dockerfile.dev b/dev/Dockerfile.dev index 264c0f6171..ebc83facc5 100644 --- a/dev/Dockerfile.dev +++ b/dev/Dockerfile.dev @@ -5,7 +5,7 @@ ARG BASE_IMAGE=ubuntu:${UBUNTU_VERSION} FROM ${BASE_IMAGE} AS base ARG DEBIAN_FRONTEND=noninteractive -ARG LLVM_VERSION=19 +ARG LLVM_VERSION=22 ARG GO_VERSION=1.24.2 ARG CARGS_VERSION=1.0.0 diff --git a/doc/_readme/scripts/install_macos.sh b/doc/_readme/scripts/install_macos.sh index a2bd676820..29b6217ed0 100644 --- a/doc/_readme/scripts/install_macos.sh +++ b/doc/_readme/scripts/install_macos.sh @@ -1,7 +1,7 @@ # shellcheck disable=all brew update -brew install llvm@19 lld@19 bdw-gc openssl cjson libffi libuv pkg-config +brew install llvm@22 lld@22 bdw-gc openssl cjson libffi libuv pkg-config brew install python@3.12 # optional -brew link --overwrite llvm@19 lld@19 libffi +brew link --overwrite llvm@22 lld@22 libffi # curl https://raw.githubusercontent.com/xgo-dev/llgo/refs/heads/main/install.sh | bash ./install.sh diff --git a/doc/_readme/scripts/install_ubuntu.sh b/doc/_readme/scripts/install_ubuntu.sh index 62476e355e..c6da026735 100644 --- a/doc/_readme/scripts/install_ubuntu.sh +++ b/doc/_readme/scripts/install_ubuntu.sh @@ -1,8 +1,8 @@ # shellcheck disable=all -echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-19 main" | sudo tee /etc/apt/sources.list.d/llvm.list +echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-22 main" | sudo tee /etc/apt/sources.list.d/llvm.list wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - sudo apt-get update -sudo apt-get install -y llvm-19-dev clang-19 libclang-19-dev lld-19 libunwind-19-dev libc++-19-dev pkg-config libgc-dev libssl-dev zlib1g-dev libffi-dev libcjson-dev libsqlite3-dev libuv1-dev +sudo apt-get install -y llvm-22-dev clang-22 libclang-22-dev lld-22 libunwind-22-dev libc++-22-dev pkg-config libgc-dev libssl-dev zlib1g-dev libffi-dev libcjson-dev libsqlite3-dev libuv1-dev sudo apt-get install -y python3.12-dev # optional #curl https://raw.githubusercontent.com/xgo-dev/llgo/refs/heads/main/install.sh | bash ./install.sh diff --git a/doc/coro-async-core-contract.md b/doc/coro-async-core-contract.md index f10582188e..46932f5eec 100644 --- a/doc/coro-async-core-contract.md +++ b/doc/coro-async-core-contract.md @@ -579,7 +579,7 @@ deadline 修改必须更新已 park operation，且 timeout 恢复后重新读 Native表中的有界fleet已从production-island推进为可执行target profile；它证明无栈continuation可在固定8-route物理topology上按动态逻辑quota并行，并能用统一event/worker模型运行标准库TCP链。物理route/source identity不随`GOMAXPROCS`变化；locked-M同P replacement已证明阻塞期间原route继续服务channel/timer/poll并可嵌套，clean succession又证明未解锁退出不会复用被污染M，但当前证据仍不能外推为全部可运行G迁移、所有callback/reentry/非command shutdown/affinity场景或完整标准库兼容。 -无栈 continuation 和 target-neutral operation core 不依赖 libuv、BDWGC 或 pthread；但当前 native worker adapter 确实使用固定 pthread pool，collector 集成也仍是 target profile 的独立责任。LLVM 支持基线只是 19–22，不考虑 LLVM 19 以下版本；每个支持版本都要分别验证 CoroSplit、frame layout/root metadata 和 module verification。 +无栈 continuation 和 target-neutral operation core 不依赖 libuv、BDWGC 或 pthread；但当前 native worker adapter 确实使用固定 pthread pool，collector 集成也仍是 target profile 的独立责任。LLVM 支持基线固定为 22；LLVM 21 及以下不再维护兼容配置或 CI。LLVM 22 必须验证 CoroSplit、frame layout/root metadata 和 module verification。 full-native Darwin/Linux 的 signal adapter 已改为 C `sigaction` + nonblocking self-pipe；signal handler 只发布 lock-free POD signum，Go `signal_recv` 复用 coroutine poll owner，不依赖 legacy libuv timer loop。当前 source/C/race 验证已通过，`os/signal.Notify` 的真实 LLGo 链接执行仍随全程序 acceptance 推进。 diff --git a/doc/coro-callable-contract.md b/doc/coro-callable-contract.md index 0de5cca00d..e35afc3d74 100644 --- a/doc/coro-callable-contract.md +++ b/doc/coro-callable-contract.md @@ -72,7 +72,7 @@ storage、archive 或开放动态边界传递的 foreign callable 使用显式 - 不让 wrapper 的信任标注跳过其余 Go body、循环、panic、cleanup、pollWait 或调用图 分析； - 不承诺不存在相应 OS/HAL/host capability 的 target 实现完整文件、网络或进程 API； -- 不以 LLVM 19 以下版本、旧 LLGo 二进制 ABI 或旧 coroutine prototype 为兼容目标。 +- 不以 LLVM 22 以外版本、旧 LLGo 二进制 ABI 或旧 coroutine prototype 为兼容目标。 ## 4. 不可协商的硬约束 @@ -1169,9 +1169,9 @@ Native bounded worker、regular-file与双owner TCP链已有生产证据；forwa | RTOS/embedded | QEMU或硬件notification、one-shot alarm、ISR publish、DMA cancel、容量填满、task affinity | | Baremetal | QEMU main loop、hardware compare、IRQ ring、WFI/WFE、stale generation、IRQ fence、无pthread/libuv依赖 | -最终支持范围要求LLVM 19、20、21、22分别验证CoroSplit、descriptor/frame layout、 -module verification和关键E2E；不覆盖LLVM 19以下。这是待完成的验收门，不是当前 -通过声明。GC profile亦需分别验证nogc、conservative和最终的exact frame root/pin +最终支持范围固定为LLVM 22，并要求验证CoroSplit、descriptor/frame layout、module +verification和关键E2E；不再维护LLVM 21及以下的兼容分支。这是待完成的验收门， +不是当前通过声明。GC profile亦需分别验证nogc、conservative和最终的exact frame root/pin contract，compile-only不能替代production platform E2E。 ## 20. 当前实现状态与差距 diff --git a/doc/coro-ir-design.md b/doc/coro-ir-design.md index c6611a40d3..1b67fa28ef 100644 --- a/doc/coro-ir-design.md +++ b/doc/coro-ir-design.md @@ -93,7 +93,7 @@ ProgramModelBuilder fixed point - 让 timer、文件、网络、host Promise、worker、RTOS notification、IRQ 等扩展复用同一 `Park/Operation` 模型，而不是新增 compiler semantic family。 - 把 select 多候选和执行取消作为公共底层语义，保证结果 lease、loser detach 和 cleanup 次序。 - 降低新增语言特性或 event source 时同时修改多个 compiler 模块的概率。 -- 保留当前 LLVM 19–22 支持范围；不为 LLVM 19 以下版本增加设计负担。 +- 固定 LLVM 22 支持基线；不为 LLVM 21 及以下保留兼容分支。 ### 2.2 非目标 @@ -876,7 +876,7 @@ Op(frame-spilled for internal waits, registry-backed for external callbacks) 它应保留为 LLVM backend，不应把 Go effect、select或cleanup语义塞入其中。新 emitter可减少 feature callback数量，但无需重写 `CoroBuilder`。 -LLVM CoroSplit继续负责普通 SSA liveness和frame materialization。精确 GC需要在CoroSplit后取得可靠frame layout/root metadata，或在显式slot层为GC-managed值提供自己的descriptor；这需要LLVM 19–22分别验证，不能仅凭pre-split IR推断最终offset。 +LLVM CoroSplit继续负责普通 SSA liveness和frame materialization。精确 GC需要在CoroSplit后取得可靠frame layout/root metadata，或在显式slot层为GC-managed值提供自己的descriptor；这需要在LLVM 22各目标上分别验证，不能仅凭pre-split IR推断最终offset。 非移动保守GC不必等待这项精确metadata：当前`wasip1-gc`把G task和整个LLVM frame allocation都放入tinygogc heap，static P/G与wait链提供owner root，collector递归扫描完整frame block即可观察CoroSplit spill。该路径没有全局current-root chain、函数地址反查或额外sidecar allocation；代价是false-positive retention，并且仅允许stop-the-world单executor。它是精确/移动GC之前的独立可验收profile，不可把通过结果外推到并行STW、weak/finalizer或GC Full。 @@ -1011,7 +1011,7 @@ cl/coro_recipe_*.go ordinary lowering recipe planning/emission pairs - 以 `897d251f8` 为迁移基线，不混入新runtime功能。 - 先实现plan/semantic CFG canonicalizer；只为小型代表fixture保存plain/await/preempt/park/timer/channel/select/spawn/panic投影，不保存整个支持subset或完整post-CoroSplit文本；panic标注为focused/manual fixture，不冒充production build path。 -- 各LLVM版本分别做module verify和结构断言；frame/object size记录版本内基线与阈值，不要求LLVM 19–22文本或精确size相同。 +- LLVM 22各目标分别做module verify和结构断言；frame/object size记录目标内基线与阈值，不要求不同目标的文本或精确size相同。 - 定义固定fixture/target、warm cache、重复次数/中位数、alloc和peak RSS采集方式；先报告compile wall、node/bytes、block/instruction、frame和object size，取得噪声后再冻结回退阈值。 验收：不改生成IR。 @@ -1410,7 +1410,7 @@ site只接受ProgramIR projection。旧callback/classifier在同一提交删除 - 对同一fixture执行两个独立test-only compile/module invocation：legacy读取raw SSA，新backend读取FunctionIR，避免同名symbol在一个module双发；production config没有双backend开关。 - 比较canonical semantic projection、suspend/continuation/helper/descriptor、post-CoroSplit verify、frame阈值和运行结果，不要求physical CFG同构。 -验收：native+nogc E2E、host race/shuffle、JS/WASM test adapter、native64/wasm32、LLVM 19–22全部通过后，在同一cohort cutover并删除legacy emitter。只完成对照而未删除旧路径不算Phase E完成。 +验收：native+nogc E2E、host race/shuffle、JS/WASM test adapter、LLVM 22 native64/wasm32全部通过后，在同一cohort cutover并删除legacy emitter。只完成对照而未删除旧路径不算Phase E完成。 ### Phase F：按完整函数切换并删除重复实现 @@ -1460,7 +1460,7 @@ runtime hard cutover不删除`OperationRecord/ParkState/WaitSetRecord/result lea - `G`/`P`及production target不存在`WaitToken`logical queue，Timer/Poll不存在V1/V2 mode； - production config不再含多个阶段性`EnableCoro*`布尔字段； - 新增一种event source只修改source adapter、profile catalog和测试，不修改compiler opcode/feature lowerer； -- `go test -race` runtime core、LLVM 19/20/21/22结构门和六项fresh fleet标准库E2E全部通过。 +- `go test -race` runtime core、LLVM 22结构门和六项fresh fleet标准库E2E全部通过。 最终仓库验收不是上述六个探针，而是`test/*`（包括`test/std/*`）与Go 1.26 GOROOT runner的全部适用测试通过。临时xfail只用于定位；最终基线不得有unexpected failure或stale XPASS，也不得用package/function白名单、扩大xfail或修改指令期望来掩盖通用lowering/runtime缺口。当前阶段另有一条先行硬门：任何OS/host I/O、timer、同步原语或不可避免的外部等待都不能阻塞executor，必须先stack-cut并park或转交bounded worker/平台事件源；每个此类接入至少有一个“等待方未完成时另一G仍前进”的进度用例。纯计算暂时允许占用一个P，生产级完成前再以M/P/G、P-neutral runnable/result、动态P/steal及blocking compensation把密集计算和不可避免的阻塞彼此解耦。 @@ -1533,7 +1533,7 @@ gate应解析Go AST/build constraints或检查冻结catalog，不依赖容易被 ### 16.3 target矩阵 -当前feature PR必跑层按现有 `coroutine.yml`：Ubuntu 22.04；Go 1.26.5上的LLVM 19/20/21/22 compatibility矩阵，并在LLVM 19单独运行integration与targets lane；host runtime core `-race -shuffle`、JS/WASM test adapter、native timer/time.Sleep focused E2E、arm/riscv/WASM/baremetal compile/link检查。快速structural/verify矩阵与LLVM 19完整E2E保持拆分，避免四个compatibility job重复重runtime而超过20分钟job预算。 +当前feature PR必跑层按现有 `coroutine.yml`：Ubuntu 22.04、Go 1.26.5、LLVM 22；按compatibility、integration、targets和分片E2E lane拆分host runtime core `-race -shuffle`、JS/WASM test adapter、native timer/time.Sleep focused E2E及arm/riscv/WASM/baremetal compile/link检查，避免重runtime任务在同一进程累积资源。 `cpunion/llgo`的coro开发分支只触发`Format Check`和上述focused coroutine workflow； Go、cache、LLGo、target、release、docs及stdlib coverage等宽工作流只接受`main`的push/PR。 @@ -1620,7 +1620,7 @@ channel/select physical operation choice、panic/outcome/cleanup choice及remain feature分支；不得把B.11统一call dispatcher误当成whole-function emitter已经完成。 5. runtime Phase R已经完成：fleet唯一target、Park/Operation唯一logical wait、统一source dispatcher和 mandatory stackless架构均以旧production符号/配置入口为零，并由hard-cutover gate持续约束。 -6. 本轮合并门运行runtime race、LLVM 19–22、native/wasm32结构验证和六项fresh stdlib E2E。后续compiler +6. 本轮合并门运行runtime race、LLVM 22 native/wasm32结构验证和六项fresh stdlib E2E。后续compiler whole-function emitter仍按完整replacement cohort推进；single-source、Channel/select、HostOp/keyed P-neutral materialization、动态逻辑execution quota、locked-M同P replacement、standby M、`SetMaxThreads`、三类clean owner succession和command main-return blocked-M终止已独立落地，callback-reentry与非command shutdown、GC、 panic/Goexit和更多平台adapter属于功能阶段，不能借机恢复旧runtime轨道。 diff --git a/doc/coro-performance-baseline.md b/doc/coro-performance-baseline.md index 57e0b8841d..9d086e789d 100644 --- a/doc/coro-performance-baseline.md +++ b/doc/coro-performance-baseline.md @@ -1212,3 +1212,526 @@ catalog-scan problem: an immediately matched channel operation still suspends both endpoints and runs the durable A/ack/B plus typed-cleanup transaction. Avoiding that suspension on the exact local ready path is the next performance gate. + +### Closed-program worker demand and pure static checkpoint + +The 2026-08-13 local candidate based on `914379fbd` separates target support +from closed-program demand. After the final physical ProgramIR transaction is +sealed, the compiler projects a worker bit only from reachable +`worker-syscall`, `worker-foreign`, `worker-cgo`, and `worker-cgo-errno` +recipes. The bit is included in both startup-table and manifest hashes. The +runtime validates it before binding executor sources and starts the native +worker pool only when the program requests it. + +This exposed one real false demand in `g_pthread.go`: an unrecoverable TLS-key +failure tried to call `fprintf` before a scheduler existed. That diagnostic +could never suspend and resume correctly. It now delegates directly to the +existing terminal synchronous abort path, while ordinary stdio remains a +potentially blocking worker operation. + +Two no-import, no-output fixtures provide a closed negative gate. Both were +built with LLVM 22.1.8 using `-a -trimpath -O3 -lto=full -ldflags='-s -w'` and +an independent cache. Their linked V2 bootstrap has flags zero, the generated +worker foreign-call thunk is absent, and the four-second compute run has one +OS thread. The existing general workload retains flags `1`, providing the +positive worker-demand side of the gate. + +The table reports medians of seven AB/BA-interleaved `/usr/bin/time -lp` runs. +Retired instructions and cycles are authoritative because process wall time +was heavily affected by unrelated host scheduling. + +| Pure fixture / metric | Go 1.26.5 | LLGo coroutine | Difference | +| --- | ---: | ---: | ---: | +| idle file bytes | 1,161,922 | 897,968 | -22.72% | +| idle retired instructions | 23,433,796 | 40,681,350 | +73.60% | +| idle cycles | 19,374,249 | 22,141,783 | +14.29% | +| idle peak RSS | 3,293,184 | 3,768,320 | +475,136 (+14.43%) | +| compute file bytes | 1,161,922 | 898,608 | -22.66% | +| one-billion-call instructions | 25,235,748,943 | 23,219,390,643 | -7.99% | +| one-billion-call cycles | 6,876,063,156 | 5,779,240,893 | -15.95% | +| compute peak RSS | 3,276,800 | 3,768,320 | +491,520 (+15.00%) | + +The full seven-run instruction ranges were 25.172--25.507 billion for Go and +23.192--23.275 billion for LLGo. Cycle ranges were 6.709--7.295 billion and +5.696--5.861 billion respectively. Thus the direct static-call plus bounded +safepoint path now beats Go on both stable hardware counters; it is no longer +the general performance blocker. The LLGo compute payload adds only 640 bytes +to its pure-idle artifact. + +The fixed path is not finished. Pure LLGo startup still retires about 17.25 +million more instructions than Go and holds about 464 KiB more resident +memory. Also, 64 worker-named runtime symbols remain linked even when the +capability bit is zero: the change avoids their threads and operation source, +but the runtime's dynamic source-binding references prevent complete LTO code +elimination. The next fixed-cost work should profile scheduler/bootstrap +initialization and then make optional service code itself link-time removable; +it should not perturb the now-competitive static kernel. + +A same-source full-LTO `-tags=nogc` control isolates most of that fixed gap. +Seven runs gave 27,229,137 median retired instructions and 2,424,832-byte +median peak RSS; the stripped artifact was 897,552 bytes and had no `libgc` +dependency. Compared with the default coroutine build, removing BDWGC startup +saves about 13.45 million instructions and 1.34 MiB RSS. That accounts for +roughly 78% of LLGo's 17.25-million-instruction fixed deficit to Go. The +remaining `nogc` gap is about 3.80 million instructions (+16.2%), while its +RSS is about 0.83 MiB below Go. Default-GC lazy initialization/linkage is +therefore the primary fixed-cost target; scheduler bootstrap is a smaller, +separately measurable remainder. + +### Compact structural emission type graph checkpoint + +The same 2026-08-13 candidate removes a compiler-side exponential path that +was exposed by the complete cmd/cgo fixture. The previous structural emission +keys recursively serialized every occurrence of a type child. A shared +go/types DAG was therefore expanded as a tree, even though repeated acyclic +children carry no additional identity. Deep generic C and managed-interface +signatures could spend more time and memory constructing key strings than +emitting their LLVM body. + +One parameterized graph now serves all three required equivalence policies: +strict emission retains named identity, tuple names, and struct metadata; +strict ABI omits only tuple names; identity-free ABI also expands named types +and erases struct field metadata. Tarjan SCCs preserve recursive topology, +ordered root-local references encode cycles, and acyclic children contribute a +Merkle digest. Each public key is a schema-separated SHA-256 digest, so +incidental go/types pointer sharing no longer changes equality or output size. +This replaces the separate recursive builder rather than adding a second type +authority; the implementation and tests have a net growth of 71 lines. + +At depth 18, the adversarial shared-DAG benchmark completes in about 61 us with +76,880 bytes and 509 allocations per operation, while returning a fixed-length +key. Before the change, `cgofull` had not reached its IR check after more than +720 seconds. The compact graph reached that check in about 374 seconds; a +subsequent complete build-and-run gate passed in 616 seconds. The latter also +includes compilation, linking, and execution, so it is recorded as a +correctness gate rather than compared directly with the IR-only cutoff. + +A follow-up retains root digests in the owning `EmissionUniverse`. The cache is +mode-separated, canonicalizes aliases, and dies with that one compiler +session; it cannot pin go/types graphs from earlier requests in a process-wide +map. All production structural-key consumers use this boundary, including +interface descriptors, worker/cgo thunks, callable shadows, frame-retention +proofs, dispatch layouts, and archive effect summaries. A cache hit takes +about 318--560 ns with zero bytes and zero allocations on the loaded validation +host, versus about 0.8--1.1 ms, 76,880 bytes, and 509 allocations when rebuilding +the depth-18 graph. The affected semantic suite and the complete `cgobasic` +build/link/run fixture pass. + +The large `cgofull` fixture was not promoted as a post-cache timing result. Both +its IR-only and full paths entered the existing LLVM emission peak above the +1.6-GiB local validation ceiling and were terminated cleanly. The earlier +post-graph 616-second pass remains the correctness result; no claim is made +from incomplete, heavily host-contended wall-clock samples. + +### Closed direct-channel transaction checkpoint + +The next 2026-08-13 local candidate, still based on `914379fbd`, isolates the +remaining unbuffered handoff cost with `testdata/pure_handoff`. The fixture has +no imports or output and performs 100,000 request/ack round trips, hence 200,000 +successful unbuffered rendezvous. Go 1.26.5 and the LLVM 22.1.8 coroutine +compiler build the exact same source. Every LLGo variant uses an independent +cache plus `-a -trimpath -O3 -lto=full -ldflags='-s -w'`. + +The retained direct path is a closed transaction rather than a second channel +implementation: + +1. compiler lowering passes its exact hidden `*G` task to channel helpers; +2. zero-filled coroutine-frame allocation replaces redundant caller and + prepare aggregate stores; +3. `PrepareCurrentChannelParkCleanup` authenticates the compiler park window, + reserves and exposes one source operation, publishes the parked frame, and + binds typed cleanup in one transaction; catalog exhaustion returns an exact + zero-effect `NeedsCapacity` result; +4. after an hchan has detached an ordinary one-case waiter under its lock, the + same-P matcher acquires the private select claim without first taking the + generic producer lifetime admission; +5. an already published Ready fact upgrades that transaction to the complete + admission/mailbox path. After a successful physical commit, the source is + sealed before any frame detach. Producers admitted just before the seal are + joined through the owner-local FIFO; otherwise the common closed Apply tail + resolves, detaches, materializes, recycles, and promotes the peer inline. + +Multi-case select, cross-P matching, buffered channels, close, cancellation, +and every uncertain shape retain the durable source transaction. The fast path +therefore removes proof work only where hchan serialization plus the exact +current-P capability already supplies that proof; it does not weaken the +general producer ABI. + +Each intermediate change had its own seven-run instruction gate. Removing the +caller zero saved 0.382%, removing the duplicate prepare zero saved 0.139%, the +narrow compiler-task capability saved 1.027%, and the closed park preparation +saved 1.264% in their respective AB/BA campaigns. A shared struct-return park +helper was rejected after increasing instructions by 0.359%; LLVM did not +inline its three boundaries. These independent campaign ratios are not +compounded below. + +The final measurement rotates Go, the retained closed-park parent, and the +sealed same-P candidate through seven process runs. Retired instructions are +the primary metric because unrelated host load moved wall time by more than an +order of magnitude. Complete hardware-counter ranges are included. + +| Pure handoff metric | Go 1.26.5 | closed-park parent | sealed same-P candidate | +| --- | ---: | ---: | ---: | +| file bytes | 1,161,970 | 916,256 | 916,544 | +| retired instructions, median | 312,088,337 | 2,332,891,820 | 2,270,337,552 | +| retired instructions, range | 311,143,041--319,408,760 | 2,328,641,148--2,337,837,686 | 2,266,369,974--2,273,049,208 | +| cycles, median | 94,773,661 | 531,823,719 | 524,401,481 | +| cycles, range | 88,386,522--104,969,862 | 516,014,589--614,000,723 | 499,670,027--554,399,515 | +| peak RSS, median | 3,309,568 | 3,817,472 | 3,801,088 | + +Against the exact parent, the candidate retires 2.681% fewer instructions and +uses 1.396% fewer cycles. Its instruction ratio to Go narrows from 7.48x to +7.27x, while the cycle ratio narrows from 5.61x to 5.53x. Peak RSS drops one +16-KiB page versus the parent but remains 14.85% above Go. The stripped LLGo +artifact remains 21.12% smaller than Go; versus its parent it grows 288 bytes +and its Mach-O `__text` grows 1,524 bytes. No compiler-spilled channel-frame +layout changes. + +Correctness gates cover the capability boundaries, not only the benchmark +output. The ordinary direct transaction must leave admission count and linear +lease zero. A Ready race must upgrade to the generic admission and complete +source/claim/packet/frame recycling. A producer paused after admission but +before its physical store must observe the sealed source, release its lease, +and defer inline detach until quiescence. Capacity failure must leave all frame +and scheduler state untouched. The complete `runtime/internal/coro` suite, the +CI-shaped race/shuffle typed-hchan and owner-lock suite, and compiler native plus +WASM channel-lowering tests pass. + +This checkpoint is a real reduction, not completion of channel optimization. +About 11,350 instructions are still retired per successful rendezvous versus +about 1,560 for Go. The next gate should collapse the remaining ordinary +park/resolve/materialize metadata transitions under the same closed capability +or avoid suspending the immediately matched endpoint. General select and +cross-owner machinery should not be cloned into another fast runtime. + +#### Closed inline state transaction follow-up + +The retained follow-up executes that first next gate without introducing a +second channel state machine. The exact same-P, uncanceled, one-case capability +now performs one complete read-only preflight over the source slot, frozen park +descriptor, active/affected queue edges, result packet, and ready-queue header. +Only after every fallible observation succeeds does one no-fail write half +publish the canonical end states: source slot `Free`, claim `Open`, packet and +park `Materialized`, wait record detached, and peer `Runnable`. The transient +`Detaching`, `Ready`, `Consumed`, `Detached`, `Quiesced`, and `Taken` states are +not externally observable while producer ingress is sealed, its admission is +quiesced, the claim excludes another resolver, and the current P excludes +owner mutation. The general state machine remains authoritative for select, +cancellation, cross-P work, an admitted producer, and every failed fast-path +shape. + +This replaces repeated construction and validation of the same state; it does +not remove a lifecycle gate. A deterministic corruption test changes the +bound packet after the physical effect and verifies that failed preflight has +not detached, recycled, or promoted any owner state. Restoring the descriptor +then lets the same closed transaction reach its canonical terminal state. The +existing Ready-race admission upgrade and producer-before-seal join tests also +remain passing. + +The comparison below is a fresh seven-run rotation of Go, the immediately +preceding source-proof-tail candidate, and the closed inline candidate. It is +not mixed with the earlier checkpoint's host-counter session. + +| Pure handoff metric | Go 1.26.5 | source-proof-tail parent | closed inline candidate | +| --- | ---: | ---: | ---: | +| file bytes | 1,161,970 | 916,784 | 933,104 | +| retired instructions, median | 321,618,389 | 2,190,867,063 | 1,814,602,002 | +| retired instructions, range | 319,044,013--323,486,769 | 2,187,207,643--2,194,970,866 | 1,813,240,660--1,816,758,591 | +| cycles, median | 90,885,434 | 488,143,750 | 387,515,097 | +| cycles, range | 84,637,855--101,931,134 | 447,473,831--517,061,273 | 367,009,332--419,624,483 | +| peak RSS, median | 3,260,416 | 3,801,088 | 3,817,472 | + +Against the exact parent, the closed transaction retires 17.174% fewer +instructions and 20.615% fewer cycles. The instruction ratio to Go narrows +from 6.81x to 5.64x, and the cycle ratio narrows from 5.37x to 4.26x. The +candidate's median RSS is one 16-KiB page above its parent. Its `__text` grows +only 672 bytes, but an additional 256 bytes in the on-disk data/metadata +section crosses the next Mach-O 16-KiB segment-file alignment boundary; that +accounts for the apparent 16,320-byte file increase. The artifact remains +19.70% smaller than Go, with no G, wait-record, source-slot, or coroutine-frame +layout changes. + +The complete target-neutral runtime suite, the CI-shaped race/shuffle typed +hchan and owner-lock suite, and LLVM 22 native plus WASM32 compiler channel +lowering tests pass. The remaining roughly 9,073 instructions per rendezvous +versus Go's roughly 1,608 are no longer dominated by resolve/apply/materialize +validation. The next profile gate should separate unavoidable stackless +suspend/resume and scheduler handoff cost from hchan locking, compiler-emitted +park preparation, and the still-general producer-side transaction. An +immediately matched endpoint may avoid more work, but only if its no-suspend +hchan critical section can retain the same close/claim lifetime proof. + +#### Fused direct park preparation follow-up + +The next retained checkpoint applies the same closed-transaction rule to the +ordinary compiler-generated park half. `PrepareCurrentChannelParkCleanup` +first authenticates the exact G/P/frame/source relation, fresh frame storage, +reusable ParkState, source capacity, operation identity, and private direct +reservation. It then seals the selected producer generation before changing +owner state. Once that atomic begin succeeds, one no-fail write suffix builds +the final single-link `Parked` graph, committed wait record, cleanup packet, +frame transition, source cursor, and reserved external endpoint. Release +publication of the initialized generation and then the hchan endpoint are the +only visibility boundaries. + +This removes the hot chain through generic operation prepare, commit-mode +declaration, link attachment, post-build graph audit, and external-exposure +revalidation. Those APIs remain authoritative for multi-case select, pending +task cancellation, compatibility callers, and any non-private reservation. +The generation/admission protocol is unchanged. A deterministic stale- +reservation gate forces the generation begin to fail and verifies that the +ParkState, pending transition, frame link, wait/claim/packet/cleanup storage, +source scan limit, and reserve cursor all remain untouched. + +The table is a fresh seven-run candidate/parent/Go rotation using the same +LLVM 22.1.8 full-LTO build and `/usr/bin/time -lp` hardware counters as the +preceding checkpoint. + +| Pure handoff metric | Go 1.26.5 | closed inline parent | fused park candidate | +| --- | ---: | ---: | ---: | +| file bytes | 1,161,970 | 933,104 | 933,104 | +| retired instructions, median | 311,240,084 | 1,802,711,153 | 1,620,832,698 | +| retired instructions, range | 309,587,998--316,837,619 | 1,800,541,881--1,812,707,051 | 1,618,790,552--1,624,955,168 | +| cycles, median | 82,996,555 | 381,292,958 | 365,235,840 | +| cycles, range | 77,067,945--91,384,658 | 352,567,507--448,725,174 | 357,100,740--376,275,824 | +| peak RSS, median | 3,227,648 | 3,817,472 | 3,833,856 | + +Against the exact parent, the candidate retires 10.089% fewer instructions +and uses 4.211% fewer cycles. Its instruction ratio to Go narrows from 5.79x +to 5.21x and its cycle ratio from 4.59x to 4.40x. That is about 8,104 retired +instructions per rendezvous versus Go's 1,556 in this campaign. The stripped +file size is unchanged, Mach-O `__text` grows by 400 bytes, and median RSS +grows by one 16-KiB page. The artifact remains 19.70% smaller than Go and no +runtime or coroutine-frame layout changes. + +The target-neutral core suite, full race/shuffle core gate, native and WASM +typed-hchan adapter gates, and LLVM 22 native/WASM32 compiler lowering gate +pass. The remaining gap now needs a new profile: the old park-preparation +stack no longer describes the emitted transaction, so further optimization +must target measured hchan, scheduler, or suspend/resume cost rather than +guessing at another metadata layer. + +#### Hchan direct-shape certificate follow-up + +The typed hchan matcher already validates the dequeued coroutine operation, +waiter back-pointers, source route, payload size, direction, and terminal +status before it acquires the select claim. For the same open, unbuffered, +direct waiter, the hchan mutex then keeps queue linkage and channel state +stable across the existing no-suspend begin/effect/commit span. The retained +follow-up records that proof in the transient commit context and consumes it +at the later phase boundaries instead of calling the complete operation +validator and direct-shape recognizer again. Select, buffered, close, and every +caller without that certificate still perform the ordinary checks. No bit is +stored in the waiter, operation source, coroutine frame, or producer ABI. + +A fresh seven-run parent/candidate rotation measured: + +| Pure handoff metric | fused park parent | hchan certificate candidate | Change | +| --- | ---: | ---: | ---: | +| file bytes | 933,104 | 933,104 | 0 | +| retired instructions, median | 1,621,999,749 | 1,587,608,197 | -2.120% | +| retired instructions, range | 1,621,031,474--1,625,614,230 | 1,585,121,525--1,588,455,666 | | +| cycles, median | 366,395,409 | 362,085,650 | -1.176% | +| cycles, range | 354,485,472--371,605,891 | 352,947,743--379,396,740 | | +| peak RSS, median | 3,833,856 | 3,817,472 | -16,384 | + +Mach-O `__text` grows by 176 bytes. A separate fresh candidate/Go rotation +gave 1,586,785,281 versus 312,146,532 median retired instructions (5.08x) and +343,619,997 versus 81,370,142 median cycles (4.22x), or about 7,934 versus +1,561 retired instructions per rendezvous. The complete core and race/shuffle +gates, native/WASM typed-hchan gates, and LLVM 22 native/WASM32 lowering gate +remain passing. + +#### Issued-action scheduler and scalar-resume checkpoint + +The final 2026-08-14 optimization pass follows one exact same-owner channel +handoff from hchan commit through coroutine resume. The root cause was no +longer one expensive function. It was duplicated interpretation at each layer: +the scheduler rebuilt ready headers and compatibility context, the runtime +reopened execution ownership around each action, the completion path decoded +the same driver/route/source relation after hchan had already proved it, and +the compiler/runtime resume ABI returned an aggregate whose fields represented +one small finite outcome. + +The retained implementation keeps one managed execution lease across adjacent +actions, dispatches the already validated ready FIFO directly, and enters the +same-owner issued-completion branch before reconstructing compatibility +context. The exact V2 resume ABI consumes a single `uint32` word containing +outcome class and payload; the typed aggregate wrapper remains only as an +internal test/compatibility helper. OS-thread suspend bookkeeping is skipped +unless the task has actually acquired thread-lock depth. These are projections +of the existing scheduler, completion, and cancellation state machines, not a +second protocol. Multi-case select, cross-owner completion, cancellation, and +uncertain shapes still take the durable path. + +A review follow-up then restored the intended architecture boundaries without +changing the hot transaction. The shared park emitter now owns static state-ID +allocation and accepts one typed prepare callback, so fused channel try-or-park +does not expose `nextState`, instruction counters, or a `parkPrepared` mode in +the feature lowerer. Structural type-key memoization is a standalone +compilation-lifetime component rather than twelve new direct consumers of +`EmissionUniverse`. The architecture gate therefore remains at 296 direct +plan-authority observations, legacy Park steps remain zero, raw helper +physical-type observations remain zero, and physical-body capability access +drops from 35 to 33. The unconsumed external `frame_publish_v2` ABI was removed; +V3 is the sole current metadata-bearing publication ABI. + +The same no-import `pure_handoff` source performs 100,000 request/ack round +trips, or 200,000 successful unbuffered rendezvous. Both binaries are stripped; +LLGo uses LLVM 22.1.8 with `-O3 -lto=full`. The table is a fresh nine-run +same-host measurement after the architecture review. The first LLGo run and +first Go run show ordinary cold-start counter elevation; medians and complete +ranges are reported. + +| Pure handoff metric | Go 1.26.5 | LLGo coroutine | LLGo / Go | +| --- | ---: | ---: | ---: | +| file bytes | 1,161,970 | 932,752 | 0.803x | +| retired instructions, median | 308,400,621 | 347,676,747 | 1.127x | +| retired instructions, range | 308,281,359--313,822,784 | 347,611,493--350,025,044 | | +| cycles, median | 68,603,883 | 58,946,249 | 0.859x | +| cycles, range | 66,893,098--74,009,098 | 57,796,591--62,107,540 | | +| peak footprint, median | 2,245,136 | 2,769,208 | 1.233x | +| maximum RSS, median | 3,194,880 | 3,801,088 | 1.190x | + +LLGo still retires 12.7% more instructions because each rendezvous explicitly +advances a stackless continuation and its scheduler state. On this CPU that is +not a remaining throughput deficit: its median cycle count is 14.1% below Go, +and the stripped artifact is 19.7% smaller. Relative to the roughly +1.59-billion-instruction hchan-certificate checkpoint, the cumulative issued +action, lease, scheduler, completion, and resume work removes about 78% of the +remaining instructions. The previously reported broad “LLGo is slower than +Go” handoff gap is therefore closed for this single-executor core case. + +Build configuration is material. Rebuilding the same source with full LTO but +the default optimization level retired about 350.4 million instructions; with +both full LTO and `-O3`, it retired about 347.7 million. Disabling LTO produced +an 810,512-byte artifact but retired about 394.2 million instructions, roughly +13.4% more than the matched `-O3 -lto=full` binary. Cross-package inlining of +the compiler/runtime transaction is consequently a required performance +profile today, not merely a benchmark decoration. Making equivalent +optimization available without full LTO is future build-pipeline work; it is +not evidence of a missing scheduler mechanism. + +#### Stable-boundary scheduler review follow-up + +A final commit review found that the remaining retired-instruction gap was not +the LLVM coroutine operation itself. Four already-linearized facts were being +re-observed on every channel action: + +1. `ExecutionQuota.TryAcquire` was already the exact live concurrency gate, + but ready placement reloaded the GOMAXPROCS limit after every action. +2. The private issued-action interval authenticated P, G, driver and action + before `llvm.coro.resume`, but its adjacent return replayed the immutable + executor binding. +3. Direct-channel materialization published one complete runnable certificate, + but dequeue and continuation entry rechecked fields written and consumed by + that same owner-only transaction. +4. `readyDebt` semantically requires one already-materialized runnable to run + before any lower-priority source, yet the selector still traversed all idle-P + and producer probes before making that forced choice. + +The retained fix keeps those proofs at their actual stable boundaries. A +successful public `runtime.GOMAXPROCS` change performs one compiler scheduler +yield; the next bounded slice refreshes its cached placement epoch, while +`TryAcquire` continues to enforce shrink immediately. Issued resume consumes +the P-owned action instead of round-tripping a duplicate runtime value. The +direct continuation consumes the private materialized/dequeued certificate, +and an ordinary same-owner `readyDebt` takes a priority-preserving selector +shortcut. General commits, LockOSThread islands, source work, cancellation, +cross-owner completion and compatibility entries retain their complete gates. +An additional dispatch-capability parameter measured only 0.06% and was +discarded rather than retained as architectural surface. + +The final race review also found one producer-side read of the owner-only +direct-channel inbox tail. The MPSC publisher now observes only its atomic head +and immutable retained-ingress binding; the single consumer remains the sole +tail owner. A source-shape gate prevents that cursor ownership from leaking +back into the producer, and the complete target-neutral core race suite passes. +All measurements below use the corrected publisher. + +The following table is a fresh nine-run same-host rotation of the same stripped +100,000-round-trip binaries. It supersedes the preceding short-run checkpoint; +the build remains LLVM 22.1.8, `-O3 -lto=full`. + +| Pure handoff metric | Go 1.26.5 | LLGo coroutine | LLGo / Go | +| --- | ---: | ---: | ---: | +| file bytes | 1,161,970 | 916,784 | 0.789x | +| retired instructions, median | 316,755,514 | 334,224,188 | 1.055x | +| retired instructions, range | 316,415,548--318,241,479 | 333,610,648--336,123,375 | | +| cycles, median | 71,281,738 | 64,066,897 | 0.899x | +| cycles, range | 69,396,625--73,644,923 | 62,764,022--79,084,562 | | +| peak footprint, median | 2,228,752 | 2,785,592 | 1.250x | +| maximum RSS, median | 3,162,112 | 3,801,088 | 1.202x | + +The short process still includes LLGo's roughly 13.45-million-instruction GC +startup cost measured above. A 100,000,000-round-trip steady-state run therefore +provides the more useful transaction comparison: + +| Long pure handoff metric | Go 1.26.5 | LLGo coroutine | LLGo / Go | +| --- | ---: | ---: | ---: | +| retired instructions | 293,936,307,783 | 296,022,059,612 | 1.007x | +| cycles | 60,430,528,016 | 50,224,458,184 | 0.831x | +| peak footprint | 2,327,056 | 2,785,592 | 1.197x | +| maximum RSS | 3,342,336 | 3,801,088 | 1.137x | + +Thus steady retired instructions are within 0.7% of Go, while LLGo uses 16.9% +fewer cycles and its stripped artifact is 21.1% smaller. The remaining fixed +memory difference in these runs is about 0.45--0.61 MiB; parked-G slopes and +broader workloads remain separate measurements. The target-neutral runtime suites, coroutine +`cl` subset, full `internal/build` `TestCoro*`, architecture/directive gates, +bootstrap/runtime-link tests, cross-route channel/select, dynamic GOMAXPROCS, +LockOSThread suspension, native/WASM lowering and timer gates pass. The complete +`cl` command reached its explicit 15-minute limit in the unrelated testgo +`cursor` case; no full-suite pass is claimed. A full `ssa` run likewise has no +result beyond its previously reported explicit timeout. + +### Static interface and bounded worker-completion checkpoint + +The 2026-08-14 LLVM 22-only candidate removes allocation and handoff work from +the standard-library file/network path without changing its blocking policy. +Concrete-to-interface conversions now emit immutable package-local itabs; +constant indirect payloads use immutable backing instead of `AllocU`. C ABI and +large-result rewrites preserve all function-index call-site attributes, +including LLVM's `coro_elide_safe`. The native worker boundary passes its +reservation and nine argument words as scalars, initializes synchronous-fault +signal state once per physical worker, and uses one bounded between-job handoff +poller. Regular files remain worker operations because POSIX nonblocking mode +does not make their storage access nonblocking; sockets retain the poll/event +path. + +Immediately before a native executor arms its retained wait, it may also inspect +the exact worker source and spin for at most 32,768 pending-word observations. +This is an advisory latency window: publication still goes through the normal +mailbox, doorbell, source reducer, cancellation, and retained-wait protocol. +Expiry always falls back to that protocol. The source scan is bounded by the +existing high-water catalog limit and does not add another operation index or +completion queue. + +A controlled A/B changed only that executor spin budget from zero to 32,768. +The binaries differed by 16 bytes. Eleven AB/BA-interleaved runs used the same +LLVM 22 compiler, fresh caches, full LTO, `GOMAXPROCS=1`, and 500 operations: + +| Workload | zero budget median [range] | 32K budget median [range] | Delta | +| --- | ---: | ---: | ---: | +| cache-hot 4 KiB file round trips | 11.895 ms [10.805, 13.202] | 10.712 ms [9.857, 12.981] | -9.95% | +| loopback 4 KiB TCP echo round trips | 25.401 ms [24.063, 27.481] | 25.632 ms [23.776, 27.292] | +0.91% | + +The TCP movement is noise-sized and no TCP benefit is claimed. The bounded +file-path gain justifies retaining the window, while a future power/latency +campaign should tune or disable active polling for energy-constrained native +profiles. WASM and bare-metal profiles do not select this native-pipe file. + +The complete candidate was then compared with Go 1.26.5 using the same source +and another eleven-run AB/BA rotation: + +| Workload | Go median [range] | LLGo coroutine median [range] | LLGo / Go | +| --- | ---: | ---: | ---: | +| cache-hot 4 KiB file round trips | 1.254 ms [1.031, 1.351] | 9.804 ms [9.159, 11.327] | 7.82x | +| loopback 4 KiB TCP echo round trips | 9.967 ms [8.000, 14.629] | 21.728 ms [21.496, 23.886] | 2.18x | + +The stripped binaries are 1,797,730 bytes for Go and 7,944,320 bytes for LLGo +(4.42x). Runtime-core, C11 queue, ABI/CABI attribute, static-interface, +architecture-debt, native target-plan, representative native E2E, and actual +file/TCP execution gates pass. The full `internal/cabi` suite was deliberately +stopped after an unrelated fixture reached about 5 GiB RSS; the exact changed +CABI tests pass, so no full-suite result is claimed here. diff --git a/doc/llvm-coro-runtime-design.md b/doc/llvm-coro-runtime-design.md index f0918daab1..ad82ce4283 100644 --- a/doc/llvm-coro-runtime-design.md +++ b/doc/llvm-coro-runtime-design.md @@ -2013,14 +2013,14 @@ Pure sync library/archive不需要链接scheduler。Executable一旦选择 `-sch 当前落地状态（2026-07-29，实验 physical ABI v0/v1；scheduler ABI 已扩展到 `llgo.coro.scheduler.program-bootstrap.v2.closed-static-spawn.v0`）： - 全程序 SSA 的 Effect、Demand、FuncRep、稳定 FunctionID、精确 emission universe、单 primary symbol 选择和 `CoroPlanDigest` 已落地。明确 plain 或 coro 的函数仍只有一个主体；仅真正动态的 func/`any`/interface consumer 才进入 descriptor/dispatch。缺失、过期或目标布局不匹配的计划与 cache manifest 均 fail closed。 -- LLGo 已固定使用 `cpunion/llvm` PR #5 的 LLVM 19–22 绑定。该分支吸收上游 LLVM 22 的完整 switch API 变更，并保留 LLGo 所需的 switched-resume builder/CoroSplit API；19、20、21、22 CI 均通过。LLGo 不再覆盖 LLVM 19 以下版本。 +- LLGo 已固定使用 `cpunion/llvm` 的 LLVM 22-only 绑定。该分支吸收上游 LLVM 22 的完整 switch API 变更，并保留 LLGo 所需的 switched-resume builder/CoroSplit API；编译标签、工具链配置和当前 CI 均不再维护 LLVM 21 及以下版本。 - closed static `CallDirect + DirectCoro` 已使用 caller-frame await：父 frame 按 Go 从左到右顺序求值参数、保存 typed result slot、创建 initial-suspended child，然后由 scheduler 独占 resume/done/destroy。值传输已覆盖 pointer、uintptr、function、string、slice、named struct、fixed array 和多返回值；不是仅支持 scalar。 - exact pure-SSA physical audit 已覆盖 stack alloc、local/global typed load/store、`FieldAddr`、static `IndexAddr`/`Index`、完整 fixed-array slice、`Field`/`Extract`/`Phi`、empty-interface direct value、受限 conversion/binop/unop、`len`/`cap`。heap escape、需要 allocation 的 interface box、slice 动态越界检查、pointer-containing global store、closure/type assertion/dynamic call 和任何隐藏 runtime helper 仍明确拒绝。 - `program-bootstrap.v2` 在 codegen 前冻结五阶段表：`[internal runtime.init, init$abitypes, public runtime.init, selected main-package init, main.main]`。managed Go 阶段根据唯一 primary 选择 `DirectPlain` 或 `CoroRoot`；public runtime init 若存在则必须使用其 exact managed body，不存在时才由 compiler 生成 no-op。Coro 表项只绑定 package anchor/descriptor index，不复制函数体，也不把 catalog 当启动列表。 - planner 已把 internal runtime init、selected package init 和 `main.main` 注入 managed demand。普通同步 Go/标准库调用风格不变，调用者根据精确 effect 自动被染成 coro；scheduler-stack hook closure 则是单独审计的 NoSuspend island，不能通过强改 demand 或放宽 trusted closure 绕过。 - frozen foreign `//llgo:coro noblock` certificate 当前只授予已审计的 `time`、`pthread_self`、`pthread_mutex_init` 和 `pthread_mutex_unlock`。证书只移除未知阻塞，`IRQUnsafe` 仍保留但允许在普通 G 上执行。真实 runtime init 仍被 `pthread_key_create`、`rand`/`srand`、`GC_malloc`、mutex lock、Memcpy/Memset 等未完成边界挡住。 - legacy PanicABI 仍是完整启动链的正式 blocker。exact proof 可追踪 `runtime.Panic → Rethrow → TracePanic → printany`，并在动态 `error.Error` 调用处停止；不能把该动态调用误标为 plain。新的 `llgo.coro.panic.explicit-status.v0` 已进入 digest、summary、cache、manifest 和 package/root ABI hash；`cl` 已有下述严格子集的 compiler lowering，但 `internal/build` 仍保留 target-wide 全局 gate，尚不能把它作为完整程序的 production PanicABI 开启。 -- 多基本块 CFG、聚合值、PHI 和抢占 lowering 已完成。自然循环、循环入口及每 64 条有效指令的长直线块插入 poll；scheduler 的 P 级原子 request 只有在 slow path 才执行 publish/yield/`llvm.coro.suspend`，fast path 不切换。LLVM 19–22 上均有 native64/wasm32 pre-/post-CoroSplit 与 object 测试。 +- 多基本块 CFG、聚合值、PHI 和抢占 lowering 已完成。自然循环、循环入口及每 64 条有效指令的长直线块插入 poll；scheduler 的 P 级原子 request 只有在 slow path 才执行 publish/yield/`llvm.coro.suspend`，fast path 不切换。LLVM 22 有 native64/wasm32 pre-/post-CoroSplit 与 object 测试。 - 第一条 production `go` 路径已经落地：严格限定为 closed static、top-level、非捕获、非泛型、非变参、零返回的 `go f(args)`。编译器先按 Go 顺序完整求值参数，再以显式 parent G 执行 begin，调用 target 唯一的 `DirectCoro` primary 到 LLVM initial suspend，commit 后在 parent 上 poll/yield；runtime 不接收用户 callback，也不依赖 TLS。owner 与 target 都由精确 `YieldOnly` seed 进入 effect 传播，因此 target 即使当前很短也保留抢占点，普通同步 caller 则透明 await 同一主体。 - Command `main` 的正常 continuation 现在显式通知 runtime。main root 完成后，single-P shutdown 先整体校验 ready/wait/current/action 状态，再封闭调度 gate，按 FIFO 取 ready G、按 active-child 到 root 顺序直接 `llvm.coro.destroy`，最后每个 task storage 只释放一次。该 v1 路径只接收 `YieldOnly|AwaitStructured` target 且拒绝非空 wait set；panic/Goexit 不经过正常 main-return hook。 - ExplicitStatus runtime core 已有 task-local 两字 `PanicRecord`、原子 once publication和无 TLS 的 `__llgo_coro_panic_prepare_v1(g, handle, header, typeWord, dataWord)`。compiler 对PhysicalABIV1 body生成 `SuspendPanic`/`FinalSuspended`，panic与normal return共享LLVM final suspend；managed child先发布parent-owned completion并destroy，parent再执行显式cleanup/recover，最终未恢复路径从深到root直接destroy。每个descriptor现携带逻辑function/file，header携带source line；destroyed frame allocation被无新增分配地转移到G-owned trace chain，recover/Goexit/replacement同步清零释放。普通嵌套child replacement由await祖先recover scope推导，只有cleanup内联且same-payload不可判别的替换使用一次compiler语义钩子。Native V2 entry验证canonical Panic tuple后调用no-return reporter，真实LLGo用例已输出`inner -> outer -> main`并以状态2退出；四种O0/O2×DWARF on/off caller acceptance均通过。implicit language fault也接入同一显式结果：无操作数的nil、bounds、channel和unsafe fault使用allocation-free V1静态payload；需要现场值的slice-to-array失败使用V2的两个target-width operand，由异常路径分配可跨LLVM frame destroy存活的GC可见`boundsError`。V1/V2及trace replacement hook都是required raw/plain runtime root并进入bootstrap/physical ABI identity；尚未定义的parameterized kind或不匹配operand必须fail closed。当前证据仍只覆盖coroutine-frame logical trace；plain activation、用户`Error/String`完整格式、WASM/embedded terminal presentation及完整GOROOT panic矩阵仍是后续门槛。 @@ -2079,7 +2079,7 @@ Pure sync library/archive不需要链接scheduler。Executable一旦选择 `-sch storage，exported runtime hook同时进入LLVM optimizer与final-link retention。 file/pipe/TCP/time同步风格acceptance、select合法并发输出、native defer/panic、 multi-executor fleet/channel shutdown以及C2 errno IR均已在本地分层通过；完整 - `test/*`、Go 1.26 GOROOT、跨LLVM 19–22和各target CI仍是合并前验收，不得由这些 + `test/*`、Go 1.26 GOROOT、LLVM 22和各target CI仍是合并前验收，不得由这些 focused结果外推为完整标准库兼容。 - host complete-operation网络族现以同一个8-word以内的typed recipe覆盖 read/write/accept/connect/recvfrom/sendto/recvmsg/sendmsg；deadline metadata只选择 @@ -2130,7 +2130,7 @@ Pure sync library/archive不需要链接scheduler。Executable一旦选择 `-sch `test/*`和GOROOT不得存在unexpected failure。 - compiler的所有现有initial、child-await、yield和legacy-park resume边已接入terminating dispatch gate。zero-ticket路径调用scalar `__llgo_coro_run_decision_take_zero_v1(g) uint32`，正常值进入唯一normal continuation，Abort/Shutdown在cleanup lowering完成前进入共享trap而不会误执行用户continuation；full ticket/lease ABI继续供bootstrap与未来park-site reconciliation使用。同一LLVM/target的gate开关对照证明scalar gate不会增加stackless coroutine frame，CoroSplit ramp/destroy也没有可达gate。 - 两字Operation identity已冻结为`source:8/route:9/local:15 + generation:32`，保持size 8、align 4。route按runtime instance单调分配且永不复用，关闭后保留永久tombstone；Manual/TaskControl ingress的producer lease覆盖`source.Post -> executor.Request`完整tail，strong join后才允许清除source/executor pointer；Timer V2 reserve、publish、Apply和result lease也验证exact route/local/generation。52-byte pointer-free `ResumePacket`已覆盖零/单Timer、Manual、Poll、Worker以及经typed plan物化的direct Channel/multi-case select，HostOp deadline与keyed/private-registry cleanup也复用同一POD plan。P-neutral global injection/work sharing已在1/4/8 route core生命周期以及1/8 P真实linked E2E验证；bounded half-batch又覆盖非neutral前缀、mailbox余量、并发claim、close join和零分配prepared storage。Command root用G尾部padding中的一字节`AnyOwner/CurrentOwner` mobility保持program-P边界；这不增加G大小、P指针或registry，也不等于完整thread/realm affinity。运行期逻辑execution quota和locked-M同P replacement handoff均已完成：active resume先变为私有rooted的`GForeignWaiting`，初始8项inline、其余64项稳定分页的10,000 logical-slot M目录以generation/owner-epoch baton把active route从parent scalar slot切到replacement，replacement复用原driver/source/common reducer并服务channel、timer和poll；原C返回后request/ring、确认exact Returned owner并将raw M放入8项standby cache；C factory仅按实际pthread分配record，并用不复用的单调token拒绝旧句柄，随后重获permit并恢复exact resume。未解锁G退出通过Action padding retire bit和clean-factory succession覆盖program、peer及temporary replacement lineage；TLS destructor E2E证明旧pthread退出，main-return遇不可取消locked C按进程终止语义退出而不join。统一physical-thread ledger覆盖program/factory/worker/peer/replacement/successor/standby，标准`runtime/debug.SetMaxThreads`已通过C lifecycle、14线程上限replacement和普通stdlib同步调用。`GOMAXPROCS=1`下同route channel、timer、真实socket poll及nested M0→M1→M2 linked E2E均已通过。尚未完成的是完整affinity以及标准库/GOROOT的locked syscall/callback/reentry矩阵；并发local deque保留为profiling驱动的可选优化，不再是正确性gate。 -- 第一个标准库同步风格原型已以GOROOT source patch实现`time.Sleep`：普通`time.Sleep(d)`被Effect分析自动传播为`DirectCoro/AwaitStructured`，不修改public signature，不依赖libuv、BDWGC、pthread producer或用户goroutine。真实linked native+nogc E2E已编译production runtime island，实际等待30ms并恢复原frame；timer/wake路径由monotonic clock与pipe/poll/fcntl实现，符号审计确认不依赖libuv、BDWGC或pthread producer。另一focused production-overlay测试直接读取真实注入的`time.Sleep`源，不用测试effect seed，验证跨包同步caller染色、frame证书和CoroSplit，但不声称链接执行标准库`time.Sleep`。LLVM 19–22都跑该契约，Go 1.24跑真实linked E2E，Go 1.26也跑production overlay分析/codegen。 +- 第一个标准库同步风格原型已以GOROOT source patch实现`time.Sleep`：普通`time.Sleep(d)`被Effect分析自动传播为`DirectCoro/AwaitStructured`，不修改public signature，不依赖libuv、BDWGC、pthread producer或用户goroutine。真实linked native+nogc E2E已编译production runtime island，实际等待30ms并恢复原frame；timer/wake路径由monotonic clock与pipe/poll/fcntl实现，符号审计确认不依赖libuv、BDWGC或pthread producer。另一focused production-overlay测试直接读取真实注入的`time.Sleep`源，不用测试effect seed，验证跨包同步caller染色、frame证书和CoroSplit，但不声称链接执行标准库`time.Sleep`。历史开发期曾在LLVM 19–22运行该契约；当前验收只运行LLVM 22，Go 1.24跑真实linked E2E，Go 1.26也跑production overlay分析/codegen。 - Phase 22 仍是有界prototype，不是完整`time`runtime：第65个同时live timer会按fail-stop ABI终止，尚需dynamic/sharded table和heap；`Timer`/`Ticker`/`AfterFunc`仍使用legacy libuv路径；`f := time.Sleep`、interface/reflect和dynamic dispatch还没有end-to-end callable coroutine descriptor；`Sleep(0)`/负值在Sleep体内不注册timer，但value-insensitive caller仍会创建并await child frame，尚需conditional effect或call-site fast path才能避免可观测的多余handoff。完整`Do`标准库构建现在先被`sync.Pool` TLS destructor的捕获闭包挡住：exact同步C callback ABI没有closure context slot，不能直接放宽。后续需改成显式`owner/local` TLS state，并同时为`tls.Handle[T]`经`Pool.local`的unsafe transport建立字段级whole-program证书。这里最后一句是Phase 22历史边界：当前JS command timer与WASI Preview 1 command timer/file source已经运行，RTOS、baremetal及WASI Preview 2仍未完成。 - wait/preempt core 要求目标提供可靠的 32-bit atomic load/store/CAS。WASM 可直接满足；带 A 扩展的 RISC-V 可满足；ESP32-C3 RV32IMC 当前会在链接时缺少 `__atomic_*_4`，直到平台用 IRQ critical section 提供单核适配。这里故意不使用非原子 fallback。 - `wasip1`、`wasip2` 和 `wasm-unknown` 默认仍明确选择 leaking/nogc frame backend，不依赖 libuv 或 BDWGC；另有显式`wasip1-gc`选择单executor tinygogc conservative档。后者已用真实`runtime.GC`在child timer frame suspended期间执行两轮collection和8192个同尺寸压力分配，随后恢复并验证两级pointer graph；产物由Wasmtime运行且无`GC_*`符号。它没有改变默认target，也不允许WASI threads/precise GC。`wasip2` 与 `wasm-unknown` 已通过真实 `llgo build -target=...`、wasm magic/symbol closure、无 `GC_*`/undefined 检查，并由 wasmtime 运行返回 0。普通`wasip1` executable现在固定为单线程、自有并导出memory的command profile；`-nostartfiles`保证LLGo `_start`拥有startup，并在managed initial slice返回后进入固定64-operation C reactor。该reactor不分配、不建立第二个scheduler，以一个`poll_oneoff`集合等待absolute alarm和fd read/write，已运行并发timer及`syscall.Open/Read/Write/Seek/Close/Unlink`。Preview 1网络明确`ENOSYS`，`path_open`/unlink/metadata的同步host import只具备Degraded compensation，不能保证慢host filesystem期间其他G前进；高层`os.File`还受通用interface exact-effect缺口阻挡。当前`wasip2`产物仍只是Preview 2目标的core module，不是WIT component，也没有对应pollable component lifecycle。 diff --git a/go.mod b/go.mod index 74879e363d..84fab96fc7 100644 --- a/go.mod +++ b/go.mod @@ -27,4 +27,4 @@ require ( replace github.com/goplus/llgo/runtime => ./runtime -replace github.com/xgo-dev/llvm => github.com/cpunion/llvm v0.9.4-0.20260810084926-4779f6f20229 +replace github.com/xgo-dev/llvm => github.com/cpunion/llvm v0.9.4-0.20260814091229-d916b8fd9c7e diff --git a/go.sum b/go.sum index 3320e993f5..0e46baed64 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ -github.com/cpunion/llvm v0.9.4-0.20260801090257-a37656e95932 h1:E04IKl70srRZmLvzHz6WPLHkbyjCwBKQu502h/KHQhQ= -github.com/cpunion/llvm v0.9.4-0.20260801090257-a37656e95932/go.mod h1:42vav2/cI5BAIcL543DZSMO9do8/aCK2z7JERH+AE+M= -github.com/cpunion/llvm v0.9.4-0.20260810084926-4779f6f20229 h1:C1t7HZS9F0H3vhEd2qOtQ0o5kVFUciB0Ja9/FcAAceQ= -github.com/cpunion/llvm v0.9.4-0.20260810084926-4779f6f20229/go.mod h1:42vav2/cI5BAIcL543DZSMO9do8/aCK2z7JERH+AE+M= +github.com/cpunion/llvm v0.9.4-0.20260814091229-d916b8fd9c7e h1:RU3ctn2nFvusFVb79+DanxMHWZIQvwu4lRoKi+JGeFY= +github.com/cpunion/llvm v0.9.4-0.20260814091229-d916b8fd9c7e/go.mod h1:42vav2/cI5BAIcL543DZSMO9do8/aCK2z7JERH+AE+M= github.com/creack/goselect v0.1.2 h1:2DNy14+JPjRBgPzAd1thbQp4BSIihxcBf0IXhQXDRa0= github.com/creack/goselect v0.1.2/go.mod h1:a/NhLweNvqIYMuxcMOuWY516Cimucms3DglDzQP3hKY= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -26,8 +24,6 @@ github.com/qiniu/x v1.18.0 h1:iMfc7Gqy1au+akr+Tl5Z40px7TR8VBLLkJsIeajKIbc= github.com/qiniu/x v1.18.0/go.mod h1:Sx3Wy+0GI9OsX4a53mYj6A0o7mHJ94PUvraqGYb4EIs= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/xgo-dev/llvm v0.9.6 h1:DtjcgENgDItbf7LyUukssMwnnWXDpERSzGQ9jRQ1CRM= -github.com/xgo-dev/llvm v0.9.6/go.mod h1:42vav2/cI5BAIcL543DZSMO9do8/aCK2z7JERH+AE+M= github.com/xgo-dev/plan9asm v0.3.5 h1:886BmpjMK6JfJ03VWA3nPK01jkVZA1a7/mZia3BOsdg= github.com/xgo-dev/plan9asm v0.3.5/go.mod h1:0yM4CCIp2PyT8h+Ro3Ukro3lHL8ji9mzHEv5yfhOckc= go.bug.st/serial v1.6.4 h1:7FmqNPgVp3pu2Jz5PoPtbZ9jJO5gnEnZIvnI1lzve8A= diff --git a/internal/abi/large.go b/internal/abi/large.go index 44bb23e843..86d50bf7a4 100644 --- a/internal/abi/large.go +++ b/internal/abi/large.go @@ -76,7 +76,6 @@ func (l largeAggregateLowerer) transformCall(m llvm.Module, call llvm.Value) { result := l.allocResult(m, ctx, b, retType) params := make([]llvm.Value, 1, oldType.ParamTypesCount()+1) params[0] = result - reflectMethodByName := call.GetCallSiteStringAttribute(-1, "llgo.reflect.methodbyname") reflectNameParam := -1 for i := 0; i < oldType.ParamTypesCount(); i++ { params = append(params, call.Operand(i)) @@ -85,13 +84,13 @@ func (l largeAggregateLowerer) transformCall(m llvm.Module, call llvm.Value) { } } newCall := llvm.CreateCall(b, newType, call.CalledValue(), params) + for _, attr := range call.GetCallSiteAttributesAtIndex(-1) { + newCall.AddCallSiteAttribute(-1, attr) + } newCall.AddCallSiteAttribute(1, sretAttribute(ctx, retType)) for i := 0; i < oldType.ParamTypesCount(); i++ { copyClosureContextCallSiteAttributes(call, i+1, newCall, i+2) } - if !reflectMethodByName.IsNil() { - newCall.AddCallSiteAttribute(-1, reflectMethodByName) - } if reflectNameParam >= 0 { newCall.AddCallSiteAttribute(reflectNameParam, ctx.CreateStringAttribute( "llgo.reflect.methodbyname.name", "1", diff --git a/internal/abi/large_test.go b/internal/abi/large_test.go index 0a9373dfcb..9ab8c2c3b3 100644 --- a/internal/abi/large_test.go +++ b/internal/abi/large_test.go @@ -78,7 +78,7 @@ entry: ret %Small %value } -attributes #0 = { "llgo.reflect.methodbyname"="value" } +attributes #0 = { nounwind "llgo.reflect.methodbyname"="value" } attributes #1 = { noinline } ` @@ -135,6 +135,18 @@ attributes #1 = { noinline } if !strings.Contains(mod.String(), `"llgo.reflect.methodbyname"="value"`) { t.Fatalf("reflect MethodByName call marker was not preserved:\n%s", mod.String()) } + callerFn := mod.NamedFunction("caller") + var loweredCall llvm.Value + for block := callerFn.FirstBasicBlock(); !block.IsNil(); block = llvm.NextBasicBlock(block) { + for instruction := block.FirstInstruction(); !instruction.IsNil(); instruction = llvm.NextInstruction(instruction) { + if call := instruction.IsACallInst(); !call.IsNil() && call.CalledValue().Name() == "callee" { + loweredCall = call + } + } + } + if loweredCall.IsNil() || loweredCall.GetCallSiteEnumAttribute(-1, llvm.AttributeKindID("nounwind")).IsNil() { + t.Fatalf("large-return call lowering lost a function-index call attribute:\n%s", callerFn.String()) + } for _, name := range []string{"wrapper", "indirect", "self_copy"} { ir := mod.NamedFunction(name).String() diff --git a/internal/build/build.go b/internal/build/build.go index 8d23d9511a..f20ea4db38 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -1040,7 +1040,7 @@ func (in CoroPlanInput) Analyze(roots coro.Roots, config coro.SSAConfig) (*coro. // implementation. A may-block callable is admitted only when this // exact member belongs to the compiler-owned raw host/scheduler-stack // island. - const supportedExec = coro.MayUnwind | coro.NeedsCleanupFrame | coro.IRQUnsafe + const supportedExec = coro.MayUnwind | coro.NeedsCleanupFrame | coro.IRQUnsafe | coro.NeedsRuntimeContext allowedExec := coro.ExecFlags(supportedExec) callableWaitsForeign := false callableNoReturn := false @@ -1123,6 +1123,27 @@ func (in CoroPlanInput) Analyze(roots coro.Roots, config coro.SSAConfig) (*coro. if !frozen { return false, fmt.Errorf("call %q in %q is absent from the frozen ProgramIR", call.String(), caller.Name()) } + if frontend.Elision == cl.CoroCallElidedFrontendUnevaluated { + // Preserve the ProgramIR decision when the omitted instruction also + // satisfies AnalyzeSSA's narrow static-call gate. Generated cgo adapters + // place _Cgo_use/_Cgo_keepalive behind runtime.cgoAlwaysFalse; their + // blocks are absent from physical lowering and must not reappear as raw + // closure edges. Dynamic calls, invokes, builtins, spawns, and alternate + // defer stacks remain conservative until the analyzer has a complete + // frontend-unevaluated instruction projection for those shapes. + common := call.Common() + if common == nil || common.StaticCallee() == nil || common.IsInvoke() { + return false, nil + } + switch exact := call.(type) { + case *ssa.Call: + return exact != nil, nil + case *ssa.Defer: + return exact != nil && exact.DeferStack == nil, nil + default: + return false, nil + } + } if requested && !frontend.ElidesCall() { return false, fmt.Errorf("builder cannot elide ordinary call in %q; only calls omitted by the build frontend may be elided", caller.Name()) } @@ -2293,7 +2314,8 @@ func validateLiveCoroRawABIPlainClosure(plan *coro.SSAPlan, raw *coroRawABIPlain return fmt.Errorf("live raw ABI plain closure validation requires an exact closure") } const managedOnlyEffects = coro.YieldOnly | coro.AwaitStructured | coro.OutcomeStructured - const legacyExec = coro.IRQUnsafe | coro.NeedsPreempt | coro.MayUnwind | coro.NeedsCleanupFrame | coro.NoReturn | coro.PanicOnly + const legacyExec = coro.IRQUnsafe | coro.NeedsPreempt | coro.MayUnwind | coro.NeedsCleanupFrame | + coro.NoReturn | coro.PanicOnly | coro.NeedsRuntimeContext validateTarget := func(owner, target *ssa.Function, terminalOnly bool, site string) error { if target == nil { @@ -4237,6 +4259,19 @@ func buildCoroPlan(ctx *context, packages ...*aPackage) error { CoroLibraryEffects: maps.Clone(importedLibraryEffects), CoroLibraryForeignCallables: maps.Clone(libraryForeign), } + if ctx.coroEmission != nil && ctx.coroEmission.CompleteRuntimeABI() { + programCapabilities, err := ctx.clCompilation.CoroProgramCapabilities() + if err != nil { + ctx.coroPlan = nil + ctx.coroPlanDigest = "" + ctx.coroPlanMetadata = coro.PlanDigestMetadata{} + ctx.coroLoweringFacts = coro.LoweringFacts{} + ctx.coroLoweringFactsDigest = "" + ctx.clCompilation = nil + return fmt.Errorf("freeze coroutine program capabilities: %w", err) + } + ctx.coroProgramCapabilities = programCapabilities + } if ctx.prog != nil { ctx.prog.SetLogicalLocality(true) } @@ -5190,6 +5225,8 @@ func requiredCoroProgramRuntimePlanWithLibrary( names = append(names, "__llgo_coro_frame_alloc_v1", "__llgo_coro_frame_publish_v1", + "__llgo_coro_frame_publish_v3", + "__llgo_coro_frame_destroy_commit_v2", "__llgo_coro_await_prepare_v1", "__llgo_coro_preempt_poll_v1", "__llgo_coro_yield_prepare_v1", @@ -5215,13 +5252,13 @@ func requiredCoroProgramRuntimePlanWithLibrary( // plain runtime island as the raw channel hooks below. "CoroChanTrySend", "CoroChanTryRecv", - "CoroChanTryClose", + "CoroChanTryCloseTask", "CoroChanSelectTry", "CoroChanSelectPark", "CoroChanSelectResume", - coroChanSendParkSymbolV1, - coroChanRecvParkSymbolV1, - coroChanResumeSymbolV1, + coroChanSendTryParkSymbolV2, + coroChanRecvTryParkSymbolV2, + coroChanResumeSymbolV2, "__llgo_coro_fault_prepare_v1", "__llgo_coro_fault_prepare_v2", ) @@ -5687,22 +5724,28 @@ func requiredCoroProgramRuntimePlanWithLibrary( } } } - if name == coroChanSendParkSymbolV1 || name == coroChanRecvParkSymbolV1 { + if name == coroChanSendTryParkSymbolV2 || name == coroChanRecvTryParkSymbolV2 { sig := fn.Signature - if sig == nil || sig.Recv() != nil || sig.Variadic() || sig.Params().Len() != 7 || sig.Results().Len() != 0 || + if sig == nil || sig.Recv() != nil || sig.Variadic() || sig.Params().Len() != 9 || sig.Results().Len() != 1 || + !types.Identical(sig.Results().At(0).Type(), types.Typ[types.Uint32]) || typeParamLen(sig.TypeParams()) != 0 || typeParamLen(sig.RecvTypeParams()) != 0 || len(fn.FreeVars) != 0 { - return nil, nil, nil, nil, fmt.Errorf("coroutine channel park ABI %q must have exact func(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uintptr) signature", name) + return nil, nil, nil, nil, fmt.Errorf("coroutine channel try-or-park ABI %q must have exact func(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uintptr, uint32, uint32) uint32 signature", name) } for parameter := 0; parameter < 6; parameter++ { if !types.Identical(sig.Params().At(parameter).Type(), types.Typ[types.UnsafePointer]) { - return nil, nil, nil, nil, fmt.Errorf("coroutine channel park ABI %q must use unsafe.Pointer for parameter %d", name, parameter) + return nil, nil, nil, nil, fmt.Errorf("coroutine channel try-or-park ABI %q must use unsafe.Pointer for parameter %d", name, parameter) } } if !types.Identical(sig.Params().At(6).Type(), types.Typ[types.Uintptr]) { - return nil, nil, nil, nil, fmt.Errorf("coroutine channel park ABI %q must use uintptr element size", name) + return nil, nil, nil, nil, fmt.Errorf("coroutine channel try-or-park ABI %q must use uintptr element size", name) + } + for parameter := 7; parameter < 9; parameter++ { + if !types.Identical(sig.Params().At(parameter).Type(), types.Typ[types.Uint32]) { + return nil, nil, nil, nil, fmt.Errorf("coroutine channel try-or-park ABI %q must use uint32 for parameter %d", name, parameter) + } } } - if name == coroChanResumeSymbolV1 { + if name == coroChanResumeSymbolV2 { sig := fn.Signature if sig == nil || sig.Recv() != nil || sig.Variadic() || sig.Params().Len() != 2 || sig.Results().Len() != 1 || !types.Identical(sig.Params().At(0).Type(), types.Typ[types.UnsafePointer]) || @@ -6644,6 +6687,10 @@ type context struct { coroPlanMetadata coro.PlanDigestMetadata coroLoweringFacts coro.LoweringFacts coroLoweringFactsDigest string + // coroProgramCapabilities is the closed-world projection of optional + // physical runtime services. It is frozen by cl preflight before bootstrap + // selection and survives only as hashed entry-module flags. + coroProgramCapabilities coro.ProgramCapabilities // Frozen immediately after whole-program analysis, before package codegen. // linkMainPkg only consumes these exact per-entry-package tables. coroProgramBootstraps map[string]*coroProgramBootstrapV1 @@ -7975,19 +8022,20 @@ func compilePackageModule(ctx *context, aPkg *aPackage, externs []string, verbos mod.SetDataLayout(ctx.prog.DataLayout()) mod.SetTarget(ctx.prog.TargetSpec().Triple) stageBackend := shouldStageNativeExecutableBackend(ctx) + wholeProgramCoroLTO := shouldDeferCoroLoweringToFullLTO(ctx) // Coroutine splitting is a mandatory correctness pass, not an optimization. // In particular, native debug builds intentionally skip the default // optimization pipeline, but TargetMachine cannot select unresolved // llvm.coro.* operators. ModeGen deliberately retains frontend coroutine IR // for golden/LIT inspection and never reaches object emission here. - if ctx.mode != ModeGen && !stageBackend { + if ctx.mode != ModeGen && !stageBackend && !wholeProgramCoroLTO { if err := lowerCoroPackageModule(ctx, pkgPath, mod); err != nil { return err } } // Run the default LLVM optimization pipeline selected by the requested -O level. - if ctx.passOpt && !stageBackend { + if ctx.passOpt && !stageBackend && !wholeProgramCoroLTO { pbo := gllvm.NewPassBuilderOptions() defer pbo.Dispose() if err := gllvm.VerifyModule(mod, gllvm.ReturnStatusAction); err != nil { @@ -8006,7 +8054,11 @@ func compilePackageModule(ctx *context, aPkg *aPackage, externs []string, verbos if !stageBackend { emitFuncInfoEntrySites(ctx, ret) if ctx.mode != ModeGen { - if _, err := llssa.VerifyCoroAtomicCostModule(mod); err != nil { + verifyAtomicCost := llssa.VerifyCoroAtomicCostModule + if ctx.passOpt && !wholeProgramCoroLTO { + verifyAtomicCost = llssa.VerifyOptimizedCoroAtomicCostModule + } + if _, err := verifyAtomicCost(mod); err != nil { return fmt.Errorf("verify package %s final atomic-cost certificates: %w", pkgPath, err) } } @@ -8092,6 +8144,17 @@ func compilePackageModule(ctx *context, aPkg *aPackage, externs []string, verbos const coroPackageLoweringPipeline = "coro-early,cgscc(coro-split),coro-cleanup" +// shouldDeferCoroLoweringToFullLTO preserves presplit coroutine bodies and +// exact static call edges until LLVM owns the complete program. Package-level +// coro-cleanup irreversibly erases the coro.id information required by both +// HALO and LLVM 22's coro_elide_safe/.noalloc protocol; the full-LTO backend's +// mandatory pipeline performs CoroEarly, CoroSplit, annotation elision, and +// CoroCleanup after all package bitcode has been combined. +func shouldDeferCoroLoweringToFullLTO(ctx *context) bool { + return ctx != nil && ctx.mode != ModeGen && ctx.buildConf != nil && + ctx.buildConf.ltoMode() == lto.Full +} + func lowerCoroPackageModule(ctx *context, pkgPath string, mod gllvm.Module) error { if ctx == nil || ctx.prog == nil || mod.IsNil() { return fmt.Errorf("lower package coroutine IR for %s: missing build context or module", pkgPath) @@ -8241,7 +8304,11 @@ func exportStagedPackageObject(ctx *context, pkg *aPackage) (string, error) { if ctx.stagedFuncInfoSites && ctx.stagedFuncInfoMeta { emitFuncInfoEntrySitesForModule(mod, ctx.stagedPointerSize, ctx.stagedMachOSites) } - if _, err := llssa.VerifyCoroAtomicCostModule(mod); err != nil { + verifyAtomicCost := llssa.VerifyCoroAtomicCostModule + if ctx.passOpt { + verifyAtomicCost = llssa.VerifyOptimizedCoroAtomicCostModule + } + if _, err := verifyAtomicCost(mod); err != nil { return "", fmt.Errorf("verify package %s final detached atomic-cost certificates: %w", pkg.PkgPath, err) } if ctx.buildConf.CheckLLFiles { diff --git a/internal/build/cgo.go b/internal/build/cgo.go index 0dbe56e368..8c4d1c70fd 100644 --- a/internal/build/cgo.go +++ b/internal/build/cgo.go @@ -443,7 +443,7 @@ func parseCgoPreambleWithCommandEnv(commands commandEnv, pos token.Position, tex // #cgo pkg-config: python3 // #cgo windows CFLAGS: -IC:/Python312/include // #cgo windows LDFLAGS: -LC:/Python312/libs -lpython312 -// #cgo linux CPPFLAGS: -I/usr/lib/llvm-19/include -D_GNU_SOURCE +// #cgo linux CPPFLAGS: -I/usr/lib/llvm-22/include -D_GNU_SOURCE // #cgo CFLAGS: -I/usr/include/python3.12 // #cgo CXXFLAGS: -I/usr/include/c++/v1 // #cgo LDFLAGS: -L/usr/lib/python3.12/config-3.12-x86_64-linux-gnu -lpython3.12 diff --git a/internal/build/cgo_test.go b/internal/build/cgo_test.go index 926b30b518..c980965513 100644 --- a/internal/build/cgo_test.go +++ b/internal/build/cgo_test.go @@ -27,11 +27,11 @@ func TestParseCgoDeclFlags(t *testing.T) { }{ { name: "CPPFLAGS with tag", - line: "#cgo linux CPPFLAGS: -I/usr/lib/llvm-19/include -D_GNU_SOURCE", + line: "#cgo linux CPPFLAGS: -I/usr/lib/llvm-22/include -D_GNU_SOURCE", want: []cgoDecl{ { tag: "linux", - cflags: []string{"-I/usr/lib/llvm-19/include", "-D_GNU_SOURCE"}, + cflags: []string{"-I/usr/lib/llvm-22/include", "-D_GNU_SOURCE"}, }, }, }, diff --git a/internal/build/coro_architecture_gate_test.go b/internal/build/coro_architecture_gate_test.go index 997960e562..28e76d2043 100644 --- a/internal/build/coro_architecture_gate_test.go +++ b/internal/build/coro_architecture_gate_test.go @@ -147,7 +147,7 @@ var currentCoroArchitectureDebtBudget = coroArchitectureDebtBudget{ legacyPhysicalSelector: 0, legacySplitEmissionState: 0, emissionSessionAccess: 20, - bodyCapabilityAccess: 35, + bodyCapabilityAccess: 33, // The outcome-plain cohort replaced the coroutine-only begin/bind/complete // entry points with one exclusive managed-body transaction. Keep the legacy // names at zero so a second physical-emission lifecycle cannot grow back. @@ -465,13 +465,13 @@ var allowedPhysicalOutcomeObservationFiles = map[string]bool{ } var allowedCoroParkOperationFields = map[string]bool{ - "shouldSuspend": true, - "park": true, - "resume": true, - "normal": true, - "faults": true, - "abort": true, - "shutdown": true, + "prepare": true, + "park": true, + "resume": true, + "normal": true, + "faults": true, + "abort": true, + "shutdown": true, } var allowedCoroParkFaultRouteFields = map[string]bool{ diff --git a/internal/build/coro_bootstrap.go b/internal/build/coro_bootstrap.go index 54c658309f..c71eb7c682 100644 --- a/internal/build/coro_bootstrap.go +++ b/internal/build/coro_bootstrap.go @@ -74,9 +74,9 @@ const ( coroNotifyPrepareOrAbortSymbolV2 = "__llgo_coro_notify_prepare_or_abort_v2" coroNotifyOneOrAbortSymbolV2 = "__llgo_coro_notify_one_or_abort_v2" coroNotifyAllOrAbortSymbolV2 = "__llgo_coro_notify_all_or_abort_v2" - coroChanSendParkSymbolV1 = "__llgo_coro_chan_send_park_v1" - coroChanRecvParkSymbolV1 = "__llgo_coro_chan_recv_park_v1" - coroChanResumeSymbolV1 = "__llgo_coro_chan_resume_v1" + coroChanSendTryParkSymbolV2 = "__llgo_coro_chan_send_try_park_v2" + coroChanRecvTryParkSymbolV2 = "__llgo_coro_chan_recv_try_park_v2" + coroChanResumeSymbolV2 = "__llgo_coro_chan_resume_v2" coroWorkerParkSymbolV1 = "__llgo_coro_worker_park_v1" coroWorkerResumeSymbolV1 = "__llgo_coro_worker_resume_v1" coroHostOperationParkSymbolV1 = "__llgo_coro_host_operation_park_v1" @@ -115,6 +115,11 @@ const ( coroProgramRunHasDeadlineV2 uint32 = 1 << 2 coroProgramRunRequestInlineV2 uint32 = 1 << 3 coroProgramRunRequestQueuedV2 uint32 = 1 << 4 + + // Program bootstrap capabilities are closed-world physical demand, not + // target support. Keep these values synchronized with ssa and + // runtime/internal/coro; the bootstrap hash binds the complete bitset. + coroProgramCapabilityWorkerV2 uint32 = 1 << 0 ) type coroProgramBootstrapStepV1 struct { @@ -133,10 +138,22 @@ type coroProgramBootstrapStepV1 struct { type coroProgramBootstrapV1 struct { Version uint32 + Flags uint32 StepHash [16]byte Steps []coroProgramBootstrapStepV1 } +func coroProgramCapabilityFlagsV2(capabilities coro.ProgramCapabilities) (uint32, error) { + if !capabilities.Valid() { + return 0, fmt.Errorf("invalid coroutine program capabilities") + } + var flags uint32 + if capabilities.Worker() { + flags |= coroProgramCapabilityWorkerV2 + } + return flags, nil +} + func validateCoroProgramBootstrapConfig(conf *Config) error { if conf == nil { return nil @@ -294,11 +311,20 @@ func selectCoroProgramBootstrapV2(ctx *context, pkg *packages.Package) (*coroPro if err := appendManaged(aPkg.SSA.Func("main"), mainSymbolPrefix+".main", aPkg.PkgPath, "main", coroProgramStepRoleMainV2); err != nil { return nil, err } - hash, err := coroProgramBootstrapHash(ctx, coroProgramBootstrapVersionV2, steps) + flags, err := coroProgramCapabilityFlagsV2(ctx.coroProgramCapabilities) if err != nil { return nil, err } - return &coroProgramBootstrapV1{Version: coroProgramBootstrapVersionV2, StepHash: hash, Steps: steps}, nil + hash, err := coroProgramBootstrapHash(ctx, coroProgramBootstrapVersionV2, flags, steps) + if err != nil { + return nil, err + } + return &coroProgramBootstrapV1{ + Version: coroProgramBootstrapVersionV2, + Flags: flags, + StepHash: hash, + Steps: steps, + }, nil } func exactCoroRuntimeABIFunction(ctx *context, name string) (*ssa.Function, error) { @@ -465,7 +491,7 @@ func selectCoroProgramManagedStepV2( // executor, never as an interrupt callback, so a bounded plain stage may // retain this flag. ThreadAffine remains rejected until the bootstrap G has // an explicit locked-M/pinned-P contract. - const supportedPlain = coro.MayUnwind | coro.NeedsCleanupFrame | coro.IRQUnsafe + const supportedPlain = coro.MayUnwind | coro.NeedsCleanupFrame | coro.IRQUnsafe | coro.NeedsRuntimeContext if unsupported := plan.Exec &^ supportedPlain; unsupported != 0 { return coroProgramBootstrapStepV1{}, fmt.Errorf("coroutine program bootstrap %s: plain function %q target %q has unsupported execution constraints %s", label, fn.String(), plan.ID, unsupported) } @@ -482,7 +508,8 @@ func selectCoroProgramManagedStepV2( // not a separate bootstrap execution protocol. Physical preflight proves // and emits that body's defer/recover frame before this selector publishes // its ordinary scheduler root descriptor. - const supportedCoroutine = coro.MayUnwind | coro.NeedsCleanupFrame | coro.NeedsPreempt | coro.IRQUnsafe + const supportedCoroutine = coro.MayUnwind | coro.NeedsCleanupFrame | coro.NeedsPreempt | + coro.IRQUnsafe | coro.NeedsRuntimeContext if unsupported := plan.Exec &^ supportedCoroutine; unsupported != 0 { trace := "" if unsupported.Contains(coro.OpaqueExec) { @@ -697,7 +724,7 @@ func typeParamLen(list *types.TypeParamList) int { return list.Len() } -func coroProgramBootstrapHash(ctx *context, version uint32, steps []coroProgramBootstrapStepV1) ([16]byte, error) { +func coroProgramBootstrapHash(ctx *context, version, flags uint32, steps []coroProgramBootstrapStepV1) ([16]byte, error) { if ctx == nil || ctx.prog == nil || ctx.buildConf == nil || ctx.coroPlan == nil { return [16]byte{}, fmt.Errorf("coroutine program bootstrap hash requires a complete build context and plan") } @@ -725,7 +752,10 @@ func coroProgramBootstrapHash(ctx *context, version uint32, steps []coroProgramB } write("llgo.coro.program-bootstrap.v" + strconv.FormatUint(uint64(version), 10)) write(strconv.FormatUint(uint64(version), 10)) - write("flags=0") + if unknown := flags &^ coroProgramCapabilityWorkerV2; unknown != 0 { + return [16]byte{}, fmt.Errorf("coroutine program bootstrap has unknown capability flags %#x", unknown) + } + write("flags=" + strconv.FormatUint(uint64(flags), 10)) write("step={kind:u32,flags:u32,target:ptr,aux:uintptr}") write("bootstrap={version:u32,flags:u32,hash-lo:u64,hash-hi:u64,step-count:uintptr,steps:ptr,factory:ptr}") write("direct-plain=" + strconv.FormatUint(uint64(coroProgramStepDirectPlainV1), 10)) @@ -794,10 +824,10 @@ func coroProgramBootstrapHash(ctx *context, version uint32, steps []coroProgramB coroPollUpdateDeadlineOrAbortSymbolV1 + "(context:uintptr,interest:u32,deadline-ns:i64)->void;" + coroPollPostClosingOrAbortSymbolV1 + "(context:uintptr,interest:u32)->void") } - write("channel-v1=" + - coroChanSendParkSymbolV1 + "(g:ptr,handle:ptr,header:ptr,channel:ptr,elem:ptr,state:ptr,size:uintptr)->void;" + - coroChanRecvParkSymbolV1 + "(g:ptr,handle:ptr,header:ptr,channel:ptr,elem:ptr,state:ptr,size:uintptr)->void;" + - coroChanResumeSymbolV1 + "(g:ptr,state:ptr)->u32;" + + write("channel-v2=" + + coroChanSendTryParkSymbolV2 + "(g:ptr,handle:ptr,header:ptr,channel:ptr,elem:ptr,state:ptr,size:uintptr,state-id:u32,line:u32)->u32;" + + coroChanRecvTryParkSymbolV2 + "(g:ptr,handle:ptr,header:ptr,channel:ptr,elem:ptr,state:ptr,size:uintptr,state-id:u32,line:u32)->u32;" + + coroChanResumeSymbolV2 + "(g:ptr,state:ptr)->u32;" + "send-closed-fault=__llgo_coro_fault_prepare_v1:kind=3") if ctx.buildConf.coroWorkerSupported() { write("worker-v1=" + diff --git a/internal/build/coro_bootstrap_test.go b/internal/build/coro_bootstrap_test.go index 8c506c1c1e..770f9db236 100644 --- a/internal/build/coro_bootstrap_test.go +++ b/internal/build/coro_bootstrap_test.go @@ -130,6 +130,29 @@ func TestSelectCoroProgramBootstrapV2ExactMixedFiveStageProgram(t *testing.T) { } } +func TestSelectCoroProgramBootstrapV2PublishesAndBindsWorkerDemand(t *testing.T) { + fixture := newCoroBootstrapV2TestContext(t) + withoutWorker, err := selectCoroProgramBootstrapV2(fixture.ctx, fixture.mainPackage) + if err != nil { + t.Fatal(err) + } + if withoutWorker.Flags != 0 { + t.Fatalf("bootstrap without worker demand flags = %#x, want zero", withoutWorker.Flags) + } + + fixture.ctx.coroProgramCapabilities = coro.NewProgramCapabilities(true) + withWorker, err := selectCoroProgramBootstrapV2(fixture.ctx, fixture.mainPackage) + if err != nil { + t.Fatal(err) + } + if withWorker.Flags != coroProgramCapabilityWorkerV2 { + t.Fatalf("bootstrap worker flags = %#x, want %#x", withWorker.Flags, coroProgramCapabilityWorkerV2) + } + if withWorker.StepHash == withoutWorker.StepHash { + t.Fatalf("bootstrap hash ignored program capabilities: %x", withWorker.StepHash) + } +} + func TestSelectCoroProgramBootstrapV2UsesOwnedNoopWhenPublicRuntimeIsAbsent(t *testing.T) { fixture := newCoroBootstrapV2TestContextWithPublicRuntime(t, false) bootstrap, err := selectCoroProgramBootstrapV2(fixture.ctx, fixture.mainPackage) diff --git a/internal/build/coro_directive_gate_test.go b/internal/build/coro_directive_gate_test.go index 8609a8a09a..d274fc6747 100644 --- a/internal/build/coro_directive_gate_test.go +++ b/internal/build/coro_directive_gate_test.go @@ -79,7 +79,10 @@ func TestCoroProductionDirectiveInventory(t *testing.T) { } want := map[string]int{ - "contract": 7, + // The eighth bottom contract is the native scheduler doorbell write: + // its descriptor is created O_NONBLOCK, a semantic fact unavailable + // from the C signature and therefore not soundly compiler-inferable. + "contract": 8, "noblock": 35, "sync": 28, } @@ -91,7 +94,7 @@ func TestCoroProductionDirectiveInventory(t *testing.T) { ) } sort.Strings(manifest) - const wantManifestSHA256 = "a1773b45067110dc275caee403ab26bf6a69b167f3df40439b2e0c4f7298e26f" + const wantManifestSHA256 = "79e04f606714069df05f69f48ddce3c7108b38bc503b734370a021ab5aadc912" manifestSHA256 := fmt.Sprintf( "%x", sha256.Sum256([]byte(strings.Join(manifest, "\n"))), ) diff --git a/internal/build/coro_host_entry_test.go b/internal/build/coro_host_entry_test.go index 094133df90..46cdeb9399 100644 --- a/internal/build/coro_host_entry_test.go +++ b/internal/build/coro_host_entry_test.go @@ -58,6 +58,8 @@ func __llgo_coro_notify_all_or_abort_v2(unsafe.Pointer, uint32) {} func __llgo_coro_frame_allocator_bootstrap_v1() {} func __llgo_coro_frame_alloc_v1() {} func __llgo_coro_frame_publish_v1() {} +func __llgo_coro_frame_publish_v3() {} +func __llgo_coro_frame_destroy_commit_v2() {} func __llgo_coro_await_prepare_v1() {} func __llgo_coro_await_prepare_v3(g, parent, child unsafe.Pointer, mode uint32, typeWord, dataWord unsafe.Pointer) {} func __llgo_coro_await_inline_v1(g, parent, child unsafe.Pointer) bool { return false } @@ -73,14 +75,14 @@ func __llgo_coro_critical_exit_v1(unsafe.Pointer) bool { return false } func __llgo_coro_os_thread_lock_v1(unsafe.Pointer) {} func __llgo_coro_os_thread_unlock_v1(unsafe.Pointer) {} func __llgo_coro_frame_free_v1() {} -func __llgo_coro_chan_send_park_v1(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uintptr) {} -func __llgo_coro_chan_recv_park_v1(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uintptr) {} -func __llgo_coro_chan_resume_v1(unsafe.Pointer, unsafe.Pointer) uint32 { return 0 } +func __llgo_coro_chan_send_try_park_v2(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uintptr, uint32, uint32) uint32 { return 0 } +func __llgo_coro_chan_recv_try_park_v2(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uintptr, uint32, uint32) uint32 { return 0 } +func __llgo_coro_chan_resume_v2(unsafe.Pointer, unsafe.Pointer) uint32 { return 0 } type Chan struct{} type ChanOp struct{} -func CoroChanTrySend(*Chan, unsafe.Pointer, int) bool { return false } -func CoroChanTryRecv(*Chan, unsafe.Pointer, int) (bool, bool) { return false, false } -func CoroChanTryClose(*Chan) uint32 { return 0 } +func CoroChanTrySend(unsafe.Pointer, *Chan, unsafe.Pointer, int) bool { return false } +func CoroChanTryRecv(unsafe.Pointer, *Chan, unsafe.Pointer, int) (bool, bool) { return false, false } +func CoroChanTryCloseTask(unsafe.Pointer, *Chan) uint32 { return 0 } func CoroChanSelectTry(...ChanOp) (int, bool, bool, bool) { return 0, false, false, false } func CoroChanSelectPark(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, ...ChanOp) {} func CoroChanSelectResume(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, ...ChanOp) (int, bool, uint32) { return 0, false, 0 } @@ -153,6 +155,8 @@ func __llgo_coro_program_main_return_v1() {} coroNotifyAllOrAbortSymbolV2, "__llgo_coro_frame_alloc_v1", "__llgo_coro_frame_publish_v1", + "__llgo_coro_frame_publish_v3", + "__llgo_coro_frame_destroy_commit_v2", "__llgo_coro_await_prepare_v1", "__llgo_coro_preempt_poll_v1", "__llgo_coro_yield_prepare_v1", @@ -170,13 +174,13 @@ func __llgo_coro_program_main_return_v1() {} coroOSThreadUnlockSymbolV1, "CoroChanTrySend", "CoroChanTryRecv", - "CoroChanTryClose", + "CoroChanTryCloseTask", "CoroChanSelectTry", "CoroChanSelectPark", "CoroChanSelectResume", - coroChanSendParkSymbolV1, - coroChanRecvParkSymbolV1, - coroChanResumeSymbolV1, + coroChanSendTryParkSymbolV2, + coroChanRecvTryParkSymbolV2, + coroChanResumeSymbolV2, "__llgo_coro_fault_prepare_v1", "__llgo_coro_fault_prepare_v2", "__llgo_coro_panic_prepare_v1", diff --git a/internal/build/coro_native_fleet_e2e_test.go b/internal/build/coro_native_fleet_e2e_test.go index bbcdcda575..cda8dfe5a2 100644 --- a/internal/build/coro_native_fleet_e2e_test.go +++ b/internal/build/coro_native_fleet_e2e_test.go @@ -647,8 +647,19 @@ var Done chan uint32 var Failed uint32 //llgo:coro noblock -//go:linkname gomaxprocs command-line-arguments.CoroGOMAXPROCS -func gomaxprocs(int) int +//go:linkname gomaxprocsRaw command-line-arguments.CoroGOMAXPROCS +func gomaxprocsRaw(int) int + +//go:linkname schedulerYield llgo.coroYield +func schedulerYield() + +func gomaxprocs(n int) int { + previous := gomaxprocsRaw(n) + if n > 0 && previous != n { + schedulerYield() + } + return previous +} //llgo:coro noblock //go:linkname quotaReset C.__llgo_coro_native_fleet_e2e_quota_reset_v1 @@ -1728,7 +1739,10 @@ func TestCoroNativeFleetPeerSpawnReturnsToProgramE2E(t *testing.T) { } func TestCoroNativeFleetLockedGExitRetiresPhysicalPeerE2E(t *testing.T) { - runCoroNativeFleetE2E(t, coroNativeFleetLockedGExitRetiresPeerE2ESource, "locked-g-exit-retire-peer", false, 1) + // The fixture explicitly rejects the adopted program M and therefore needs + // one quota-backed peer M. GOMAXPROCS=1 deliberately starts no idle peer; + // locked program-owner succession at that limit is covered separately. + runCoroNativeFleetE2E(t, coroNativeFleetLockedGExitRetiresPeerE2ESource, "locked-g-exit-retire-peer", false, 2) } func TestCoroNativeFleetLockedGExitRetiresProgramOwnerE2E(t *testing.T) { @@ -1893,10 +1907,10 @@ func runCoroNativeFleetE2E(t *testing.T, source, name string, enableChannel bool prog.TypeSizes(types.SizesFor("gc", runtime.GOARCH)) defer prog.Dispose() - userObject, anchor, setupSymbol, checkSymbol := buildCoroSpawnNativeE2EUserSource( - t, prog, temp, source, enableChannel, cl.CoroNativeTargetCapabilities(), + userObject, anchor, setupSymbol, checkSymbol, capabilities := buildCoroSpawnNativeE2EUserSource( + t, prog, temp, source, enableChannel, cl.CoroNativeTargetCapabilities(), false, ) - entryObject := buildCoroSpawnNativeE2EEntry(t, prog, temp, anchor) + entryObject := buildCoroSpawnNativeE2EEntry(t, prog, temp, anchor, capabilities) driverObject := buildCoroSpawnNativeE2EDriver(t, prog, temp, setupSymbol, checkSymbol) runtimeArchive := cachedCoroNativeFleetE2ERuntimeArchive(t, clang, ar) @@ -1985,6 +1999,7 @@ func buildCoroNativeFleetE2ERuntimeIsland(t *testing.T, temp string) []string { filepath.Join("..", "..", "runtime", "internal", "runtime", "coro_sched.go"), filepath.Join("..", "..", "runtime", "internal", "runtime", "coro_executor.go"), filepath.Join("..", "..", "runtime", "internal", "runtime", "coro_executor_driver_timer_llgo.go"), + filepath.Join("..", "..", "runtime", "internal", "runtime", "coro_worker_completion_window_llgo.go"), filepath.Join("..", "..", "runtime", "internal", "runtime", "coro_operation_capacity.go"), filepath.Join("..", "..", "runtime", "internal", "runtime", "coro_nil_fault.go"), filepath.Join("..", "..", "runtime", "internal", "runtime", "coro_panic_payload.go"), @@ -2022,6 +2037,7 @@ func buildCoroNativeFleetE2ERuntimeIsland(t *testing.T, temp string) []string { }...) requireCoroRuntimeIslandProductionSource(t, files, "coro_current_task_route.go") requireCoroRuntimeIslandProductionSource(t, files, "coro_operation_capacity.go") + requireCoroRuntimeIslandProductionSource(t, files, "coro_worker_completion_window_llgo.go") requireCoroRuntimeIslandProductionSource(t, files, "coro_worker_result_llgo.go") requireCoroRuntimeIslandProductionSource(t, files, "coro_keyed_registry_atomic_llgo.go") files = materializeCoroChannelNativeE2ERuntimeIsland(t, files) diff --git a/internal/build/coro_native_target_plan_test.go b/internal/build/coro_native_target_plan_test.go index 65cbf2414e..c7cdb6c899 100644 --- a/internal/build/coro_native_target_plan_test.go +++ b/internal/build/coro_native_target_plan_test.go @@ -538,6 +538,43 @@ func TestRealNativeCoroTargetIsTrustedPlainSchedulerIsland(t *testing.T) { } return nil } + validateDeclaredExecutorSafeRawHost := func(record coroForeignUseDomainRecord, identity string) error { + if !record.DirectExecutorCertified || record.NoBlockCertified || record.SyncCertified || + !record.rawHostOnly() { + return fmt.Errorf( + "native target declared executor-safe raw-host declaration %q = %s", + identity, record.diagnostic(), + ) + } + if _, legacy := plan.ForeignNoBlockCertificate(record.Function); legacy { + return fmt.Errorf( + "native target declared executor-safe raw-host declaration %q retained a foreign-noblock certificate", + identity, + ) + } + if _, legacy := plan.ForeignSyncCertificate(record.Function); legacy { + return fmt.Errorf( + "native target declared executor-safe raw-host declaration %q retained a foreign-sync certificate", + identity, + ) + } + callable, certified := plan.CallableContractCertificate(record.Function) + function, planned := plan.FunctionPlan(record.Function) + if !certified || callable.Scope != coro.CallableContractScopeDeclaration || + !strings.HasPrefix(string(callable.Contract.ID), "foreign.v1/") || + !coro.CallableContractDirectExecutorCompatible(callable.Contract) || + !planned || function.External != coro.ExternalKnown || + function.Effect != coro.NoSuspend || function.Exec != coro.IRQUnsafe || + function.ManagedDemand != coro.NoDemand || !function.RawPlainDemand || + function.Emission != coro.EmitExternal { + return fmt.Errorf( + "native target declared executor-safe raw-host declaration %q lost its exact contract: "+ + "callable=%+v certified=%t function=%+v planned=%t", + identity, callable, certified, function, planned, + ) + } + return nil + } validateElidedTypedControl := func(record coroForeignUseDomainRecord, identity string) error { if record.DirectExecutorCertified || record.NoBlockCertified || @@ -597,13 +634,12 @@ func TestRealNativeCoroTargetIsTrustedPlainSchedulerIsland(t *testing.T) { "__llgo_coro_worker_queue_stop_v1": false, "__llgo_coro_worker_queue_destroy_after_join_v1": false, "__llgo_coro_worker_call_v1": false, - "__llgo_coro_worker_queue_reserve_v1": false, - "__llgo_coro_worker_queue_cancel_reservation_v1": false, - "__llgo_coro_worker_queue_submit_reserved_v1": false, + "__llgo_coro_worker_queue_reserve_v2": false, + "__llgo_coro_worker_queue_cancel_reservation_v2": false, + "__llgo_coro_worker_queue_submit_reserved_v4": false, "__llgo_coro_fleet_owner_retire_self_v1": false, "__llgo_coro_doorbell_open_v1": false, "__llgo_coro_doorbell_read_v1": false, - "__llgo_coro_doorbell_write_v1": false, "__llgo_coro_doorbell_close_v1": false, "GC_init": false, "GC_add_roots": false, @@ -612,6 +648,9 @@ func TestRealNativeCoroTargetIsTrustedPlainSchedulerIsland(t *testing.T) { migratedRawHostDeclarations := map[string]string{ runtimePath + ".coroPanicTerminalFputc": "__llgo_coro_panic_fputc_v1", } + declaredExecutorSafeRawHostSymbols := map[string]bool{ + "__llgo_coro_doorbell_write_v1": false, + } elidedTypedControlSymbols := map[string]bool{ "siglongjmp": false, } @@ -657,6 +696,20 @@ func TestRealNativeCoroTargetIsTrustedPlainSchedulerIsland(t *testing.T) { t.Log("elided typed control " + record.diagnostic()) continue } + if _, declared := declaredExecutorSafeRawHostSymbols[record.PhysicalSymbol]; declared { + if declaredExecutorSafeRawHostSymbols[record.PhysicalSymbol] { + return nil, fmt.Errorf( + "native target declared executor-safe raw-host declaration %q is ambiguous", + record.PhysicalSymbol, + ) + } + declaredExecutorSafeRawHostSymbols[record.PhysicalSymbol] = true + if err := validateDeclaredExecutorSafeRawHost(record, record.PhysicalSymbol); err != nil { + return nil, err + } + t.Log("declared executor-safe " + record.diagnostic()) + continue + } if _, migrated := migratedRawHostSymbols[record.PhysicalSymbol]; !migrated { continue } @@ -682,6 +735,14 @@ func TestRealNativeCoroTargetIsTrustedPlainSchedulerIsland(t *testing.T) { ) } } + for symbol, found := range declaredExecutorSafeRawHostSymbols { + if !found { + return nil, fmt.Errorf( + "native target declared executor-safe raw-host declaration %q is absent from the closed report", + symbol, + ) + } + } for symbol, found := range elidedTypedControlSymbols { if !found { return nil, fmt.Errorf("native target typed-control declaration %q is absent from the closed report", symbol) diff --git a/internal/build/coro_panic_native_e2e_test.go b/internal/build/coro_panic_native_e2e_test.go index 93767bc35f..fd362d17bc 100644 --- a/internal/build/coro_panic_native_e2e_test.go +++ b/internal/build/coro_panic_native_e2e_test.go @@ -51,7 +51,7 @@ const ( coroPanicNativeE2EThirdDestroy = "__llgo_coro_panic_e2e_third_destroy" coroPanicNativeE2EExplicitStatus = uint64(1) coroPanicNativeE2EDrivePanic = uint64(4) - coroPanicNativeE2EExpectedDestroys = uint64(3) + coroPanicNativeE2EExpectedDestroys = uint64(2) ) const coroPanicNativeE2ESource = `package main @@ -104,9 +104,10 @@ func main() { // full-runtime caller acceptance test exercises production presentation. The // report accepts only the exact drive status, a // published record on a dead, non-reclaimable G, the original package-global -// payload word, and exactly one destroy of each distinct handle in the child -// -> main -> bootstrap chain. It does not turn panic into production success -// or provide a replacement printer. +// payload word, and exactly one destroy of each distinct handle in the main -> +// bootstrap chain. panicChild is now outcome-plain, so it propagates an +// explicit status without allocating a third coroutine frame. The test does +// not turn panic into production success or provide a replacement printer. func TestCoroExplicitPanicNativeNoStdlibRuntimeE2E(t *testing.T) { if runtime.GOOS != "darwin" && runtime.GOOS != "linux" { t.Skip("native coroutine link smoke requires Darwin or Linux") @@ -473,10 +474,8 @@ func buildCoroPanicNativeE2EDriver(t *testing.T, prog llssa.Program, temp string third := reportBody.Load(thirdDestroy.Expr) requireCondition(reportBody.BinOp(token.NEQ, first, nilPointer)) requireCondition(reportBody.BinOp(token.NEQ, second, nilPointer)) - requireCondition(reportBody.BinOp(token.NEQ, third, nilPointer)) + requireCondition(reportBody.BinOp(token.EQL, third, nilPointer)) requireCondition(reportBody.BinOp(token.NEQ, first, second)) - requireCondition(reportBody.BinOp(token.NEQ, first, third)) - requireCondition(reportBody.BinOp(token.NEQ, second, third)) requireCondition(reportBody.BinOp(token.EQL, reportBody.Load(before.Expr), one32)) requireCondition(reportBody.BinOp(token.EQL, reportBody.Load(after.Expr), zero32)) requireCondition(reportBody.BinOp( @@ -573,7 +572,7 @@ func assertCoroPanicNativeE2ELinkedSymbols(t *testing.T, executable string) { "github.com/goplus/llgo/runtime/internal/coro.PreparePanic", "github.com/goplus/llgo/runtime/internal/coro.PanicDestroyed", "github.com/goplus/llgo/runtime/internal/coro.LoadPanicRecord", - coroNativeE2EMainPhysicalSymbol("panicChild$coro"), + coroNativeE2EMainPhysicalSymbol("panicChild$outcome"), coroNativeE2EMainPhysicalSymbol("panicLeaf$outcome"), } { if !strings.Contains(symbols, required) { diff --git a/internal/build/coro_plan_test.go b/internal/build/coro_plan_test.go index 9923b1284f..b84b5ca28f 100644 --- a/internal/build/coro_plan_test.go +++ b/internal/build/coro_plan_test.go @@ -171,6 +171,34 @@ func calls(fn func()) { t.Fatalf("ordinary builder elision error = %v, want fail-closed rejection", err) } + unevaluatedInput := input + unevaluatedInput.callSitePlan = func(call ssa.CallInstruction) (cl.CoroCallSitePlan, bool, error) { + return cl.CoroCallSitePlan{Elision: cl.CoroCallElidedFrontendUnevaluated}, true, nil + } + unevaluatedPlan, err := unevaluatedInput.Analyze( + coro.Roots{{Function: ssaPkg.Func("calls"), Demand: coro.AsyncDemand}}, + coro.SSAConfig{MaxPlainInstructions: -1}, + ) + if err != nil { + t.Fatal(err) + } + for _, call := range ordinaryCalls { + wantElided := false + switch exact := call.(type) { + case *ssa.Call: + wantElided = exact != nil && exact.Common().StaticCallee() != nil && !exact.Common().IsInvoke() + case *ssa.Defer: + wantElided = exact != nil && exact.DeferStack == nil && exact.Common().StaticCallee() != nil && !exact.Common().IsInvoke() + } + if got := unevaluatedPlan.ElidesCall(call); got != wantElided { + t.Fatalf("frontend-unevaluated %T call elided=%t, want %t: %s", call, got, wantElided, call) + } + _, planned := unevaluatedPlan.CallPlan(call) + if planned == wantElided { + t.Fatalf("frontend-unevaluated %T CallPlan present=%t, want %t: %s", call, planned, !wantElided, call) + } + } + var ordinaryInitCall ssa.CallInstruction for _, call := range initCalls { callee := call.Common().StaticCallee() @@ -552,6 +580,8 @@ func __llgo_coro_notify_all_or_abort_v2(notifyAddr unsafe.Pointer, waitSnapshot func __llgo_coro_frame_allocator_bootstrap_v1() {} func __llgo_coro_frame_alloc_v1() {} func __llgo_coro_frame_publish_v1() {} +func __llgo_coro_frame_publish_v3() {} +func __llgo_coro_frame_destroy_commit_v2() {} func __llgo_coro_await_prepare_v1() {} func __llgo_coro_await_prepare_v3(g, parent, child unsafe.Pointer, mode uint32, typeWord, dataWord unsafe.Pointer) {} func __llgo_coro_await_inline_v1(g, parent, child unsafe.Pointer) bool { return false } @@ -568,14 +598,14 @@ func __llgo_coro_critical_exit_v1(g unsafe.Pointer) bool { return false } func __llgo_coro_os_thread_lock_v1(g unsafe.Pointer) {} func __llgo_coro_os_thread_unlock_v1(g unsafe.Pointer) {} func __llgo_coro_frame_free_v1() {} -func __llgo_coro_chan_send_park_v1(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uintptr) {} -func __llgo_coro_chan_recv_park_v1(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uintptr) {} -func __llgo_coro_chan_resume_v1(unsafe.Pointer, unsafe.Pointer) uint32 { return 0 } +func __llgo_coro_chan_send_try_park_v2(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uintptr, uint32, uint32) uint32 { return 0 } +func __llgo_coro_chan_recv_try_park_v2(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uintptr, uint32, uint32) uint32 { return 0 } +func __llgo_coro_chan_resume_v2(unsafe.Pointer, unsafe.Pointer) uint32 { return 0 } type Chan struct{} type ChanOp struct{} -func CoroChanTrySend(*Chan, unsafe.Pointer, int) bool { return false } -func CoroChanTryRecv(*Chan, unsafe.Pointer, int) (bool, bool) { return false, false } -func CoroChanTryClose(*Chan) uint32 { return 0 } +func CoroChanTrySend(unsafe.Pointer, *Chan, unsafe.Pointer, int) bool { return false } +func CoroChanTryRecv(unsafe.Pointer, *Chan, unsafe.Pointer, int) (bool, bool) { return false, false } +func CoroChanTryCloseTask(unsafe.Pointer, *Chan) uint32 { return 0 } func CoroChanSelectTry(...ChanOp) (int, bool, bool, bool) { return 0, false, false, false } func CoroChanSelectPark(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, ...ChanOp) {} func CoroChanSelectResume(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, ...ChanOp) (int, bool, uint32) { return 0, false, 0 } @@ -676,6 +706,8 @@ func atomicExchange(*uint32, uint32) uint32 coroProgramContinueSymbolV1, "__llgo_coro_frame_alloc_v1", "__llgo_coro_frame_publish_v1", + "__llgo_coro_frame_publish_v3", + "__llgo_coro_frame_destroy_commit_v2", "__llgo_coro_await_prepare_v1", "__llgo_coro_preempt_poll_v1", "__llgo_coro_yield_prepare_v1", @@ -693,13 +725,13 @@ func atomicExchange(*uint32, uint32) uint32 coroOSThreadUnlockSymbolV1, "CoroChanTrySend", "CoroChanTryRecv", - "CoroChanTryClose", + "CoroChanTryCloseTask", "CoroChanSelectTry", "CoroChanSelectPark", "CoroChanSelectResume", - coroChanSendParkSymbolV1, - coroChanRecvParkSymbolV1, - coroChanResumeSymbolV1, + coroChanSendTryParkSymbolV2, + coroChanRecvTryParkSymbolV2, + coroChanResumeSymbolV2, "__llgo_coro_fault_prepare_v1", "__llgo_coro_fault_prepare_v2", "__llgo_coro_panic_prepare_v1", @@ -1161,13 +1193,13 @@ func atomicExchange(*uint32, uint32) uint32 channelNames := []string{ "CoroChanTrySend", "CoroChanTryRecv", - "CoroChanTryClose", + "CoroChanTryCloseTask", "CoroChanSelectTry", "CoroChanSelectPark", "CoroChanSelectResume", - coroChanSendParkSymbolV1, - coroChanRecvParkSymbolV1, - coroChanResumeSymbolV1, + coroChanSendTryParkSymbolV2, + coroChanRecvTryParkSymbolV2, + coroChanResumeSymbolV2, "__llgo_coro_fault_prepare_v1", "__llgo_coro_fault_prepare_v2", } @@ -1180,7 +1212,7 @@ func atomicExchange(*uint32, uint32) uint32 t.Fatalf("channel runtime hook %q is not a required plain root", name) } } - channelResume := ssaPkg.Func(coroChanResumeSymbolV1) + channelResume := ssaPkg.Func(coroChanResumeSymbolV2) originalChannelResumeSignature := channelResume.Signature channelResume.Signature = types.NewSignatureType(nil, nil, nil, types.NewTuple(types.NewParam(token.NoPos, nil, "g", types.Typ[types.UnsafePointer])), @@ -1409,6 +1441,8 @@ func __llgo_coro_program_continue_v1(uint32) {} func __llgo_coro_frame_allocator_bootstrap_v1() {} func __llgo_coro_frame_alloc_v1() {} func __llgo_coro_frame_publish_v1() {} +func __llgo_coro_frame_publish_v3() {} +func __llgo_coro_frame_destroy_commit_v2() {} func __llgo_coro_await_prepare_v1() {} func __llgo_coro_await_prepare_v3(g, parent, child unsafe.Pointer, mode uint32, typeWord, dataWord unsafe.Pointer) {} func __llgo_coro_await_inline_v1(g, parent, child unsafe.Pointer) bool { return false } @@ -1424,14 +1458,14 @@ func __llgo_coro_critical_exit_v1(g unsafe.Pointer) bool { return false } func __llgo_coro_os_thread_lock_v1(g unsafe.Pointer) {} func __llgo_coro_os_thread_unlock_v1(g unsafe.Pointer) {} func __llgo_coro_frame_free_v1() {} -func __llgo_coro_chan_send_park_v1(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uintptr) {} -func __llgo_coro_chan_recv_park_v1(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uintptr) {} -func __llgo_coro_chan_resume_v1(unsafe.Pointer, unsafe.Pointer) uint32 { return 0 } +func __llgo_coro_chan_send_try_park_v2(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uintptr, uint32, uint32) uint32 { return 0 } +func __llgo_coro_chan_recv_try_park_v2(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uintptr, uint32, uint32) uint32 { return 0 } +func __llgo_coro_chan_resume_v2(unsafe.Pointer, unsafe.Pointer) uint32 { return 0 } type Chan struct{} type ChanOp struct{} -func CoroChanTrySend(*Chan, unsafe.Pointer, int) bool { return false } -func CoroChanTryRecv(*Chan, unsafe.Pointer, int) (bool, bool) { return false, false } -func CoroChanTryClose(*Chan) uint32 { return 0 } +func CoroChanTrySend(unsafe.Pointer, *Chan, unsafe.Pointer, int) bool { return false } +func CoroChanTryRecv(unsafe.Pointer, *Chan, unsafe.Pointer, int) (bool, bool) { return false, false } +func CoroChanTryCloseTask(unsafe.Pointer, *Chan) uint32 { return 0 } func CoroChanSelectTry(...ChanOp) (int, bool, bool, bool) { return 0, false, false, false } func CoroChanSelectPark(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, ...ChanOp) {} func CoroChanSelectResume(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, ...ChanOp) (int, bool, uint32) { return 0, false, 0 } @@ -2101,6 +2135,8 @@ func __llgo_coro_program_continue_v1(uint32) {} func __llgo_coro_frame_allocator_bootstrap_v1() {} func __llgo_coro_frame_alloc_v1() {} func __llgo_coro_frame_publish_v1() {} +func __llgo_coro_frame_publish_v3() {} +func __llgo_coro_frame_destroy_commit_v2() {} func __llgo_coro_await_prepare_v1() {} func __llgo_coro_preempt_poll_v1() bool { return false } func __llgo_coro_yield_prepare_v1() {} @@ -2108,14 +2144,14 @@ func __llgo_coro_run_decision_take_v1(unsafe.Pointer, uint32, uint32, *uint32, * func __llgo_coro_run_decision_take_zero_v1(unsafe.Pointer) uint32 { return 0 } func __llgo_coro_complete_prepare_v1() {} func __llgo_coro_frame_free_v1() {} -func __llgo_coro_chan_send_park_v1(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uintptr) {} -func __llgo_coro_chan_recv_park_v1(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uintptr) {} -func __llgo_coro_chan_resume_v1(unsafe.Pointer, unsafe.Pointer) uint32 { return 0 } +func __llgo_coro_chan_send_try_park_v2(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uintptr, uint32, uint32) uint32 { return 0 } +func __llgo_coro_chan_recv_try_park_v2(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, uintptr, uint32, uint32) uint32 { return 0 } +func __llgo_coro_chan_resume_v2(unsafe.Pointer, unsafe.Pointer) uint32 { return 0 } type Chan struct{} type ChanOp struct{} -func CoroChanTrySend(*Chan, unsafe.Pointer, int) bool { return false } -func CoroChanTryRecv(*Chan, unsafe.Pointer, int) (bool, bool) { return false, false } -func CoroChanTryClose(*Chan) uint32 { return 0 } +func CoroChanTrySend(unsafe.Pointer, *Chan, unsafe.Pointer, int) bool { return false } +func CoroChanTryRecv(unsafe.Pointer, *Chan, unsafe.Pointer, int) (bool, bool) { return false, false } +func CoroChanTryCloseTask(unsafe.Pointer, *Chan) uint32 { return 0 } func CoroChanSelectTry(...ChanOp) (int, bool, bool, bool) { return 0, false, false, false } func CoroChanSelectPark(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, ...ChanOp) {} func CoroChanSelectResume(unsafe.Pointer, unsafe.Pointer, unsafe.Pointer, ...ChanOp) (int, bool, uint32) { return 0, false, 0 } @@ -3256,9 +3292,9 @@ func TestCoroUnsupportedEntryResolutionReturnsErrorBeforeCodegen(t *testing.T) { if !builderBuilt { t.Fatalf("CoroPlanBuilder did not successfully return a plan: %v", err) } - if err == nil || !strings.Contains(err.Error(), "compile package") || + if err == nil || !strings.Contains(err.Error(), "declared may-park effect has no exact structured park intrinsic") { - t.Fatalf("Do error = %v, want exact coroutine physical-ABI rejection returned from buildPkg", err) + t.Fatalf("Do error = %v, want exact coroutine physical-ABI rejection before codegen", err) } if len(pkgs) != 0 { t.Fatalf("Do packages = %+v, want none", pkgs) diff --git a/internal/build/coro_raw_global_symbol.go b/internal/build/coro_raw_global_symbol.go index 92e180f968..13198174d3 100644 --- a/internal/build/coro_raw_global_symbol.go +++ b/internal/build/coro_raw_global_symbol.go @@ -566,6 +566,13 @@ func inferCoroRawLLGoFileExecutorLeaves( if err != nil { continue } + if dataLayout, canonical := coroExecutorLeafCanonicalDataLayout(ctx, module); canonical { + if rebound, reboundErr := llvmproof.ProveExecutorLeafForDataLayout( + module, function.Name(), dataLayout, + ); reboundErr == nil { + proof = rebound + } + } inventory.addForeignExecutorLeafProof(cl.CoroForeignExecutorLeafProof{ ProducerIdentity: producerIdentity, PhysicalSymbol: proof.Symbol, @@ -578,6 +585,39 @@ func inferCoroRawLLGoFileExecutorLeaves( } } +// coroExecutorLeafCanonicalDataLayout recognizes only LLVM 22 wasm Clang's +// omission of LLGo's explicit i128:128 entry. The llvmproof package still has +// to prove that every type in each individual call closure has identical ABI +// layout before that closure may consume this canonical spelling. +func coroExecutorLeafCanonicalDataLayout( + ctx *context, + module llvm.Module, +) (string, bool) { + if ctx == nil || ctx.prog == nil || module.IsNil() || + !strings.HasPrefix(module.Target(), "wasm32-") || + !strings.HasPrefix(ctx.prog.TargetSpec().Triple, "wasm32-") { + return "", false + } + expected, observed := ctx.prog.DataLayout(), module.DataLayout() + if expected == "" || observed == "" || expected == observed { + return "", false + } + parts := strings.Split(expected, "-") + withoutI128 := make([]string, 0, len(parts)) + removed := 0 + for _, part := range parts { + if part == "i128:128" { + removed++ + continue + } + withoutI128 = append(withoutI128, part) + } + if removed != 1 || strings.Join(withoutI128, "-") != observed { + return "", false + } + return expected, true +} + func coroRawLLVMNM(ctx *context) (string, error) { if ctx == nil { return "", fmt.Errorf("missing build context") diff --git a/internal/build/coro_raw_global_symbol_test.go b/internal/build/coro_raw_global_symbol_test.go index be1d1471dc..47141959f5 100644 --- a/internal/build/coro_raw_global_symbol_test.go +++ b/internal/build/coro_raw_global_symbol_test.go @@ -228,8 +228,11 @@ func TestCoroRawLLGoFilesInfersWasmDebugTrap(t *testing.T) { if err != nil { t.Skip("clang is unavailable") } + prog := llssa.NewProgram(&llssa.Target{GOOS: "wasip1", GOARCH: "wasm"}) + defer prog.Dispose() ctx := &context{ buildConf: &Config{Goos: "wasip1", Goarch: "wasm"}, + prog: prog, crossCompile: crosscompile.Export{ CC: compiler, CCFLAGS: []string{"-target", "wasm32-unknown-unknown"}, @@ -261,8 +264,6 @@ func TestCoroRawLLGoFilesInfersWasmDebugTrap(t *testing.T) { t.Fatalf("WASM debugtrap executor-leaf proof = %+v, %t", proof, inferred) } - prog := llssa.NewProgram(&llssa.Target{GOOS: "wasip1", GOARCH: "wasm"}) - defer prog.Dispose() if proof.LLVMDataLayout != prog.DataLayout() { t.Fatalf( "WASM debugtrap proof data layout = %q, frontend = %q", diff --git a/internal/build/coro_registry.go b/internal/build/coro_registry.go index 9c5a2c3f18..3701988cc8 100644 --- a/internal/build/coro_registry.go +++ b/internal/build/coro_registry.go @@ -97,6 +97,7 @@ func coroProgramManifestHashV1(ctx *context, anchors []string, bootstrap ...*cor return [16]byte{}, fmt.Errorf("coroutine program manifest requires the unique V2 startup table") } write("llgo.coro.program-bootstrap.v2") + write(fmt.Sprintf("%d", program.Flags)) write(hex.EncodeToString(program.StepHash[:])) for _, step := range program.Steps { write(fmt.Sprintf("%d", step.Kind)) diff --git a/internal/build/coro_registry_test.go b/internal/build/coro_registry_test.go index 30fa0e5e0a..ad1595217d 100644 --- a/internal/build/coro_registry_test.go +++ b/internal/build/coro_registry_test.go @@ -105,6 +105,18 @@ func TestCoroProgramManifestHashV1StableAndComplete(t *testing.T) { if changedBootstrap == withBootstrap { t.Fatal("manifest hash ignored bootstrap StepHash") } + bootstrapC := &coroProgramBootstrapV1{ + Version: coroProgramBootstrapVersionV2, + Flags: coroProgramCapabilityWorkerV2, + StepHash: bootstrapA.StepHash, + } + changedCapabilities, err := coroProgramManifestHashV1(ctx, []string{a, b}, bootstrapC) + if err != nil { + t.Fatal(err) + } + if changedCapabilities == withBootstrap { + t.Fatal("manifest hash ignored bootstrap program capabilities") + } if _, err := coroProgramManifestHashV1(ctx, []string{a, b}, bootstrapA, bootstrapB); err == nil { t.Fatal("manifest hash accepted multiple bootstrap tables") } diff --git a/internal/build/coro_spawn_native_e2e_test.go b/internal/build/coro_spawn_native_e2e_test.go index 46cd8880e9..2ee6c8368f 100644 --- a/internal/build/coro_spawn_native_e2e_test.go +++ b/internal/build/coro_spawn_native_e2e_test.go @@ -285,8 +285,8 @@ func TestCoroChannelAndClosedStaticSpawnNativeNoStdlibRuntimeE2E(t *testing.T) { prog.TypeSizes(types.SizesFor("gc", runtime.GOARCH)) defer prog.Dispose() - userObject, anchor, setupSymbol, checkSymbol := buildCoroSpawnNativeE2EUser(t, prog, temp) - entryObject := buildCoroSpawnNativeE2EEntry(t, prog, temp, anchor) + userObject, anchor, setupSymbol, checkSymbol, capabilities := buildCoroSpawnNativeE2EUser(t, prog, temp) + entryObject := buildCoroSpawnNativeE2EEntry(t, prog, temp, anchor, capabilities) driverObject := buildCoroSpawnNativeE2EDriver(t, prog, temp, setupSymbol, checkSymbol) runtimeObjects := buildCoroSpawnNativeE2ERuntimeIsland(t, temp) runtimeArchive := filepath.Join(temp, "libllgo-coro-runtime-island.a") @@ -318,9 +318,12 @@ func TestCoroChannelAndClosedStaticSpawnNativeNoStdlibRuntimeE2E(t *testing.T) { } } -func buildCoroSpawnNativeE2EUser(t *testing.T, prog llssa.Program, temp string) (object, anchor, setupSymbol, checkSymbol string) { +func buildCoroSpawnNativeE2EUser(t *testing.T, prog llssa.Program, temp string) ( + object, anchor, setupSymbol, checkSymbol string, + capabilities coro.ProgramCapabilities, +) { return buildCoroSpawnNativeE2EUserSource( - t, prog, temp, coroSpawnNativeE2ESource, true, coro.TargetCapabilities(0), + t, prog, temp, coroSpawnNativeE2ESource, true, coro.TargetCapabilities(0), true, ) } @@ -330,7 +333,11 @@ func buildCoroSpawnNativeE2EUserSource( temp, source string, enableChannel bool, targetCapabilities coro.TargetCapabilities, -) (object, anchor, setupSymbol, checkSymbol string) { + requireRuntimeContext bool, +) ( + object, anchor, setupSymbol, checkSymbol string, + capabilities coro.ProgramCapabilities, +) { t.Helper() ssaPkg, files := buildCoroPlanTestPackage(t, coroSpawnNativeE2EPackage, source, nil) universe, err := cl.PrepareEmissionUniverseWithOptions(prog, nil, []cl.EmissionPackage{{ @@ -386,6 +393,7 @@ func buildCoroSpawnNativeE2EUserSource( EmissionUniverse: ssaUniverse, FunctionIDs: functionIDs, MaxPlainInstructions: -1, + ClassifyLocalBody: universe.CoroLocalBodyFacts, ClassifyFunction: func(fn *ssa.Function) (coro.SSAFunctionPolicy, error) { effect := coro.NoSuspend if _, required := spawnSeeded[fn]; required { @@ -486,6 +494,10 @@ func buildCoroSpawnNativeE2EUserSource( if err != nil { t.Fatal(err) } + if mainPlan, ok := plan.FunctionPlan(mainFn); requireRuntimeContext && + (!ok || !mainPlan.Exec.Contains(coro.NeedsRuntimeContext)) { + t.Fatalf("select main lacks inferred runtime-context requirement: %+v, present=%t", mainPlan, ok) + } compilation := &cl.Compilation{ CoroPlan: plan, @@ -503,6 +515,10 @@ func buildCoroSpawnNativeE2EUserSource( if err != nil { t.Fatal(err) } + capabilities, err = compilation.CoroProgramCapabilities() + if err != nil { + t.Fatal("resolve native E2E program capabilities:", err) + } module := pkg.Module() runCoroSpawnNativeE2EPasses(t, prog, module) ir := module.String() @@ -518,18 +534,28 @@ func buildCoroSpawnNativeE2EUserSource( if module.NamedFunction(setupSymbol).IsNil() { t.Fatalf("compiled E2E user module has no plain setup %q:\n%s", setupSymbol, ir) } - return emitCoroSpawnNativeE2EObject(t, prog, module, filepath.Join(temp, "user.o")), match[1], setupSymbol, checkSymbol + return emitCoroSpawnNativeE2EObject(t, prog, module, filepath.Join(temp, "user.o")), match[1], setupSymbol, checkSymbol, capabilities } -func buildCoroSpawnNativeE2EEntry(t *testing.T, prog llssa.Program, temp, anchor string) string { +func buildCoroSpawnNativeE2EEntry( + t *testing.T, + prog llssa.Program, + temp, anchor string, + capabilities coro.ProgramCapabilities, +) string { t.Helper() conf := &Config{ BuildMode: BuildModeExe, Goos: runtime.GOOS, Goarch: runtime.GOARCH} ctx := &context{prog: prog, buildConf: conf} + flags, err := coroProgramCapabilityFlagsV2(capabilities) + if err != nil { + t.Fatal("encode native E2E program capabilities:", err) + } bootstrap := &coroProgramBootstrapV1{ Version: coroProgramBootstrapVersionV2, + Flags: flags, Steps: []coroProgramBootstrapStepV1{ {Kind: coroProgramStepDirectPlainV1, Role: coroProgramStepRoleRuntimeInitV2, FunctionID: "e2e-runtime-init", Target: "__llgo_coro_e2e_runtime_init"}, {Kind: coroProgramStepDirectPlainV1, Role: coroProgramStepRoleABIInitV2, FunctionID: "e2e-abi-init", Target: "init$abitypes"}, @@ -656,21 +682,25 @@ func buildCoroSpawnNativeE2EDriver(t *testing.T, prog llssa.Program, temp, setup newChanBody := newChan.MakeBody(1) newChanBody.Return(newChanBody.Call(rawNewChan.Expr, newChan.Param(0), newChan.Param(1))) rawTrySend := pkg.NewFunc("command-line-arguments.CoroChanTrySend", newSignature( - []types.Type{pointer, pointer, intType}, []types.Type{boolType}, + []types.Type{pointer, pointer, pointer, intType}, []types.Type{boolType}, ), llssa.InGo) trySend := pkg.NewFunc(llssa.PkgRuntime+".CoroChanTrySend", newSignature( - []types.Type{pointer, pointer, intType}, []types.Type{boolType}, + []types.Type{pointer, pointer, pointer, intType}, []types.Type{boolType}, ), llssa.InGo) trySendBody := trySend.MakeBody(1) - trySendBody.Return(trySendBody.Call(rawTrySend.Expr, trySend.Param(0), trySend.Param(1), trySend.Param(2))) + trySendBody.Return(trySendBody.Call( + rawTrySend.Expr, trySend.Param(0), trySend.Param(1), trySend.Param(2), trySend.Param(3), + )) rawTryRecv := pkg.NewFunc("command-line-arguments.CoroChanTryRecv", newSignature( - []types.Type{pointer, pointer, intType}, []types.Type{boolType, boolType}, + []types.Type{pointer, pointer, pointer, intType}, []types.Type{boolType, boolType}, ), llssa.InGo) tryRecv := pkg.NewFunc(llssa.PkgRuntime+".CoroChanTryRecv", newSignature( - []types.Type{pointer, pointer, intType}, []types.Type{boolType, boolType}, + []types.Type{pointer, pointer, pointer, intType}, []types.Type{boolType, boolType}, ), llssa.InGo) tryRecvBody := tryRecv.MakeBody(1) - tryRecvResult := tryRecvBody.Call(rawTryRecv.Expr, tryRecv.Param(0), tryRecv.Param(1), tryRecv.Param(2)) + tryRecvResult := tryRecvBody.Call( + rawTryRecv.Expr, tryRecv.Param(0), tryRecv.Param(1), tryRecv.Param(2), tryRecv.Param(3), + ) tryRecvBody.Return(tryRecvBody.Extract(tryRecvResult, 0), tryRecvBody.Extract(tryRecvResult, 1)) chanOpSliceType := types.NewSlice(prog.RuntimeType("ChanOp").RawType()) uint32Type := types.Typ[types.Uint32] @@ -781,6 +811,7 @@ func buildCoroSpawnNativeE2ERuntimeIsland(t *testing.T, temp string) []string { filepath.Join("..", "..", "runtime", "internal", "runtime", "coro_panic_payload.go"), filepath.Join("..", "..", "runtime", "internal", "runtime", "coro_panic_trace_release.go"), filepath.Join("..", "..", "runtime", "internal", "runtime", "coro_executor_driver_worker_llgo.go"), + filepath.Join("..", "..", "runtime", "internal", "runtime", "coro_worker_completion_window_llgo.go"), filepath.Join("..", "..", "runtime", "internal", "runtime", "coro_operation_capacity.go"), filepath.Join("..", "..", "runtime", "internal", "runtime", "coro_spawn.go"), filepath.Join("..", "..", "runtime", "internal", "runtime", "coro_physical_thread_capacity_native_llgo.go"), @@ -805,6 +836,7 @@ func buildCoroSpawnNativeE2ERuntimeIsland(t *testing.T, temp string) []string { requireCoroRuntimeIslandProductionSource(t, files, "coro_current_task_route_default.go") requireCoroRuntimeIslandProductionSource(t, files, "coro_ready_distribution_default.go") requireCoroRuntimeIslandProductionSource(t, files, "coro_target_executor_retired_default.go") + requireCoroRuntimeIslandProductionSource(t, files, "coro_worker_completion_window_llgo.go") requireCoroRuntimeIslandProductionSource(t, files, "coro_worker_completion_program_llgo.go") requireCoroRuntimeIslandProductionSource(t, files, "coro_worker_result_llgo.go") requireCoroRuntimeIslandProductionSource(t, files, "coro_nil_fault.go") @@ -1073,9 +1105,9 @@ func assertCoroSpawnNativeE2ELinkedSymbols(t *testing.T, executable string) { "__llgo_coro_doorbell_poll_one_v1", "__llgo_coro_spawn_begin_v1", "__llgo_coro_spawn_commit_v1", - "__llgo_coro_chan_send_park_v1", - "__llgo_coro_chan_recv_park_v1", - "__llgo_coro_chan_resume_v1", + "__llgo_coro_chan_send_try_park_v2", + "__llgo_coro_chan_recv_try_park_v2", + "__llgo_coro_chan_resume_v2", "__llgo_coro_fault_prepare_v1", "__llgo_coro_fault_prepare_v2", "__llgo_coro_fault_payload_v2", diff --git a/internal/build/coro_static_outcome_stdlib_test.go b/internal/build/coro_static_outcome_stdlib_test.go new file mode 100644 index 0000000000..ab7496dfc2 --- /dev/null +++ b/internal/build/coro_static_outcome_stdlib_test.go @@ -0,0 +1,363 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/goplus/llgo/internal/coro" + "golang.org/x/tools/go/ssa" +) + +func TestCoroStaticStdlibAtomicMethodPublishesOutcomeEntry(t *testing.T) { + t.Setenv(llgoBuildCache, "off") + source := filepath.Join(t.TempDir(), "main.go") + if err := os.WriteFile(source, []byte(`package main + +import "sync/atomic" + +var value atomic.Uint64 + +func main() { value.Add(1) } +`), 0o644); err != nil { + t.Fatal(err) + } + + var ( + atomicPlan coro.FunctionPlan + atomicFacts coro.SSAFunctionBodyFacts + atomicID coro.FunctionID + found bool + factsFound bool + diagnostic string + candidates int + ) + conf := NewDefaultConf(ModeGen) + conf.ForceRebuild = true + conf.CoroPlanBuilder = func(input CoroPlanInput) (*coro.SSAPlan, error) { + if input.EmissionUniverse != nil && input.localBodyFacts != nil { + for _, fn := range input.EmissionUniverse.Functions() { + if fn == nil || fn.Pkg == nil || fn.Pkg.Pkg == nil || fn.Name() != "Add" || + !strings.HasSuffix(fn.Pkg.Pkg.Path(), "/sync/atomic") || + !strings.Contains(fn.String(), "Uint64") { + continue + } + facts, err := input.localBodyFacts(fn) + if err != nil { + return nil, err + } + atomicFacts = facts + ids := coro.FunctionIDConfig{} + if input.augmentFunctionIDs != nil { + ids = input.augmentFunctionIDs(ids) + } + atomicID, err = coro.StableFunctionID(fn, ids) + if err != nil { + return nil, err + } + factsFound = true + break + } + } + plan, err := defaultCoroPlanBuilder(input) + if err != nil || !factsFound { + return plan, err + } + if got, ok := plan.BasePlan().Lookup(atomicID); ok { + atomicPlan = got + diagnostic += fmt.Sprintf("builder-plan=%+v; ", got) + } + return plan, nil + } + conf.CoroPlanObserver = func(_ *ssa.Package, plan *coro.SSAPlan) { + if found || plan == nil { + return + } + for _, function := range plan.Functions() { + fn := function.Function + if fn != nil && fn.Pkg != nil && fn.Pkg.Pkg != nil && fn.Name() == "Add" { + candidates++ + if candidates <= 32 { + diagnostic += fmt.Sprintf("candidate=%s path=%s; ", fn.String(), fn.Pkg.Pkg.Path()) + } + } + if fn == nil || fn.Pkg == nil || fn.Pkg.Pkg == nil || + !strings.HasSuffix(fn.Pkg.Pkg.Path(), "/sync/atomic") || fn.Name() != "Add" || + !strings.Contains(fn.String(), "Uint64") { + continue + } + found = true + atomicPlan = function.Plan + } + if !found { + return + } + for _, function := range plan.Functions() { + fn := function.Function + if fn == nil { + continue + } + for _, block := range fn.Blocks { + for _, instruction := range block.Instrs { + call, ok := instruction.(ssa.CallInstruction) + if !ok { + continue + } + callee := "" + if common := call.Common(); common != nil && common.StaticCallee() != nil { + callee = common.StaticCallee().String() + } + callPlan, planned := plan.CallPlan(call) + relevant := callee != "" && strings.Contains(callee, "Uint64") && strings.HasSuffix(callee, ".Add") + if !relevant { + for _, target := range callPlan.Targets { + if target == atomicPlan.ID { + relevant = true + break + } + } + } + if !relevant && fn.Pkg != nil && fn.Pkg.Pkg != nil && + strings.HasSuffix(fn.Pkg.Pkg.Path(), "/sync/atomic") && fn.Name() == "Add" { + relevant = true + } + if !relevant { + continue + } + diagnostic += fmt.Sprintf( + "owner=%s call=%s callee=%s elided=%t planned=%t plan=%+v; ", + fn.String(), call.String(), callee, plan.ElidesCall(call), planned, callPlan, + ) + } + } + } + } + _, err := Do([]string{source}, conf) + if err != nil { + t.Fatalf("compile stdlib static-outcome fixture: %v", err) + } + if !found { + t.Fatalf("sync/atomic.(*Uint64).Add is absent from the whole-program coroutine plan (%d Add candidates): %s", candidates, diagnostic) + } + if !factsFound { + t.Fatal("sync/atomic.(*Uint64).Add has no frozen ProgramIR local-body facts") + } + if !atomicPlan.AtomicCostProof.ProvesOutcomePlain() || atomicPlan.AtomicCost == 0 { + t.Fatalf( + "sync/atomic.(*Uint64).Add plan = %+v, facts = %+v, want a static outcome entry; source calls: %s", + atomicPlan, atomicFacts, diagnostic, + ) + } +} + +func TestCoroSpawnWaitGroupStaticOutcomeCoverage(t *testing.T) { + t.Setenv(llgoBuildCache, "off") + source := filepath.Join(t.TempDir(), "main.go") + if err := os.WriteFile(source, []byte(`package main + +import "sync" + +func main() { + var done sync.WaitGroup + done.Add(1) + go done.Done() + done.Wait() +} +`), 0o644); err != nil { + t.Fatal(err) + } + + targets := []string{ + "(*sync.WaitGroup).Add", + "(*sync.WaitGroup).Done", + "runtime.semaRelease", + "github.com/goplus/llgo/runtime/internal/runtime.coroKeyedPostOneV2", + } + seen := make(map[string]coro.FunctionPlan) + frozenFacts := make(map[string]coro.SSAFunctionBodyFacts) + allFacts := make(map[*ssa.Function]coro.SSAFunctionBodyFacts) + traces := make(map[string]string) + observed := false + conf := NewDefaultConf(ModeGen) + conf.ForceRebuild = true + conf.CoroPlanBuilder = func(input CoroPlanInput) (*coro.SSAPlan, error) { + if input.EmissionUniverse != nil && input.localBodyFacts != nil { + for _, fn := range input.EmissionUniverse.Functions() { + if fn == nil { + continue + } + facts, err := input.localBodyFacts(fn) + if err != nil { + return nil, err + } + allFacts[fn] = facts + for _, target := range targets { + if fn.String() == target || strings.HasSuffix(fn.String(), target) || strings.Contains(fn.String(), target) { + frozenFacts[target] = facts + } + } + } + } + return defaultCoroPlanBuilder(input) + } + conf.CoroPlanObserver = func(_ *ssa.Package, plan *coro.SSAPlan) { + if plan == nil || observed { + return + } + observed = true + for _, function := range plan.Functions() { + if function.Function == nil { + continue + } + name := function.Function.String() + if strings.Contains(name, "WaitGroup") { + t.Logf("WaitGroup candidate: %s", name) + } + if name == "(*sync.WaitGroup).Add" { + var text bytes.Buffer + function.Function.WriteTo(&text) + t.Logf("WaitGroup.Add SSA:\n%s", text.String()) + for _, lowered := range plan.LoweredCalls(function.Function) { + targetPlan, _ := plan.FunctionPlan(lowered.Target) + t.Logf("WaitGroup.Add lowered %s -> %s: no-unwind=%t unwind-only=%t status-elided=%t raw=%t plan=%+v", + lowered.LogicalName, lowered.Target.String(), lowered.NoUnwind, lowered.UnwindOnly, + lowered.ExplicitStatusElided, lowered.RawPlain, targetPlan) + } + for _, block := range function.Function.Blocks { + for _, instruction := range block.Instrs { + call, ok := instruction.(ssa.CallInstruction) + if !ok { + continue + } + callPlan, planned := plan.CallPlan(call) + t.Logf("WaitGroup.Add call %s: planned=%t elided=%t plan=%+v", call.String(), planned, plan.ElidesCall(call), callPlan) + } + } + for _, atomicBlock := range allFacts[function.Function].AtomicPath.Blocks { + for _, occurrence := range atomicBlock.Calls { + callPlan, planned := plan.CallPlan(occurrence.Instruction) + for _, id := range callPlan.Targets { + target, _ := plan.Function(id) + targetPlan, _ := plan.BasePlan().Lookup(id) + t.Logf("WaitGroup.Add reachable call %s: planned=%t target=%s plan=%+v facts=%+v", + occurrence.Instruction.String(), planned, target.String(), targetPlan, allFacts[target]) + } + } + } + } + for _, target := range targets { + if name == target || strings.HasSuffix(name, target) || strings.Contains(name, target) { + seen[target] = function.Plan + traces[target] = fmt.Sprintf( + "wait=%s; await=%s; yield=%s", + plan.SuspensionEffectTrace(function.Function, coro.WaitForeign), + plan.SuspensionEffectTrace(function.Function, coro.AwaitStructured), + plan.SuspensionEffectTrace(function.Function, coro.YieldOnly), + ) + t.Logf("matched %s as %s", target, name) + } + } + } + for _, rootName := range []string{"(*sync.WaitGroup).Add", "github.com/goplus/llgo/runtime/internal/lib/runtime.sync_fatal", "github.com/goplus/llgo/runtime/internal/lib/runtime.sync_runtime_Semrelease"} { + var root *ssa.Function + for _, function := range plan.Functions() { + if function.Function != nil && function.Function.String() == rootName { + root = function.Function + break + } + } + if root == nil { + continue + } + seenClosure := map[*ssa.Function]bool{} + var visit func(*ssa.Function, int) + visit = func(owner *ssa.Function, depth int) { + if owner == nil || seenClosure[owner] || depth > 5 { + return + } + seenClosure[owner] = true + ownerPlan, _ := plan.FunctionPlan(owner) + facts := allFacts[owner] + t.Logf("static closure root=%s depth=%d owner=%s static=%t emission=%s effect=%s exec=%s local=%t lowered=%d", + rootName, depth, owner.String(), ownerPlan.HasStaticOutcome(), ownerPlan.Emission, + ownerPlan.Effect, ownerPlan.Exec, facts.StaticOutcomeLocal, len(plan.LoweredCalls(owner))) + if facts.AtomicPath != nil { + for _, block := range facts.AtomicPath.Blocks { + for _, occurrence := range block.Calls { + callPlan, ok := plan.CallPlan(occurrence.Instruction) + if !ok || len(callPlan.Targets) != 1 { + continue + } + target, _ := plan.Function(callPlan.Targets[0]) + visit(target, depth+1) + } + } + } + for _, lowered := range plan.LoweredCalls(owner) { + if !lowered.ExplicitStatusElided { + visit(lowered.Target, depth+1) + } + } + } + visit(root, 0) + } + for _, function := range plan.Functions() { + fn := function.Function + if fn == nil || fn.String() != "github.com/goplus/llgo/runtime/internal/runtime.StringCat" { + continue + } + t.Logf("StringCat owner plan=%+v facts=%+v", function.Plan, allFacts[fn]) + for _, lowered := range plan.LoweredCalls(fn) { + targetPlan, _ := plan.FunctionPlan(lowered.Target) + t.Logf("StringCat lowered %s -> %s: record=%+v target=%+v", lowered.LogicalName, lowered.Target, lowered, targetPlan) + } + for _, block := range fn.Blocks { + for _, instruction := range block.Instrs { + call, ok := instruction.(ssa.CallInstruction) + if !ok { + continue + } + callPlan, planned := plan.CallPlan(call) + t.Logf("StringCat source call %s: planned=%t elided=%t plan=%+v", call, planned, plan.ElidesCall(call), callPlan) + for _, targetID := range callPlan.Targets { + targetPlan, _ := plan.BasePlan().Lookup(targetID) + target, _ := plan.Function(targetID) + t.Logf("StringCat source target %s: %+v", target, targetPlan) + } + } + } + } + } + if _, err := Do([]string{source}, conf); err != nil { + t.Fatalf("compile spawn/WaitGroup static-outcome fixture: %v", err) + } + for _, name := range targets { + plan, ok := seen[name] + if !ok { + t.Errorf("%s is absent from the whole-program plan", name) + continue + } + t.Logf("%s: emission=%s static=%t declared=%s local=%s effect=%s declared-exec=%s local-exec=%s exec=%s proof=%s cost=%d facts=%+v trace=%s", name, plan.Emission, plan.StaticOutcome, plan.DeclaredEffect, plan.LocalEffect, plan.Effect, plan.DeclaredExec, plan.LocalExec, plan.Exec, plan.AtomicCostProof, plan.AtomicCost, frozenFacts[name], traces[name]) + } +} diff --git a/internal/build/coro_stdlib_sync_acceptance_test.go b/internal/build/coro_stdlib_sync_acceptance_test.go index c0281d48ee..d186a22c2c 100644 --- a/internal/build/coro_stdlib_sync_acceptance_test.go +++ b/internal/build/coro_stdlib_sync_acceptance_test.go @@ -276,6 +276,7 @@ func assertCoroStdlibSyncRuntimeSelection(t *testing.T, fixture coroStdlibSyncFi "coro_target_wait_timer_llgo.go": false, "coro_timer_owner_llgo.go": false, "coro_worker_completion_fleet_llgo.go": false, + "coro_worker_completion_window_llgo.go": false, "coro_worker_native_llgo.go": false, "coro_worker_owner_llgo.go": false, "coro_worker_result_llgo.go": false, diff --git a/internal/build/coro_tls_destructor_test.go b/internal/build/coro_tls_destructor_test.go index 554fb0270c..2a00824528 100644 --- a/internal/build/coro_tls_destructor_test.go +++ b/internal/build/coro_tls_destructor_test.go @@ -534,6 +534,8 @@ func __llgo_coro_program_continue_v1(uint32) {} func __llgo_coro_frame_allocator_bootstrap_v1() {} func __llgo_coro_frame_alloc_v1() {} func __llgo_coro_frame_publish_v1() {} +func __llgo_coro_frame_publish_v3() {} +func __llgo_coro_frame_destroy_commit_v2() {} func __llgo_coro_await_prepare_v1() {} func __llgo_coro_preempt_poll_v1() bool { return false } func __llgo_coro_yield_prepare_v1() {} diff --git a/internal/build/main_module.go b/internal/build/main_module.go index 26e6fd47f3..a96d8ba87c 100644 --- a/internal/build/main_module.go +++ b/internal/build/main_module.go @@ -345,6 +345,7 @@ func emitCoroProgramManifest(ctx *context, pkg llssa.Package, cfg *genConfig) co } bootstrap = pkg.NewCoroProgramBootstrap(coroProgramBootstrapSymbolV2, llssa.CoroProgramBootstrapOptions{ Version: coroProgramBootstrapVersionV2, + Flags: cfg.coroBootstrap.Flags, // The runtime validates one program ABI identity across the manifest // and startup table. StepHash is an input to this final manifest hash, // not a second externally visible ABI identity. diff --git a/internal/cabi/cabi.go b/internal/cabi/cabi.go index ac94320db3..79bfaaf2b6 100644 --- a/internal/cabi/cabi.go +++ b/internal/cabi/cabi.go @@ -604,7 +604,6 @@ func (p *Transformer) transformCallInstr(m llvm.Module, ctx llvm.Context, call l } nft, attrs, paramMap := p.transformFuncType(ctx, &info) preloweredSRet := call.GetCallSiteEnumAttribute(1, llvm.AttributeKindID("sret")) - reflectMethodByNameAttr := call.GetCallSiteStringAttribute(-1, "llgo.reflect.methodbyname") b := ctx.NewBuilder() b.SetInsertPointBefore(call) @@ -664,15 +663,13 @@ func (p *Transformer) transformCallInstr(m llvm.Module, ctx llvm.Context, call l // updateCallAttr receives the replacement call, but closure-context // attributes must be read from the original call before it is erased. updateCallAttr := func(replacement llvm.Value) { + copyCallSiteFunctionAttrs(call, replacement) for i, attr := range attrs { replacement.AddCallSiteAttribute(i, attr) } if !preloweredSRet.IsNil() { replacement.AddCallSiteAttribute(1, preloweredSRet) } - if !reflectMethodByNameAttr.IsNil() { - replacement.AddCallSiteAttribute(-1, reflectMethodByNameAttr) - } if remappedReflectMethodByNameArgAttrIndex >= 0 { replacement.AddCallSiteAttribute(remappedReflectMethodByNameArgAttrIndex, ctx.CreateStringAttribute( "llgo.reflect.methodbyname.name", "1", @@ -834,6 +831,12 @@ var closureEnvAttributeKinds = []uint{ llvm.AttributeKindID("swiftself"), } +func copyCallSiteFunctionAttrs(from, to llvm.Value) { + for _, attr := range from.GetCallSiteAttributesAtIndex(-1) { + to.AddCallSiteAttribute(-1, attr) + } +} + func copyClosureEnvFunctionAttrs(from, to llvm.Value, paramMap []int) { for oldIndex, newIndex := range paramMap { if newIndex == 0 { diff --git a/internal/cabi/cabi_patch_test.go b/internal/cabi/cabi_patch_test.go index 21ff95e613..0478ad8e72 100644 --- a/internal/cabi/cabi_patch_test.go +++ b/internal/cabi/cabi_patch_test.go @@ -110,9 +110,11 @@ entry: define %Value @caller(ptr %g, ptr %out, ptr nest %env, %Value %value) { entry: - %result = call %Value @callee(ptr %g, ptr %out, ptr nest %env, %Value %value) + %result = call %Value @callee(ptr %g, ptr %out, ptr nest %env, %Value %value) #0 ret %Value %result } + +attributes #0 = { nounwind "llgo.callsite.test"="value" } ` ctx := llvm.NewContext() defer ctx.Dispose() @@ -129,6 +131,15 @@ entry: t.Fatal(err) } defer mod.Dispose() + coroElideSafe := llvm.AttributeKindID("coro_elide_safe") + if coroElideSafe != 0 { + caller := mod.NamedFunction("caller") + call := caller.FirstBasicBlock().FirstInstruction().IsACallInst() + if call.IsNil() { + t.Fatalf("test fixture has no direct callee call:\n%s", caller.String()) + } + call.AddCallSiteAttribute(-1, ctx.CreateEnumAttribute(coroElideSafe, 0)) + } prog := llssa.NewProgram(&llssa.Target{GOOS: "linux", GOARCH: "amd64"}) defer prog.Dispose() @@ -162,6 +173,15 @@ entry: if attr := nestedCall.GetCallSiteEnumAttribute(3, nest); !attr.IsNil() { t.Fatalf("C ABI lowering left nest on the old call parameter:\n%s", caller.String()) } + if attr := nestedCall.GetCallSiteEnumAttribute(-1, llvm.AttributeKindID("nounwind")); attr.IsNil() { + t.Fatalf("C ABI lowering lost a standard function-index call attribute:\n%s", caller.String()) + } + if attr := nestedCall.GetCallSiteStringAttribute(-1, "llgo.callsite.test"); attr.IsNil() { + t.Fatalf("C ABI lowering lost a string function-index call attribute:\n%s", caller.String()) + } + if coroElideSafe != 0 && nestedCall.GetCallSiteEnumAttribute(-1, coroElideSafe).IsNil() { + t.Fatalf("C ABI lowering lost coro_elide_safe:\n%s", caller.String()) + } if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { t.Fatalf("C ABI closure-env module is invalid: %v\n%s", err, mod.String()) } diff --git a/internal/coro/callable_contract.go b/internal/coro/callable_contract.go index 85265a16a7..a4d5840e0e 100644 --- a/internal/coro/callable_contract.go +++ b/internal/coro/callable_contract.go @@ -319,6 +319,12 @@ func CallableContractExecConstraints(contract CallableContract) ExecFlags { switch contract.Reentry { case ReentryUnknown: flags |= OpaqueExec + case ReentryManagedCallback: + // A direct same-thread C call may reenter generated Go while the outer + // physical frame is live. The callback must observe that frame's logical + // G rather than the executor placeholder, so retain the ambient runtime + // context for the complete caller closure. + flags |= NeedsRuntimeContext } switch contract.Memory { case MemoryUnknown, MemoryRetained: diff --git a/internal/coro/dimensions.go b/internal/coro/dimensions.go index a86f6304a2..4f254cd58b 100644 --- a/internal/coro/dimensions.go +++ b/internal/coro/dimensions.go @@ -39,15 +39,21 @@ const ( // unavailable. Verifiers must reject it in restricted contexts unless a // compatible external summary replaces it. OpaqueExec + // NeedsRuntimeContext records that physical execution observes or mutates + // the ambient runtime G (for example through getg/getgIfPresent), or crosses + // a boundary which can synchronously reenter Go and observe that G. It is a + // propagated may-property: a caller can omit the per-resume runtime-context + // install only when its complete managed call closure lacks this bit. + NeedsRuntimeContext ) const validExecFlags = BlockForeign | ThreadAffine | IRQUnsafe | NeedsPreempt | - MayUnwind | NeedsCleanupFrame | NoReturn | PanicOnly | OpaqueExec + MayUnwind | NeedsCleanupFrame | NoReturn | PanicOnly | OpaqueExec | NeedsRuntimeContext // propagatedExecFlags are conservative "may" constraints inherited by a // managed caller. Control-flow guarantees and local lowering requirements are // intentionally excluded. -const propagatedExecFlags = ThreadAffine | IRQUnsafe | MayUnwind | OpaqueExec +const propagatedExecFlags = ThreadAffine | IRQUnsafe | MayUnwind | OpaqueExec | NeedsRuntimeContext var execFlagNames = [...]struct { bit ExecFlags @@ -62,6 +68,7 @@ var execFlagNames = [...]struct { {NoReturn, "no-return"}, {PanicOnly, "panic-only"}, {OpaqueExec, "opaque"}, + {NeedsRuntimeContext, "needs-runtime-context"}, } func (f ExecFlags) Validate() error { diff --git a/internal/coro/graph.go b/internal/coro/graph.go index 5f5741a128..af053ee8d5 100644 --- a/internal/coro/graph.go +++ b/internal/coro/graph.go @@ -93,7 +93,7 @@ func (k CallKind) validate() error { // produced by CallableContractExecConstraints. Keeping this narrow is what // prevents an invocation refinement from suppressing independent IRQUnsafe, // MayUnwind, or future non-contract constraints. -const callableContractExecFlags = ThreadAffine | OpaqueExec +const callableContractExecFlags = ThreadAffine | OpaqueExec | NeedsRuntimeContext // CallEdge is a statically resolved call graph edge. type CallEdge struct { @@ -238,6 +238,15 @@ func (g *Graph) AddFunction(spec FunctionSpec) error { if spec.ManagedEntry != ManagedEntryNone && spec.External != ExternalKnown { return fmt.Errorf("coro: function %q: managed entry %s requires an external-known producer", spec.ID, spec.ManagedEntry) } + if spec.StaticOutcome { + if spec.External != ExternalKnown || spec.ManagedEntry != ManagedEntryCoroutine || + spec.AtomicCostProof != AtomicCostUnproven || spec.AtomicCost != 0 || spec.AtomicCostCertificate != "" || + spec.Seed&^(YieldOnly|AwaitStructured|OutcomeStructured) != 0 || + !spec.Seed.Contains(OutcomeStructured) || + spec.Exec&(BlockForeign|ThreadAffine|NeedsCleanupFrame|OpaqueExec) != 0 { + return fmt.Errorf("coro: function %q: invalid imported unbounded static outcome capability", spec.ID) + } + } switch spec.AtomicCostProof { case AtomicCostUnproven: if spec.AtomicCost != 0 || spec.AtomicCostCertificate != "" || spec.ManagedEntry == ManagedEntryOutcomePlain { @@ -245,10 +254,14 @@ func (g *Graph) AddFunction(spec FunctionSpec) error { } case AtomicCostLeaf, AtomicCostDAG: if spec.AtomicCost == 0 || spec.External != ExternalKnown || - spec.ManagedEntry != ManagedEntryOutcomePlain || spec.Seed != OutcomeStructured || + (spec.ManagedEntry != ManagedEntryCoroutine && spec.ManagedEntry != ManagedEntryOutcomePlain) || + !spec.Seed.Contains(OutcomeStructured) || spec.Seed&^(AwaitStructured|OutcomeStructured) != 0 || spec.Exec&^MayUnwind != 0 { return fmt.Errorf("coro: function %q: invalid imported outcome-plain capability", spec.ID) } + if spec.ManagedEntry == ManagedEntryOutcomePlain && spec.Seed != OutcomeStructured { + return fmt.Errorf("coro: function %q: outcome-plain primary has effect %s", spec.ID, spec.Seed) + } if err := validateSHA256Hex("atomic-cost certificate", spec.AtomicCostCertificate); err != nil { return fmt.Errorf("coro: function %q: %w", spec.ID, err) } @@ -966,6 +979,7 @@ func (g *Graph) AnalyzeWithConfig(config GraphAnalysisConfig) (*Plan, error) { emission := bodyEmissionFor(managedDemands[id], rawPlainDemands[id], effects[id], spec.External) managedEntry := ManagedEntryNone atomicCost, atomicCostProof, atomicCostCertificate := uint64(0), AtomicCostUnproven, "" + staticOutcome := false if spec.External == ExternalKnown { managedEntry = spec.ManagedEntry if managedEntry == ManagedEntryNone { @@ -978,6 +992,7 @@ func (g *Graph) AnalyzeWithConfig(config GraphAnalysisConfig) (*Plan, error) { } } atomicCost, atomicCostProof, atomicCostCertificate = spec.AtomicCost, spec.AtomicCostProof, spec.AtomicCostCertificate + staticOutcome = spec.StaticOutcome } else { switch emission { case EmitPlain, EmitRawPlain: @@ -1002,6 +1017,7 @@ func (g *Graph) AnalyzeWithConfig(config GraphAnalysisConfig) (*Plan, error) { AtomicCost: atomicCost, AtomicCostProof: atomicCostProof, AtomicCostCertificate: atomicCostCertificate, + StaticOutcome: staticOutcome, FuncRep: rep, External: spec.External, Recursive: recursive[id], diff --git a/internal/coro/graph_test.go b/internal/coro/graph_test.go index 6db939e9fb..7d130fad54 100644 --- a/internal/coro/graph_test.go +++ b/internal/coro/graph_test.go @@ -42,6 +42,26 @@ func TestAnalyzeDirectPropagation(t *testing.T) { } } +func TestAnalyzeRuntimeContextRequirementPropagatesOnlyThroughManagedCalls(t *testing.T) { + g := NewGraph() + mustAddFunction(t, g, FunctionSpec{ID: "caller"}) + mustAddFunction(t, g, FunctionSpec{ID: "spawn-owner"}) + mustAddFunction(t, g, FunctionSpec{ID: "ambient-leaf", Exec: NeedsRuntimeContext}) + mustAddCall(t, g, CallEdge{Caller: "caller", Callee: "ambient-leaf", Kind: CallDirect}) + mustAddCall(t, g, CallEdge{Caller: "spawn-owner", Callee: "ambient-leaf", Kind: CallSpawn}) + + plan, err := g.Analyze() + if err != nil { + t.Fatal(err) + } + if caller := mustLookup(t, plan, "caller"); !caller.Exec.Contains(NeedsRuntimeContext) { + t.Fatalf("direct caller execution flags = %s", caller.Exec) + } + if owner := mustLookup(t, plan, "spawn-owner"); owner.Exec.Contains(NeedsRuntimeContext) { + t.Fatalf("spawn owner inherited independently resumed child context: %s", owner.Exec) + } +} + func TestAnalyzeRecursiveSCC(t *testing.T) { g := NewGraph() for _, spec := range []FunctionSpec{ diff --git a/internal/coro/library_effect_summary.go b/internal/coro/library_effect_summary.go index 6b5d266c5f..802403f25c 100644 --- a/internal/coro/library_effect_summary.go +++ b/internal/coro/library_effect_summary.go @@ -29,13 +29,15 @@ import ( ) const ( + LibraryEffectSummaryVersion = "v9" + // LibraryEffectSummarySchema is the producer ABI summary embedded in LLGo // package objects and archives. It is deliberately independent from the // whole-program Summary and PlanDigest schemas: consumer demand may change, // while these producer effects and physically available entries may not. - LibraryEffectSummarySchema = "llgo.coro.library-effect-summary.v6" + LibraryEffectSummarySchema = "llgo.coro.library-effect-summary." + LibraryEffectSummaryVersion - LibraryEffectSummaryDigestDomain = "llgo.coro.library-effect-summary.digest.v6" + LibraryEffectSummaryDigestDomain = "llgo.coro.library-effect-summary.digest." + LibraryEffectSummaryVersion // LibraryEffectSummarySection is the portable object-section identity. // Mach-O emission uses the same leaf name in an explicit segment. @@ -53,7 +55,7 @@ const ( var libraryEffectSummaryRecordMagic = [16]byte{ 'L', 'L', 'G', 'O', 'C', 'O', 'R', 'O', - 'E', 'F', 'F', 'E', 'C', 'T', 0, 6, + 'E', 'F', 'F', 'E', 'C', 'T', 0, 9, } const libraryEffectSummaryRecordHeaderSize = len(libraryEffectSummaryRecordMagic) + 4 + sha256.Size @@ -98,8 +100,19 @@ type LibraryEffectFunction struct { AtomicCost uint64 `json:"atomic_cost"` AtomicCostProof AtomicCostProof `json:"atomic_cost_proof"` AtomicCostCertificate string `json:"atomic_cost_certificate"` - PrimarySymbol string `json:"primary_symbol"` - RawPlainSymbol string `json:"raw_plain_symbol,omitempty"` + // StaticOutcome publishes an unbounded exact-static synchronous twin. It is + // orthogonal to AtomicCostProof and never replaces the coroutine primary. + StaticOutcome bool `json:"static_outcome"` + PrimarySymbol string `json:"primary_symbol"` + // OutcomePlainSymbol is the exact synchronous static-call entry certified by + // AtomicCostProof. It equals PrimarySymbol when outcome-plain is primary and + // names a separate twin when ManagedEntry remains coroutine. + OutcomePlainSymbol string `json:"outcome_plain_symbol,omitempty"` + RawPlainSymbol string `json:"raw_plain_symbol,omitempty"` +} + +func (function LibraryEffectFunction) HasStaticOutcome() bool { + return function.AtomicCostProof.ProvesOutcomePlain() || function.StaticOutcome } // LibraryEffectForeignCallable publishes one exact producer-side C @@ -337,15 +350,39 @@ func (function LibraryEffectFunction) validate() error { return fmt.Errorf("coro: library function %q has an invalid outcome-plain entry capability", function.ID) } } - if function.ManagedEntry == ManagedEntryOutcomePlain { - if !function.AtomicCostProof.ProvesOutcomePlain() || function.AtomicCost == 0 { - return fmt.Errorf("coro: library function %q has an outcome entry without an atomic-cost proof", function.ID) + if function.AtomicCostProof.ProvesOutcomePlain() { + if function.AtomicCost == 0 { + return fmt.Errorf("coro: library function %q has an outcome entry without an atomic cost", function.ID) } if err := validateSHA256Hex("library function atomic-cost certificate", function.AtomicCostCertificate); err != nil { return err } + if err := validateStableIdentityText("library function outcome-plain symbol", function.OutcomePlainSymbol); err != nil { + return err + } } else if function.AtomicCostProof != AtomicCostUnproven || function.AtomicCost != 0 || function.AtomicCostCertificate != "" { return fmt.Errorf("coro: library function %q has atomic-cost metadata without an outcome entry", function.ID) + } else if function.OutcomePlainSymbol != "" && !function.StaticOutcome { + return fmt.Errorf("coro: library function %q has an outcome symbol without an atomic-cost proof", function.ID) + } + if function.StaticOutcome { + if function.AtomicCostProof.ProvesOutcomePlain() || function.ManagedEntry != ManagedEntryCoroutine || + function.Primary != PrimaryCoroutine || + function.Effect&^(YieldOnly|AwaitStructured|OutcomeStructured) != 0 || + !function.Effect.Contains(OutcomeStructured) || + function.Exec&(BlockForeign|ThreadAffine|NeedsCleanupFrame|OpaqueExec) != 0 { + return fmt.Errorf("coro: library function %q has an invalid unbounded static outcome capability", function.ID) + } + if err := validateStableIdentityText("library function outcome-plain symbol", function.OutcomePlainSymbol); err != nil { + return err + } + } + if function.ManagedEntry == ManagedEntryOutcomePlain && function.OutcomePlainSymbol != function.PrimarySymbol { + return fmt.Errorf("coro: library function %q outcome primary and static outcome symbols differ", function.ID) + } + if function.ManagedEntry == ManagedEntryCoroutine && + function.OutcomePlainSymbol == function.PrimarySymbol { + return fmt.Errorf("coro: library function %q coroutine primary and static outcome symbols are identical", function.ID) } if err := validateStableIdentityText("library function primary symbol", function.PrimarySymbol); err != nil { return err @@ -666,6 +703,7 @@ func (function LibraryEffectFunction) ImportedPolicy() (SSAFunctionPolicy, error AtomicCost: function.AtomicCost, AtomicCostProof: function.AtomicCostProof, AtomicCostCertificate: function.AtomicCostCertificate, + StaticOutcome: function.StaticOutcome, IgnoreBody: true, External: ExternalKnown, OverrideExternal: true, diff --git a/internal/coro/library_effect_summary_test.go b/internal/coro/library_effect_summary_test.go index 8923a31597..a027fe77a4 100644 --- a/internal/coro/library_effect_summary_test.go +++ b/internal/coro/library_effect_summary_test.go @@ -20,12 +20,26 @@ import ( "bytes" "crypto/sha256" "encoding/hex" + "strconv" "strings" "testing" ) var testAtomicCostCertificate = strings.Repeat("a", 64) +func TestLibraryEffectSummaryVersionIdentity(t *testing.T) { + version := strconv.Itoa(int(libraryEffectSummaryRecordMagic[len(libraryEffectSummaryRecordMagic)-1])) + if LibraryEffectSummaryVersion != "v"+version { + t.Fatalf("record magic version %s disagrees with version %q", version, LibraryEffectSummaryVersion) + } + if !strings.HasSuffix(LibraryEffectSummarySchema, ".v"+version) { + t.Fatalf("record magic version %s disagrees with schema %q", version, LibraryEffectSummarySchema) + } + if !strings.HasSuffix(LibraryEffectSummaryDigestDomain, ".v"+version) { + t.Fatalf("record magic version %s disagrees with digest domain %q", version, LibraryEffectSummaryDigestDomain) + } +} + func testLibraryEffectMetadata() LibraryEffectMetadata { return LibraryEffectMetadata{ FunctionIDSchema: FunctionIDSchema, @@ -238,6 +252,7 @@ func TestLibraryEffectSummaryCarriesOutcomePlainCapability(t *testing.T) { AtomicCostProof: AtomicCostLeaf, AtomicCostCertificate: testAtomicCostCertificate, PrimarySymbol: "example/outcome.Leaf$outcome", + OutcomePlainSymbol: "example/outcome.Leaf$outcome", }} summary.ForeignCallables = nil summary.ExportBindings = nil @@ -282,7 +297,7 @@ func TestLibraryEffectSummaryCarriesOutcomePlainCapability(t *testing.T) { func(function *LibraryEffectFunction) { function.AtomicCost = 0 }, func(function *LibraryEffectFunction) { function.AtomicCostProof = AtomicCostUnproven }, func(function *LibraryEffectFunction) { function.AtomicCostCertificate = "" }, - func(function *LibraryEffectFunction) { function.ManagedEntry = ManagedEntryCoroutine }, + func(function *LibraryEffectFunction) { function.OutcomePlainSymbol = "" }, } { invalid := summary invalid.Functions = append([]LibraryEffectFunction(nil), summary.Functions...) @@ -291,6 +306,16 @@ func TestLibraryEffectSummaryCarriesOutcomePlainCapability(t *testing.T) { t.Fatalf("invalid outcome library capability was accepted: %+v", invalid.Functions[0]) } } + dual := summary + dual.Functions = append([]LibraryEffectFunction(nil), summary.Functions...) + dual.Functions[0].ManagedEntry = ManagedEntryCoroutine + dual.Functions[0].Effect = AwaitStructured | OutcomeStructured + dual.Functions[0].FuncRep = Dispatch + dual.Functions[0].PrimarySymbol = "example/outcome.Leaf$coro" + dual.Functions[0].OutcomePlainSymbol = "example/outcome.Leaf$outcome" + if _, err := dual.MarshalStable(); err != nil { + t.Fatalf("marshal coroutine primary plus outcome twin: %v", err) + } } func TestLibraryEffectSummaryFailsClosed(t *testing.T) { diff --git a/internal/coro/lowering_facts.go b/internal/coro/lowering_facts.go index 4cda8e00d1..cede9c5a02 100644 --- a/internal/coro/lowering_facts.go +++ b/internal/coro/lowering_facts.go @@ -32,12 +32,12 @@ import ( // LoweringFactsSchema identifies the pointer-free sparse lowering-fact wire // format. It remains independently versioned from PlanDigestSchema; the // production plan digest binds this exact schema and its canonical digest. -const LoweringFactsSchema = "llgo.coro.lowering-facts.v0" +const LoweringFactsSchema = "llgo.coro.lowering-facts.v1" // LoweringFactsDigestDomain separates lowering-fact hashes from source, // function identity, plan, and future overlay hashes that happen to contain // the same bytes. -const LoweringFactsDigestDomain = "llgo.coro.lowering-facts.digest.v0" +const LoweringFactsDigestDomain = "llgo.coro.lowering-facts.digest.v1" // EmissionInstanceID identifies one physical owner/patch/ABI context for a // logical function. Owner and Context must be canonical, checkout-independent diff --git a/internal/coro/plan.go b/internal/coro/plan.go index 0fbe159a54..e1f85c9543 100644 --- a/internal/coro/plan.go +++ b/internal/coro/plan.go @@ -124,6 +124,9 @@ type FunctionSpec struct { AtomicCost uint64 AtomicCostProof AtomicCostProof AtomicCostCertificate string + // StaticOutcome is producer-owned only for an ExternalKnown declaration. + // Defined bodies derive it after the complete SSA effect/call fixed point. + StaticOutcome bool // TrustedBoundedRecursion is a frontend proof that this function's // recursion depth is bounded independently of scheduler preemption. It // suppresses the automatic recursive-SCC preemption seed only when every @@ -230,18 +233,26 @@ type FunctionPlan struct { // ManagedEntry is the exact physical ABI invoked by managed callers. Unlike // Emission it remains meaningful for an imported EmitExternal declaration. ManagedEntry ManagedEntryKind - // AtomicCost is the longest path-sensitive semantic work bound from the - // last scheduler cut to terminal completion. It is meaningful only when - // AtomicCostProof is not AtomicCostUnproven. + // AtomicCost is the longest path-sensitive semantic work bound of the + // optional synchronous outcome entry. It is meaningful only when + // AtomicCostProof is not AtomicCostUnproven. The ordinary ManagedEntry may + // remain a coroutine when dynamic/function-value consumers also exist; exact + // static calls can still select the separately emitted outcome entry. AtomicCost uint64 // AtomicCostProof records the closed-world proof class which authorized the - // bound. EmitOutcomePlain requires a proof. Declared/local effects remain - // unchanged; the final effect may drop AwaitStructured after every such edge - // has been replaced by a proven synchronous outcome call. + // bound. EmitOutcomePlain requires a proof and makes that entry primary; an + // EmitCoroutine plan with a proof emits both the coroutine primary and the + // static outcome entry. Declared/local effects remain unchanged; the final + // effect may drop AwaitStructured only when the outcome entry is primary. AtomicCostProof AtomicCostProof // AtomicCostCertificate is the content-addressed path proof. Imported and // local outcome entries require it; unproven functions must leave it empty. AtomicCostCertificate string + // StaticOutcome records a separately emitted synchronous outcome entry for + // exact static calls. Unlike AtomicCostProof it makes no finite scheduler-gap + // claim: source loops may execute synchronously. It is therefore only a twin + // of a coroutine primary, never the managed primary itself. + StaticOutcome bool // FuncRep is direct unless value-flow requested an open dispatch boundary. FuncRep FuncRep External ExternalKind @@ -263,6 +274,13 @@ type FunctionPlan struct { RawPlainEntry bool } +// HasStaticOutcome reports that an exact managed static call may use the +// synchronous outcome ABI. An atomic-cost proof is the bounded form of this +// capability; StaticOutcome is the explicitly unbounded form. +func (plan FunctionPlan) HasStaticOutcome() bool { + return plan.AtomicCostProof.ProvesOutcomePlain() || plan.StaticOutcome +} + // bodyEmissionFor derives the physical body independently from logical // PrimaryKind and function-value representation. No-demand nodes materialize // no symbol; a demanded external node retains a declaration, while a demanded @@ -346,6 +364,38 @@ func validateManagedEntryPlan(plan FunctionPlan) error { plan.ID, plan.Effect, plan.Exec, plan.FuncRep, plan.Recursive, plan.RawPlainDemand, ) } + if !plan.AtomicCostProof.ProvesOutcomePlain() { + return fmt.Errorf("coro: function %q has an outcome-plain primary without a static outcome proof", plan.ID) + } + } + + if plan.StaticOutcome { + if plan.AtomicCostProof.ProvesOutcomePlain() || plan.AtomicCost != 0 || plan.AtomicCostCertificate != "" { + return fmt.Errorf("coro: function %q mixes bounded and unbounded static outcome capabilities", plan.ID) + } + if plan.Recursive || plan.Exec&(BlockForeign|ThreadAffine|NeedsCleanupFrame|OpaqueExec) != 0 || + plan.Effect&^(YieldOnly|AwaitStructured|OutcomeStructured) != 0 || + !plan.Effect.Contains(OutcomeStructured) { + return fmt.Errorf( + "coro: function %q has an invalid unbounded static outcome capability (effect=%s exec=%s recursive=%t)", + plan.ID, plan.Effect, plan.Exec, plan.Recursive, + ) + } + if plan.External == Defined { + if plan.Emission != EmitCoroutine || plan.ManagedEntry != ManagedEntryCoroutine || + plan.Primary != PrimaryCoroutine { + return fmt.Errorf("coro: function %q has an unbounded static outcome without a coroutine primary twin", plan.ID) + } + } else if plan.External == ExternalKnown { + if plan.ManagedEntry != ManagedEntryCoroutine || plan.Primary != PrimaryExternal { + return fmt.Errorf( + "coro: external function %q has an unbounded static outcome with managed entry %s", + plan.ID, plan.ManagedEntry, + ) + } + } else { + return fmt.Errorf("coro: function %q has an unbounded static outcome with external kind %s", plan.ID, plan.External) + } } switch plan.AtomicCostProof { @@ -354,10 +404,24 @@ func validateManagedEntryPlan(plan FunctionPlan) error { return fmt.Errorf("coro: function %q has outcome entry/cost without an atomic-cost proof", plan.ID) } case AtomicCostLeaf, AtomicCostDAG: - if plan.AtomicCost == 0 || plan.ManagedEntry != ManagedEntryOutcomePlain || - (plan.External != Defined && plan.External != ExternalKnown) { + if plan.AtomicCost == 0 || (plan.External != Defined && plan.External != ExternalKnown) { return fmt.Errorf("coro: function %q has an invalid atomic-cost capability", plan.ID) } + if plan.External == Defined { + primaryOutcome := plan.Emission == EmitOutcomePlain && plan.ManagedEntry == ManagedEntryOutcomePlain + dualOutcome := plan.Emission == EmitCoroutine && plan.ManagedEntry == ManagedEntryCoroutine + if !primaryOutcome && !dualOutcome { + return fmt.Errorf( + "coro: function %q has an atomic outcome capability without an outcome primary or coroutine primary twin (emission=%s managed=%s)", + plan.ID, plan.Emission, plan.ManagedEntry, + ) + } + } else if plan.ManagedEntry != ManagedEntryCoroutine && plan.ManagedEntry != ManagedEntryOutcomePlain { + return fmt.Errorf( + "coro: external function %q has an atomic outcome capability with managed entry %s", + plan.ID, plan.ManagedEntry, + ) + } if err := validateSHA256Hex("atomic-cost certificate", plan.AtomicCostCertificate); err != nil { return fmt.Errorf("coro: function %q: %w", plan.ID, err) } diff --git a/internal/coro/plan_digest.go b/internal/coro/plan_digest.go index 64271d3cfb..ab0bafcc8d 100644 --- a/internal/coro/plan_digest.go +++ b/internal/coro/plan_digest.go @@ -32,11 +32,17 @@ import ( // PlanDigestSchema is the independent canonical schema used for archive cache // identity. It is deliberately separate from SummarySchema: summaries remain // diagnostic snapshots, while this document covers every lowering plan site. -const PlanDigestSchema = "llgo.coro.plan-digest.v33" +const PlanDigestSchema = "llgo.coro.plan-digest.v35" // Current experimental ABI identities. Keeping these in the analysis package // gives build, cache, and lowering code one version source of truth. const ( + // FrameDescriptorNoRuntimeContextV1 is the compiler/runtime physical-frame + // ABI bit proving that the complete inlined managed execution closure does + // not observe or replace the ambient runtime G. Runtime validation owns the + // identically valued flag in runtime/internal/coro. + FrameDescriptorNoRuntimeContextV1 uint32 = 1 << 1 + EntryResolutionABIV0 = "llgo.coro.entry-resolution.v0" PhysicalABIV0 = "llgo.coro.physical.v0" PhysicalABIV1 = "llgo.coro.physical.v1" @@ -172,6 +178,7 @@ type planDigestFunction struct { AtomicCost uint64 `json:"atomic_cost"` AtomicCostProof uint8 `json:"atomic_cost_proof"` AtomicCostCertificate string `json:"atomic_cost_certificate"` + StaticOutcome bool `json:"static_outcome"` FuncRep uint8 `json:"func_rep"` External uint8 `json:"external"` Recursive bool `json:"recursive"` @@ -894,6 +901,7 @@ func (p *SSAPlan) canonicalDigestFunctions() ([]planDigestFunction, error) { AtomicCost: plan.AtomicCost, AtomicCostProof: uint8(plan.AtomicCostProof), AtomicCostCertificate: plan.AtomicCostCertificate, + StaticOutcome: plan.StaticOutcome, FuncRep: uint8(plan.FuncRep), External: uint8(plan.External), Recursive: plan.Recursive, diff --git a/internal/coro/plan_digest_test.go b/internal/coro/plan_digest_test.go index b5a7fcf303..e85e1b33ea 100644 --- a/internal/coro/plan_digest_test.go +++ b/internal/coro/plan_digest_test.go @@ -313,8 +313,8 @@ import ( } func TestCoroPlanDigestRecordsWholeBuildRawPlainVariant(t *testing.T) { - if PlanDigestSchema != "llgo.coro.plan-digest.v33" { - t.Fatalf("plan digest schema = %q, want path-certificate schema v33", PlanDigestSchema) + if PlanDigestSchema != "llgo.coro.plan-digest.v35" { + t.Fatalf("plan digest schema = %q, want runtime-context proof schema v35", PlanDigestSchema) } prog, pkg := buildCoroTestSSA(t, "raw_variant_digest.go", `package coroid func root(seed int) int { diff --git a/internal/coro/ssa_borrowed_allocation.go b/internal/coro/ssa_borrowed_allocation.go new file mode 100644 index 0000000000..f594d4a7fb --- /dev/null +++ b/internal/coro/ssa_borrowed_allocation.go @@ -0,0 +1,491 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coro + +import ( + "go/token" + "go/types" + + "golang.org/x/tools/go/ssa" +) + +// SSABorrowedAllocationProof certifies that an allocation which x/tools +// conservatively marked Heap has no address path surviving its owning function +// return. Every static callee receiving that address is inspected transitively; +// dynamic calls, goroutines, defers, returns, stores outside the same tainted +// object, and body-less declarations fail closed. +// +// This is a storage-lifetime proof only. The physical planner still decides +// whether the object belongs in an LLVM coroutine frame or on an outcome-plain +// native stack, and separately enforces the target's stack-size/root profile. +type SSABorrowedAllocationProof struct { + Allocation *ssa.Alloc + FunctionsVisited uint32 + ParametersProven uint32 +} + +// SSABorrowedAllocationConfig defines which static callee body is authoritative +// for the final program. Frontends which replace a source SSA body with an +// intrinsic, foreign symbol, or other physical implementation must reject that +// body (or resolve it to an exact managed-Go definition); otherwise an inert +// declaration stub can incorrectly prove that an argument is not retained. +type SSABorrowedAllocationConfig struct { + ResolveCalleeBody func(*ssa.Function) (*ssa.Function, bool) +} + +type ssaBorrowParameterKey struct { + function *ssa.Function + index int +} + +type ssaBorrowProofState uint8 + +const ( + ssaBorrowProofUnknown ssaBorrowProofState = iota + ssaBorrowProofVisiting + ssaBorrowProofRejected + ssaBorrowProofAccepted +) + +type ssaBorrowedAllocationAnalyzer struct { + parameters map[ssaBorrowParameterKey]ssaBorrowProofState + functions map[*ssa.Function]struct{} + config SSABorrowedAllocationConfig +} + +// ProveSSABorrowedAllocation derives a closed interprocedural borrow proof for +// one exact SSA allocation. Non-heap allocations already have local identity +// and therefore deliberately do not receive this reclassification proof. +func ProveSSABorrowedAllocation(allocation *ssa.Alloc) (SSABorrowedAllocationProof, bool) { + return ProveSSABorrowedAllocationWithConfig(allocation, SSABorrowedAllocationConfig{}) +} + +// ProveSSABorrowedAllocationWithConfig derives the same closed proof while +// requiring every transitive call edge to use the frontend's authoritative +// physical body. A rejected or unresolved callee fails closed. +func ProveSSABorrowedAllocationWithConfig( + allocation *ssa.Alloc, + config SSABorrowedAllocationConfig, +) (SSABorrowedAllocationProof, bool) { + proof := SSABorrowedAllocationProof{Allocation: allocation} + if allocation == nil || !allocation.Heap || allocation.Parent() == nil || + allocation.Referrers() == nil { + return proof, false + } + analyzer := &ssaBorrowedAllocationAnalyzer{ + parameters: make(map[ssaBorrowParameterKey]ssaBorrowProofState), + functions: make(map[*ssa.Function]struct{}), + config: config, + } + if !analyzer.proveAddressValue(allocation.Parent(), allocation) { + return proof, false + } + proof.FunctionsVisited = uint32(len(analyzer.functions)) + for _, state := range analyzer.parameters { + if state == ssaBorrowProofAccepted { + proof.ParametersProven++ + } + } + return proof, true +} + +func (analyzer *ssaBorrowedAllocationAnalyzer) proveParameter(function *ssa.Function, index int) bool { + if analyzer == nil || function == nil || len(function.Blocks) == 0 || + index < 0 || index >= len(function.Params) || function.Params[index] == nil { + return false + } + key := ssaBorrowParameterKey{function: function, index: index} + switch analyzer.parameters[key] { + case ssaBorrowProofVisiting, ssaBorrowProofAccepted: + // Borrow is a coinductive property: a recursive cycle is safe unless an + // edge leaving that cycle is later rejected. + return true + case ssaBorrowProofRejected: + return false + } + analyzer.parameters[key] = ssaBorrowProofVisiting + if !analyzer.proveAddressValue(function, function.Params[index]) { + analyzer.parameters[key] = ssaBorrowProofRejected + return false + } + analyzer.parameters[key] = ssaBorrowProofAccepted + return true +} + +func (analyzer *ssaBorrowedAllocationAnalyzer) proveAddressValue( + function *ssa.Function, + root ssa.Value, +) bool { + if analyzer == nil || function == nil || root == nil || root.Parent() != function || + root.Referrers() == nil { + return false + } + analyzer.functions[function] = struct{}{} + + // Values in tainted carry the root address itself or an aggregate which may + // contain its exact Go pointer type. A fresh allocation's unrelated pointer + // fields cannot acquire the root through ordinary typed assignment: any + // static callee which tries to write the root into other storage is inspected + // below and rejected. Keeping this distinction is important for transaction + // objects whose ordinary endpoint fields lead to a much larger runtime graph + // while only one exact self-certificate field can carry the allocation. + tainted := map[ssa.Value]bool{root: true} + // interior is deliberately narrower than tainted: it contains only address + // expressions structurally rooted in the fresh allocation, never a pointer + // loaded from one of its fields. This distinction makes an exact self-field + // store local while still rejecting a store through an arbitrary loaded + // pointer. + interior := map[ssa.Value]bool{root: true} + queue := []ssa.Value{root} + for head := 0; head < len(queue); head++ { + value := queue[head] + refs := value.Referrers() + if refs == nil { + return false + } + for _, reference := range *refs { + derived, ok := ssaBorrowDerivedValue(reference, value, root.Type()) + if !ok || derived == nil || tainted[derived] { + continue + } + tainted[derived] = true + if interior[value] && ssaBorrowDerivedInteriorAddress(reference, value) { + interior[derived] = true + } + queue = append(queue, derived) + } + } + + for value := range tainted { + refs := value.Referrers() + if refs == nil { + return false + } + for _, reference := range *refs { + if derived, ok := ssaBorrowDerivedValue(reference, value, root.Type()); ok && + derived != nil && tainted[derived] { + continue + } + switch reference := reference.(type) { + case *ssa.DebugRef: + case *ssa.Field: + if reference.X != value { + return false + } + case *ssa.Index: + if reference.X != value { + return false + } + case *ssa.Extract: + if reference.Tuple != value { + return false + } + case *ssa.UnOp: + if reference.X != value || reference.Op != token.MUL { + return false + } + case *ssa.BinOp: + if reference.X != value && reference.Y != value || + (reference.Op != token.EQL && reference.Op != token.NEQ) { + return false + } + case *ssa.Store: + switch { + case reference.Addr == value: + // Writing ordinary data into the borrowed object does not + // change the lifetime of its address. + case reference.Val == value && interior[reference.Addr] && + ssaBorrowTypeMayCarryRoot(value.Type(), root.Type()): + // An exact self pointer (or an aggregate containing that exact + // typed pointer) may live inside the fresh object itself. Loads + // of such a field remain tainted and are checked independently. + default: + // The address may merely have been loaded from a pointer field + // and therefore need not denote storage inside the borrowed + // object. Converted/boxed roots also fail this exact typed-self + // exception. + return false + } + case *ssa.Call: + if !analyzer.proveCallArgument(reference, value) { + return false + } + case *ssa.Return, *ssa.Go, *ssa.Defer, *ssa.Send, *ssa.MapUpdate, + *ssa.MakeClosure: + return false + default: + // Every value-producing address/aggregate propagation is + // handled by ssaBorrowDerivedValue. An unknown observer is an + // open lifetime edge and must fail closed. + return false + } + } + } + return true +} + +func ssaBorrowDerivedInteriorAddress(reference ssa.Instruction, source ssa.Value) bool { + switch reference := reference.(type) { + case *ssa.FieldAddr: + return reference.X == source + case *ssa.IndexAddr: + return reference.X == source + case *ssa.ChangeType: + return reference.X == source && ssaBorrowPointerIdentityType(reference.Type()) && + ssaBorrowPointerIdentityType(source.Type()) + case *ssa.Convert: + return reference.X == source && ssaBorrowPointerIdentityType(reference.Type()) && + ssaBorrowPointerIdentityType(source.Type()) + } + return false +} + +func ssaBorrowPointerIdentityType(typ types.Type) bool { + if typ == nil { + return false + } + switch typ := types.Unalias(typ).Underlying().(type) { + case *types.Pointer: + return typ != nil + case *types.Basic: + return typ.Kind() == types.UnsafePointer + } + return false +} + +func (analyzer *ssaBorrowedAllocationAnalyzer) proveCallArgument(call *ssa.Call, value ssa.Value) bool { + if analyzer == nil || call == nil || call.Common() == nil || call.Common().IsInvoke() { + return false + } + common := call.Common() + callee := common.StaticCallee() + if callee == nil { + builtin, ok := common.Value.(*ssa.Builtin) + if !ok || (builtin.Name() != "len" && builtin.Name() != "cap") { + return false + } + return true + } + if analyzer.config.ResolveCalleeBody != nil { + var resolved bool + callee, resolved = analyzer.config.ResolveCalleeBody(callee) + if !resolved || callee == nil { + return false + } + } + found := false + for index, argument := range common.Args { + if argument != value { + continue + } + found = true + if !analyzer.proveParameter(callee, index) { + return false + } + } + return found +} + +func ssaBorrowDerivedValue(reference ssa.Instruction, source ssa.Value, rootType types.Type) (ssa.Value, bool) { + if reference == nil || source == nil || rootType == nil { + return nil, false + } + var derived ssa.Value + switch reference := reference.(type) { + case *ssa.FieldAddr: + if reference.X == source { + derived = reference + } + case *ssa.IndexAddr: + if reference.X == source { + derived = reference + } + case *ssa.Field: + if reference.X == source && ssaBorrowTypeMayCarryRoot(reference.Type(), rootType) { + derived = reference + } + case *ssa.Index: + if reference.X == source && ssaBorrowTypeMayCarryRoot(reference.Type(), rootType) { + derived = reference + } + case *ssa.Extract: + if reference.Tuple == source && ssaBorrowTypeMayCarryRoot(reference.Type(), rootType) { + derived = reference + } + case *ssa.UnOp: + if reference.X == source && reference.Op == token.MUL && + ssaBorrowTypeMayCarryRoot(reference.Type(), rootType) { + derived = reference + } + case *ssa.ChangeType: + if reference.X == source { + derived = reference + } + case *ssa.Convert: + if reference.X == source { + // Pointer-to-uintptr conversion still carries the lifetime even + // though the result's Go type no longer contains a pointer. + derived = reference + } + case *ssa.MultiConvert: + if reference.X == source { + derived = reference + } + case *ssa.ChangeInterface: + if reference.X == source { + derived = reference + } + case *ssa.MakeInterface: + if reference.X == source { + derived = reference + } + case *ssa.Slice: + if reference.X == source { + derived = reference + } + case *ssa.SliceToArrayPointer: + if reference.X == source { + derived = reference + } + case *ssa.TypeAssert: + if reference.X == source && ssaBorrowTypeMayCarryRoot(reference.Type(), rootType) { + derived = reference + } + case *ssa.Phi: + for _, edge := range reference.Edges { + if edge == source { + derived = reference + break + } + } + } + return derived, derived != nil +} + +func ssaBorrowTypeMayCarryRoot(typ, rootType types.Type) bool { + if typ == nil || rootType == nil { + return false + } + typ = types.Unalias(typ) + rootType = types.Unalias(rootType) + if ssaBorrowTypesMayMatch(typ, rootType) { + return true + } + switch typ := types.Unalias(typ).Underlying().(type) { + case *types.Interface: + // Boxing the root itself is handled by MakeInterface propagation. An + // interface loaded from the fresh object can carry it only after such a + // box was stored, and that store is rejected by the lifetime proof. + return false + case *types.Pointer: + return ssaBorrowTypesMayMatch(typ, rootType) + case *types.Slice, *types.Map, *types.Chan, *types.Signature: + return false + case *types.Basic: + // unsafe.Pointer/uintptr conversions are explicitly propagated from a + // tainted source. A fresh field of either type is not implicitly an alias. + return false + case *types.Array: + return ssaBorrowTypeMayCarryRoot(typ.Elem(), rootType) + case *types.Struct: + for index := 0; index < typ.NumFields(); index++ { + if ssaBorrowTypeMayCarryRoot(typ.Field(index).Type(), rootType) { + return true + } + } + case *types.Tuple: + for index := 0; index < typ.Len(); index++ { + if ssaBorrowTypeMayCarryRoot(typ.At(index).Type(), rootType) { + return true + } + } + case *types.TypeParam: + return true + } + return false +} + +// ssaBorrowTypesMayMatch answers only the identity facts needed by the borrow +// proof. x/tools SSA deliberately uses private opaque types for synthetic +// values such as range-function defer stacks. Passing either of those types to +// go/types.Identical panics because it is not one of go/types' closed concrete +// implementations. Unknown or synthetic shapes therefore return true here: +// the caller treats them as possibly carrying the root, follows the extra use, +// and fails the lifetime proof closed if it cannot account for that use. +func ssaBorrowTypesMayMatch(left, right types.Type) bool { + if left == nil || right == nil { + return false + } + left = types.Unalias(left) + right = types.Unalias(right) + if left == right { + return true + } + switch left := left.(type) { + case *types.Basic: + right, ok := right.(*types.Basic) + return ok && left.Kind() == right.Kind() + case *types.Array: + right, ok := right.(*types.Array) + return ok && left.Len() == right.Len() && + ssaBorrowTypesMayMatch(left.Elem(), right.Elem()) + case *types.Slice: + right, ok := right.(*types.Slice) + return ok && ssaBorrowTypesMayMatch(left.Elem(), right.Elem()) + case *types.Pointer: + right, ok := right.(*types.Pointer) + return ok && ssaBorrowTypesMayMatch(left.Elem(), right.Elem()) + case *types.Map: + right, ok := right.(*types.Map) + return ok && ssaBorrowTypesMayMatch(left.Key(), right.Key()) && + ssaBorrowTypesMayMatch(left.Elem(), right.Elem()) + case *types.Chan: + right, ok := right.(*types.Chan) + return ok && left.Dir() == right.Dir() && + ssaBorrowTypesMayMatch(left.Elem(), right.Elem()) + case *types.Named: + right, ok := right.(*types.Named) + if !ok { + return false + } + // The origin object is the exact declaration identity. Different generic + // arguments could make these types unequal, but returning true is the + // conservative lifetime answer and avoids inspecting an opaque argument. + return left.Origin().Obj() == right.Origin().Obj() + case *types.Struct: + _, ok := right.(*types.Struct) + return ok + case *types.Tuple: + _, ok := right.(*types.Tuple) + return ok + case *types.Signature: + _, ok := right.(*types.Signature) + return ok + case *types.Interface: + _, ok := right.(*types.Interface) + return ok + case *types.TypeParam: + _, ok := right.(*types.TypeParam) + return ok + case *types.Union: + _, ok := right.(*types.Union) + return ok + default: + // Private SSA type. It is intentionally opaque to this proof. + return true + } +} diff --git a/internal/coro/ssa_borrowed_allocation_test.go b/internal/coro/ssa_borrowed_allocation_test.go new file mode 100644 index 0000000000..66dc3cc1a2 --- /dev/null +++ b/internal/coro/ssa_borrowed_allocation_test.go @@ -0,0 +1,180 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coro + +import ( + "strings" + "testing" + + "golang.org/x/tools/go/ssa" +) + +func TestSSABorrowedAllocationProof(t *testing.T) { + _, pkg := buildCoroTestSSA(t, "borrowed_allocation.go", `package coroid + +type transaction struct { + self *transaction + endpoint *int + phase uint8 +} + +func begin(out *transaction, endpoint *int) bool { + if out == nil || *out != (transaction{}) { + return false + } + *out = transaction{self: out, endpoint: endpoint, phase: 1} + return true +} + +func beginEffect(value *transaction) bool { + if value == nil || value.self != value || value.phase != 1 { + return false + } + value.phase = 2 + return check(value) +} + +func check(value *transaction) bool { + return value.self == value && value.phase == 2 && value.endpoint != nil +} + +func safe(endpoint *int) bool { + var value transaction + if !begin(&value, endpoint) || !beginEffect(&value) { + return false + } + value = transaction{} + return true +} + +var escaped *transaction + +func escapeGlobal() { var value transaction; escaped = &value } +func escapeReturn() *transaction { var value transaction; return &value } +func borrow(value *transaction) { _ = value.phase } +func escapeGo() { var value transaction; go borrow(&value) } +func escapeDefer() { var value transaction; defer borrow(&value) } +func escapeDynamic(fn func(*transaction)) { var value transaction; fn(&value) } + +func recursiveSafe(value *transaction, depth int) { + if depth == 0 { return } + recursiveSafe(value, depth-1) + _ = value.phase +} + +func safeRecursive() bool { + var value transaction + recursiveSafe(&value, 2) + return value.phase == 0 +} + +func recurseA(value *transaction, escape bool) { recurseB(value, escape) } +func recurseB(value *transaction, escape bool) { + if escape { escaped = value; return } + recurseA(value, true) +} +func escapeRecursive() { var value transaction; recurseA(&value, false) } +`) + + safe := packageFunction(t, pkg, "safe") + safeAlloc := exactHeapAllocation(t, safe) + proof, ok := ProveSSABorrowedAllocation(safeAlloc) + if !ok || proof.Allocation != safeAlloc || proof.FunctionsVisited < 4 || proof.ParametersProven < 3 { + t.Fatalf("safe borrow proof = %+v, present=%t; want transitive begin/effect/check proof", proof, ok) + } + if proof, ok := ProveSSABorrowedAllocationWithConfig(safeAlloc, SSABorrowedAllocationConfig{ + ResolveCalleeBody: func(function *ssa.Function) (*ssa.Function, bool) { + if function != nil && function.Name() == "beginEffect" { + return nil, false + } + return function, function != nil + }, + }); ok { + t.Fatalf("safe allocation crossed a rejected physical callee body: %+v", proof) + } + recursiveAlloc := exactHeapAllocation(t, packageFunction(t, pkg, "safeRecursive")) + if proof, ok := ProveSSABorrowedAllocation(recursiveAlloc); !ok || proof.ParametersProven == 0 { + t.Fatalf("safe recursive borrow proof = %+v, present=%t", proof, ok) + } + + for _, name := range []string{ + "escapeGlobal", "escapeReturn", "escapeGo", "escapeDefer", "escapeDynamic", "escapeRecursive", + } { + allocation := exactHeapAllocation(t, packageFunction(t, pkg, name)) + if proof, ok := ProveSSABorrowedAllocation(allocation); ok { + t.Fatalf("%s unexpectedly received borrowed-allocation proof: %+v", name, proof) + } + } +} + +func TestSSABorrowedAllocationOpaqueSSATypeFailsClosed(t *testing.T) { + program, _ := buildCoroTestSSA(t, "borrowed_range_func.go", `package coroid +func seq(yield func(int) bool) { yield(1) } +func save(int) {} +func root() { + defer save(9) + for value := range seq { + defer save(value) + } +} +`) + + foundOpaque := false + for _, function := range matchingFunctions(program, func(*ssa.Function) bool { return true }) { + for _, block := range function.Blocks { + for _, instruction := range block.Instrs { + allocation, ok := instruction.(*ssa.Alloc) + if !ok { + continue + } + _, proven := ProveSSABorrowedAllocation(allocation) + if strings.Contains(allocation.Type().String(), "deferStack") { + foundOpaque = true + if proven { + t.Fatalf("synthetic opaque allocation %s received a borrow proof", allocation) + } + } + } + } + } + if !foundOpaque { + t.Fatal("range-function SSA did not contain its synthetic deferStack allocation") + } +} + +func exactHeapAllocation(t *testing.T, function *ssa.Function) *ssa.Alloc { + t.Helper() + var result *ssa.Alloc + for _, block := range function.Blocks { + for _, instruction := range block.Instrs { + allocation, ok := instruction.(*ssa.Alloc) + if !ok || !allocation.Heap { + continue + } + if result != nil { + t.Fatalf("%s has multiple heap allocations: %s and %s", function, result, allocation) + } + result = allocation + } + } + if result == nil { + t.Fatalf("%s has no heap allocation", function) + } + return result +} diff --git a/internal/coro/ssa_borrowed_runtime_test.go b/internal/coro/ssa_borrowed_runtime_test.go new file mode 100644 index 0000000000..44fb12fc90 --- /dev/null +++ b/internal/coro/ssa_borrowed_runtime_test.go @@ -0,0 +1,83 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coro + +import ( + "bytes" + "path/filepath" + "strings" + "testing" + + "golang.org/x/tools/go/packages" + "golang.org/x/tools/go/ssa" + "golang.org/x/tools/go/ssa/ssautil" +) + +func TestRuntimeChannelCommitTransactionsAreProvenBorrowed(t *testing.T) { + runtimeRoot, err := filepath.Abs(filepath.Join("..", "..", "runtime")) + if err != nil { + t.Fatal(err) + } + loaded, err := packages.Load(&packages.Config{ + Dir: runtimeRoot, + BuildFlags: []string{"-tags=llgo"}, + Mode: packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | + packages.NeedImports | packages.NeedDeps | packages.NeedTypes | + packages.NeedSyntax | packages.NeedTypesInfo | packages.NeedTypesSizes, + }, "./internal/runtime") + if err != nil { + t.Fatal(err) + } + if packages.PrintErrors(loaded) != 0 { + t.Fatal("load llgo runtime channel package") + } + program, roots := ssautil.AllPackages(loaded, ssa.SanityCheckFunctions|ssa.InstantiateGenerics) + program.Build() + if len(roots) != 1 || roots[0] == nil { + t.Fatalf("runtime SSA packages = %d, want one", len(roots)) + } + for _, name := range []string{ + "commitCoroRecvWaiterLockedWithContext", + "commitCoroSendWaiterLockedWithContext", + } { + function := roots[0].Func(name) + if function == nil { + t.Fatalf("runtime SSA has no %s", name) + } + var transaction *ssa.Alloc + for _, block := range function.Blocks { + for _, instruction := range block.Instrs { + allocation, ok := instruction.(*ssa.Alloc) + if ok && allocation.Heap && strings.Contains(allocation.Type().String(), "ChannelExternalCommit") { + transaction = allocation + } + } + } + if transaction == nil { + t.Fatalf("%s has no heap ChannelExternalCommit allocation", name) + } + if proof, ok := ProveSSABorrowedAllocation(transaction); !ok { + var dump bytes.Buffer + ssa.WriteFunction(&dump, function) + t.Fatalf("%s transaction lacks borrowed-allocation proof:\n%s", name, dump.String()) + } else if proof.FunctionsVisited < 2 || proof.ParametersProven == 0 { + t.Fatalf("%s transaction proof is not interprocedural: %+v", name, proof) + } + } +} diff --git a/internal/coro/ssa_callable_contract_test.go b/internal/coro/ssa_callable_contract_test.go index dbdb83d524..5fcf09611d 100644 --- a/internal/coro/ssa_callable_contract_test.go +++ b/internal/coro/ssa_callable_contract_test.go @@ -87,7 +87,7 @@ func caller() { foreign() } {"affinity unknown", func() CallableContract { c := base; c.Affinity = AffinityUnknown; return c }(), ThreadAffine}, {"owner thread", func() CallableContract { c := base; c.Affinity = AffinityOwnerThread; return c }(), ThreadAffine}, {"host main", func() CallableContract { c := base; c.Affinity = AffinityHostMain; return c }(), ThreadAffine}, - {"managed callback", func() CallableContract { c := base; c.Reentry = ReentryManagedCallback; return c }(), 0}, + {"managed callback", func() CallableContract { c := base; c.Reentry = ReentryManagedCallback; return c }(), NeedsRuntimeContext}, {"reentry unknown", func() CallableContract { c := base; c.Reentry = ReentryUnknown; return c }(), OpaqueExec}, {"memory retained", func() CallableContract { c := base; c.Memory = MemoryRetained; return c }(), OpaqueExec}, {"memory unknown", func() CallableContract { c := base; c.Memory = MemoryUnknown; return c }(), OpaqueExec}, diff --git a/internal/coro/ssa_outcome_plain_test.go b/internal/coro/ssa_outcome_plain_test.go index b9615ef4ad..69a4e2691c 100644 --- a/internal/coro/ssa_outcome_plain_test.go +++ b/internal/coro/ssa_outcome_plain_test.go @@ -476,7 +476,7 @@ func root(value any, fail bool) int { return middle(value, fail) } } } -func TestAnalyzeSSAOutcomePlainLeafFailsClosedWithoutBoundOrAtRoot(t *testing.T) { +func TestAnalyzeSSAOutcomePlainLeafFailsClosedWithoutBoundAndKeepsRootPrimary(t *testing.T) { prog, pkg := buildCoroTestSSA(t, "outcome_plain_reject.go", `package coroid func leaf(value any, fail bool) int { @@ -496,12 +496,13 @@ func caller(value any, fail bool) int { return leaf(value, fail) } return facts, nil } for _, test := range []struct { - name string - roots Roots - max int + name string + roots Roots + max int + wantTwin bool }{ {name: "budget disabled", roots: Roots{{Function: caller, ManagedDemand: AsyncDemand}}, max: -1}, - {name: "explicit root", roots: Roots{{Function: leaf, ManagedDemand: AsyncDemand}}, max: 64}, + {name: "explicit root", roots: Roots{{Function: leaf, ManagedDemand: AsyncDemand}}, max: 64, wantTwin: true}, } { t.Run(test.name, func(t *testing.T) { plan, err := AnalyzeSSA(prog, test.roots, SSAConfig{ @@ -513,7 +514,12 @@ func caller(value any, fail bool) int { return leaf(value, fail) } t.Fatal(err) } got := functionPlanFor(t, plan, leaf) - if got.Emission == EmitOutcomePlain || got.AtomicCostProof != AtomicCostUnproven || got.AtomicCost != 0 { + if test.wantTwin { + if got.Emission != EmitCoroutine || got.ManagedEntry != ManagedEntryCoroutine || + !got.AtomicCostProof.ProvesOutcomePlain() || got.AtomicCost == 0 { + t.Fatalf("root leaf plan = %+v, want coroutine primary plus static outcome entry", got) + } + } else if got.Emission == EmitOutcomePlain || got.AtomicCostProof != AtomicCostUnproven || got.AtomicCost != 0 { t.Fatalf("leaf plan = %+v, outcome-plain proof must fail closed", got) } }) diff --git a/internal/coro/ssa_plan.go b/internal/coro/ssa_plan.go index fe39c44c7b..3f3c6fc2ae 100644 --- a/internal/coro/ssa_plan.go +++ b/internal/coro/ssa_plan.go @@ -90,6 +90,10 @@ type SSAFunctionPolicy struct { // AtomicCostCertificate binds a producer-owned path proof to its exact // FunctionID, CFG projection and transitive callee certificates. AtomicCostCertificate string + // StaticOutcome carries a producer-owned unbounded synchronous twin for an + // ExternalKnown declaration. Local bodies derive the same capability from + // ProgramIR after effect and CallPlan finalization. + StaticOutcome bool // CallableIdentityCertificate is the execution-policy-neutral identity of // one exact managed C declaration. It may coexist with either a generic // behavior contract, a legacy physical capability, or neither. @@ -651,6 +655,12 @@ type SSAFunctionBodyFacts struct { // prove an acyclic call graph, and include every callee cost. OutcomePlainDAG bool OutcomePlainCallCount int + // StaticOutcomeLocal proves that every evaluated source instruction belongs + // to the wider synchronous-outcome language. Unlike OutcomePlainDAG it may + // contain CFG cycles and does not imply a bounded scheduler gap. Whole- + // program planning must still close every ordinary call over an exact plain + // or static-outcome target and reject hidden lowered calls. + StaticOutcomeLocal bool } func (facts SSAFunctionBodyFacts) validate(function *ssa.Function) error { @@ -1643,6 +1653,7 @@ func AnalyzeSSA(prog *ssa.Program, roots Roots, config SSAConfig) (*SSAPlan, err policy.AtomicCost = trusted.AtomicCost policy.AtomicCostProof = trusted.AtomicCostProof policy.AtomicCostCertificate = trusted.AtomicCostCertificate + policy.StaticOutcome = trusted.StaticOutcome policy.NeedsDispatch = policy.NeedsDispatch || trusted.NeedsDispatch policy.TrustedBoundedRecursion = trusted.TrustedBoundedRecursion _, rawFunctionAddressEntry := rawFunctionAddressEntries[fn] @@ -1706,6 +1717,17 @@ func AnalyzeSSA(prog *ssa.Program, roots Roots, config SSAConfig) (*SSAPlan, err if err != nil { return nil, err } + evaluatedSourceCalls := make(map[ssa.CallInstruction]bool) + for _, facts := range localBodyFacts { + if facts.AtomicPath == nil { + continue + } + for _, block := range facts.AtomicPath.Blocks { + for _, occurrence := range block.Calls { + evaluatedSourceCalls[occurrence.Instruction] = true + } + } + } graph := NewGraph() for _, fn := range included { @@ -1722,6 +1744,7 @@ func AnalyzeSSA(prog *ssa.Program, roots Roots, config SSAConfig) (*SSAPlan, err AtomicCost: policy.AtomicCost, AtomicCostProof: policy.AtomicCostProof, AtomicCostCertificate: policy.AtomicCostCertificate, + StaticOutcome: policy.StaticOutcome, TrustedBoundedRecursion: policy.TrustedBoundedRecursion, NeedsDispatch: policy.NeedsDispatch, RawPlainEntry: policy.RawPlainEntry, @@ -2048,6 +2071,9 @@ func AnalyzeSSA(prog *ssa.Program, roots Roots, config SSAConfig) (*SSAPlan, err ); err != nil { return nil, err } + if err := applySSAStaticOutcomePlans(base, byID, localBodyFacts, callPlans, loweredCalls, elidedCalls); err != nil { + return nil, err + } elidedCallSet := make(map[ssa.CallInstruction]struct{}, len(elidedCalls)) for call, elided := range elidedCalls { if elided { @@ -2144,7 +2170,7 @@ func validateSSAImportedOutcomePlainCosts(plan *Plan, maxAtomicCost int) error { return nil } for _, function := range plan.functions { - if function.External != ExternalKnown || function.ManagedEntry != ManagedEntryOutcomePlain || + if function.External != ExternalKnown || !function.AtomicCostProof.ProvesOutcomePlain() || function.ManagedDemand == NoDemand { continue } @@ -2189,7 +2215,9 @@ func applySSAOutcomePlainPlans( if maxAtomicCost < 0 { return nil } - blocked := make(map[FunctionID]bool) + // primaryBlocked records uses which require the ordinary coroutine entry. + // They no longer suppress a separately proven static outcome entry. + primaryBlocked := make(map[FunctionID]bool) direct := make(map[FunctionID]bool) type outcomeCall struct { instruction ssa.CallInstruction @@ -2201,10 +2229,10 @@ func applySSAOutcomePlainPlans( ids[function] = id } for _, root := range roots { - blocked[root.ID] = true + primaryBlocked[root.ID] = true } for reference := range graph.references { - blocked[reference.target] = true + primaryBlocked[reference.target] = true } for call, plan := range callPlans { exactStatic := call != nil && call.Common() != nil && call.Common().StaticCallee() != nil && @@ -2219,7 +2247,7 @@ func applySSAOutcomePlainPlans( if exactStatic { direct[target] = true } else { - blocked[target] = true + primaryBlocked[target] = true } } } @@ -2229,13 +2257,13 @@ func applySSAOutcomePlainPlans( continue } if id, ok := ids[call.Target]; ok { - blocked[id] = true + primaryBlocked[id] = true } } } eligible := func(plan FunctionPlan, function *ssa.Function, facts SSAFunctionBodyFacts, classified bool) bool { if function == nil || len(function.FreeVars) != 0 || !classified || facts.HasCycle || facts.AtomicPath == nil || - facts.InstructionCount <= 0 || blocked[plan.ID] || !direct[plan.ID] { + facts.InstructionCount <= 0 || !direct[plan.ID] { return false } for _, lowered := range loweredCalls[function] { @@ -2245,7 +2273,7 @@ func applySSAOutcomePlainPlans( } return plan.External == Defined && plan.Emission == EmitCoroutine && plan.ManagedEntry == ManagedEntryCoroutine && - plan.Primary == PrimaryCoroutine && plan.FuncRep == DirectCoro && + plan.Primary == PrimaryCoroutine && (plan.FuncRep == DirectCoro || plan.FuncRep == Dispatch) && plan.ManagedDemand != NoDemand && !plan.RawPlainDemand && !plan.RawPlainEntry && !plan.RawPlainOnly && !plan.Recursive && plan.Effect.Contains(OutcomeStructured) && plan.Effect&^(AwaitStructured|OutcomeStructured) == 0 && @@ -2255,15 +2283,21 @@ func applySSAOutcomePlainPlans( } selectPlan := func(index int, cost uint64, proof AtomicCostProof, certificate string) error { plan := base.functions[index] - plan.Emission = EmitOutcomePlain - plan.ManagedEntry = ManagedEntryOutcomePlain - // Every AwaitStructured dependency was proven to be a synchronous - // outcome entry before selection. Retain terminal outcome semantics but - // remove the now-elided physical await capability from this body. - plan.Effect = OutcomeStructured plan.AtomicCost = cost plan.AtomicCostProof = proof plan.AtomicCostCertificate = certificate + // A target with no dynamic/root/raw entry obligation can make the + // synchronous outcome ABI its managed primary as before. Otherwise retain + // the coroutine primary and emit the same proven outcome body as a static + // twin; exact direct calls select it while dynamic calls keep their ABI. + if !primaryBlocked[plan.ID] && plan.FuncRep == DirectCoro { + plan.Emission = EmitOutcomePlain + plan.ManagedEntry = ManagedEntryOutcomePlain + // Every AwaitStructured dependency was proven to be a synchronous + // outcome entry before selection. Retain terminal outcome semantics but + // remove the now-elided physical await capability from this body. + plan.Effect = OutcomeStructured + } if err := validateManagedEntryPlan(plan); err != nil { return fmt.Errorf("coro: select outcome-plain %s %q: %w", proof, plan.ID, err) } @@ -2273,8 +2307,7 @@ func applySSAOutcomePlainPlans( available := make(map[FunctionID]SSAAtomicCalleeCertificate) for _, plan := range base.functions { - if plan.External == ExternalKnown && plan.ManagedEntry == ManagedEntryOutcomePlain && - plan.AtomicCostProof.ProvesOutcomePlain() && plan.AtomicCost != 0 { + if plan.External == ExternalKnown && plan.AtomicCostProof.ProvesOutcomePlain() && plan.AtomicCost != 0 { available[plan.ID] = SSAAtomicCalleeCertificate{ Function: plan.ID, Cost: plan.AtomicCost, Certificate: plan.AtomicCostCertificate, } @@ -2341,6 +2374,181 @@ func applySSAOutcomePlainPlans( return nil } +// applySSAStaticOutcomePlans closes the exact-static no-suspend call graph +// after the bounded atomic cohort has been selected. The resulting twin uses +// the same explicit-status outcome ABI but deliberately carries no atomic cost +// certificate: a source CFG loop may compute for an unbounded interval. This +// is currently permitted only at exact static call sites; dynamic/function- +// value callers retain the ordinary preemptible coroutine primary. +func applySSAStaticOutcomePlans( + base *Plan, + byID map[FunctionID]*ssa.Function, + localBodyFacts map[*ssa.Function]SSAFunctionBodyFacts, + callPlans map[ssa.CallInstruction]SSACallPlan, + loweredCalls map[*ssa.Function][]SSALoweredCall, + elidedCalls map[ssa.CallInstruction]bool, +) error { + if base == nil { + return fmt.Errorf("coro: static-outcome planning requires a complete graph plan") + } + ids := make(map[*ssa.Function]FunctionID, len(byID)) + for id, function := range byID { + ids[function] = id + } + type staticCall struct { + target FunctionID + } + outgoing := make(map[FunctionID][]staticCall) + sourceOutgoing := make(map[FunctionID]int) + invalidCall := make(map[FunctionID]bool) + evaluatedCalls := make(map[ssa.CallInstruction]bool) + for function, facts := range localBodyFacts { + _ = function + if facts.AtomicPath == nil { + continue + } + for _, block := range facts.AtomicPath.Blocks { + for _, occurrence := range block.Calls { + evaluatedCalls[occurrence.Instruction] = true + } + } + } + for call, callPlan := range callPlans { + owner, owned := ids[call.Parent()] + if !owned || !evaluatedCalls[call] { + continue + } + exact := call != nil && call.Common() != nil && + callPlan.Kind == CallDirect && callPlan.Transport == ManagedTransport && + !callPlan.Open && !callPlan.MayBeNil && !callPlan.SyncDispatch && !callPlan.RawPlain && + len(callPlan.Targets) == 1 && + (callPlan.Rep == DirectCoro || callPlan.Rep == DirectPlain) + if !exact { + invalidCall[owner] = true + continue + } + outgoing[owner] = append(outgoing[owner], staticCall{target: callPlan.Targets[0]}) + sourceOutgoing[owner]++ + } + // Compiler-inserted calls participate in the same exact closure. A terminal + // ExplicitStatus helper is absent from the synchronous outcome body; every + // other helper participates in the same exact target closure as a source + // static call. Plain targets must be non-suspending and non-unwinding; + // structured targets must publish a bounded or unbounded synchronous outcome + // entry before their owner can enter the cohort. + for function, calls := range loweredCalls { + owner, owned := ids[function] + if !owned { + continue + } + for _, call := range calls { + if call.ExplicitStatusElided { + continue + } + targetID, identified := ids[call.Target] + target, found := base.Lookup(targetID) + if call.Target == nil || !identified || !found || call.RawPlain { + invalidCall[owner] = true + continue + } + switch target.ManagedEntry { + case ManagedEntryPlain: + if target.Effect.MaySuspend() || target.Exec.Contains(MayUnwind) { + invalidCall[owner] = true + continue + } + case ManagedEntryCoroutine, ManagedEntryOutcomePlain: + default: + invalidCall[owner] = true + continue + } + outgoing[owner] = append(outgoing[owner], staticCall{target: targetID}) + } + } + + eligible := make(map[FunctionID]bool) + for _, plan := range base.functions { + function := byID[plan.ID] + facts, classified := localBodyFacts[function] + if function == nil || !classified || !facts.StaticOutcomeLocal || facts.Effect.Contains(YieldOnly) || + len(function.FreeVars) != 0 || + invalidCall[plan.ID] || plan.HasStaticOutcome() || + plan.External != Defined || plan.Emission != EmitCoroutine || + plan.ManagedEntry != ManagedEntryCoroutine || plan.Primary != PrimaryCoroutine || + plan.ManagedDemand == NoDemand || plan.RawPlainOnly || + plan.Recursive || plan.Effect&^(YieldOnly|AwaitStructured|OutcomeStructured) != 0 || + !plan.Effect.Contains(OutcomeStructured) || + plan.Exec&(BlockForeign|ThreadAffine|NeedsCleanupFrame|OpaqueExec) != 0 { + continue + } + // Every evaluated ordinary call must appear in CallPlan or be one exact + // compiler-elided intrinsic. This prevents a builtin, raw C edge, or + // hidden helper from entering the synchronous twin by omission. + calls := 0 + for call := range evaluatedCalls { + if call.Parent() != function || elidedCalls[call] { + continue + } + if _, builtin := call.Common().Value.(*ssa.Builtin); builtin { + continue + } + calls++ + } + if calls != sourceOutgoing[plan.ID] { + continue + } + eligible[plan.ID] = true + } + + // Start from the local candidate set and monotonically remove every member + // whose exact coroutine callee cannot publish a synchronous outcome entry. + // This formulation deliberately removes recursive call-graph SCCs through + // FunctionPlan.Recursive above; future budgeted continuation can admit them + // only with an independent fairness proof. + for changed := true; changed; { + changed = false + for id := range eligible { + for _, call := range outgoing[id] { + target, found := base.Lookup(call.target) + plainUnsafe := target.ManagedEntry == ManagedEntryPlain && + (target.Effect.MaySuspend() || target.Exec.Contains(MayUnwind)) + coroutineUnavailable := (target.ManagedEntry == ManagedEntryCoroutine || + target.ManagedEntry == ManagedEntryOutcomePlain) && + !target.HasStaticOutcome() && !eligible[target.ID] + if !found || target.ManagedEntry == ManagedEntryNone || plainUnsafe || coroutineUnavailable { + delete(eligible, id) + changed = true + break + } + } + } + } + for index, plan := range base.functions { + if !eligible[plan.ID] { + continue + } + plan.StaticOutcome = true + facts := localBodyFacts[byID[plan.ID]] + if plan.LocalExec.Contains(NeedsCleanupFrame) && !facts.Exec.Contains(NeedsCleanupFrame) { + plan.LocalExec &^= NeedsCleanupFrame + plan.Exec &^= NeedsCleanupFrame + } + // NeedsCleanupFrame is local-only. If ProgramIR's evaluated projection + // retained no cleanup operation, a transitive caller cannot reintroduce + // the bit; clear the syntactic dead-defer seed on this exact function. + if !facts.Exec.Contains(NeedsCleanupFrame) { + plan.DeclaredExec &^= NeedsCleanupFrame + plan.LocalExec &^= NeedsCleanupFrame + plan.Exec &^= NeedsCleanupFrame + } + if err := validateManagedEntryPlan(plan); err != nil { + return fmt.Errorf("coro: select unbounded static outcome %q: %w", plan.ID, err) + } + base.functions[index] = plan + } + return nil +} + // validateSSACallableContractPlans prevents wrapper metadata from becoming a // second, contradictory truth beside the inferred Go body. A wrapper // contract may add conservative execution constraints, but it may never erase diff --git a/internal/coro/summary.go b/internal/coro/summary.go index 7a83b5b298..d4882feeaa 100644 --- a/internal/coro/summary.go +++ b/internal/coro/summary.go @@ -28,14 +28,14 @@ import ( ) // SummarySchema is the experimental wire schema for deterministic plan -// snapshots. Version v7 is intentionally not an archive ABI: producer +// snapshots. This schema is intentionally not an archive ABI: producer // artifacts use the separate LibraryEffectSummarySchema, and cache identity // uses PlanDigestSchema. -const SummarySchema = "llgo.coro.plan.v7" +const SummarySchema = "llgo.coro.plan.v9" // SummaryMetadata identifies ABI and target properties that affect an // experimental plan snapshot. Empty fields are permitted during early -// analysis. This v7 type must not be used as an archive compatibility record; +// analysis. This type must not be used as an archive compatibility record; // LibraryEffectMetadata owns that strict target/ABI contract. type SummaryMetadata struct { CoroABI string `json:"coro_abi"` @@ -61,6 +61,7 @@ type FunctionSummary struct { AtomicCost uint64 `json:"atomic_cost"` AtomicCostProof AtomicCostProof `json:"atomic_cost_proof"` AtomicCostCertificate string `json:"atomic_cost_certificate"` + StaticOutcome bool `json:"static_outcome"` FuncRep FuncRep `json:"func_rep"` External ExternalKind `json:"external"` Recursive bool `json:"recursive"` @@ -112,6 +113,7 @@ type functionSummaryWire struct { AtomicCost *uint64 `json:"atomic_cost"` AtomicCostProof *AtomicCostProof `json:"atomic_cost_proof"` AtomicCostCertificate *string `json:"atomic_cost_certificate"` + StaticOutcome *bool `json:"static_outcome"` FuncRep *FuncRep `json:"func_rep"` External *ExternalKind `json:"external"` Recursive *bool `json:"recursive"` @@ -149,6 +151,7 @@ func (p *Plan) Summary(metadata SummaryMetadata) Summary { AtomicCost: fn.AtomicCost, AtomicCostProof: fn.AtomicCostProof, AtomicCostCertificate: fn.AtomicCostCertificate, + StaticOutcome: fn.StaticOutcome, FuncRep: fn.FuncRep, External: fn.External, Recursive: fn.Recursive, @@ -313,6 +316,9 @@ func (w functionSummaryWire) summary(index int) (FunctionSummary, error) { if w.AtomicCostCertificate == nil { return missing("atomic_cost_certificate") } + if w.StaticOutcome == nil { + return missing("static_outcome") + } if w.FuncRep == nil { return missing("func_rep") } @@ -350,6 +356,7 @@ func (w functionSummaryWire) summary(index int) (FunctionSummary, error) { AtomicCost: *w.AtomicCost, AtomicCostProof: *w.AtomicCostProof, AtomicCostCertificate: *w.AtomicCostCertificate, + StaticOutcome: *w.StaticOutcome, FuncRep: *w.FuncRep, External: *w.External, Recursive: *w.Recursive, @@ -556,6 +563,7 @@ func (s Summary) canonical() (Summary, error) { Emission: fn.Emission, ManagedEntry: fn.ManagedEntry, AtomicCost: fn.AtomicCost, AtomicCostProof: fn.AtomicCostProof, AtomicCostCertificate: fn.AtomicCostCertificate, + StaticOutcome: fn.StaticOutcome, FuncRep: fn.FuncRep, External: fn.External, Recursive: fn.Recursive, Primary: fn.Primary, RawPlainOnly: fn.RawPlainOnly, RawPlainEntry: fn.RawPlainEntry, } diff --git a/internal/coro/target_capabilities.go b/internal/coro/target_capabilities.go index dbb6b73c1a..7d1a5d94f1 100644 --- a/internal/coro/target_capabilities.go +++ b/internal/coro/target_capabilities.go @@ -58,3 +58,32 @@ func (capabilities TargetCapabilities) Valid() bool { const known = targetCapabilityWorker | targetCapabilityNativeFleet | targetCapabilityHostOperation return capabilities&^known == 0 && (!capabilities.NativeFleet() || capabilities.Worker()) } + +// ProgramCapabilities is the closed-world demand projection of physical +// operations which need an optional target service. TargetCapabilities says +// what the selected environment can provide; ProgramCapabilities says what +// the final emitted program will actually use. Keeping the two lattices +// separate prevents a capable native target from eagerly starting services +// which have no reachable physical transaction. +type ProgramCapabilities uint8 + +const ( + programCapabilityWorker ProgramCapabilities = 1 << iota +) + +func NewProgramCapabilities(worker bool) ProgramCapabilities { + var capabilities ProgramCapabilities + if worker { + capabilities |= programCapabilityWorker + } + return capabilities +} + +func (capabilities ProgramCapabilities) Worker() bool { + return capabilities&programCapabilityWorker != 0 +} + +func (capabilities ProgramCapabilities) Valid() bool { + const known = programCapabilityWorker + return capabilities&^known == 0 +} diff --git a/internal/crosscompile/compile/rtlib/compiler_rt.go b/internal/crosscompile/compile/rtlib/compiler_rt.go index a334918215..ed2d40551f 100644 --- a/internal/crosscompile/compile/rtlib/compiler_rt.go +++ b/internal/crosscompile/compile/rtlib/compiler_rt.go @@ -114,7 +114,7 @@ func GetCompilerRTConfig() compile.LibConfig { } func GetCompilerRTCompileConfig(baseDir, target string) compile.CompileConfig { - return compile.CompileConfig{ + config := compile.CompileConfig{ Groups: []compile.CompileGroup{ { OutputFileName: fmt.Sprintf("libclang_builtins-%s.a", target), @@ -297,4 +297,13 @@ func GetCompilerRTCompileConfig(baseDir, target string) compile.CompileConfig { }, }, } + if strings.HasPrefix(target, "wasm32") || strings.HasPrefix(target, "wasm64") { + // Keep Clang's resource headers (compiler-rt uses them), but never let + // a freestanding WebAssembly build fall through to the host libc + // include directories. LLVM 22's WASI limits.h uses include_next when + // those directories are visible, which otherwise mixes Linux headers + // into the target archive. + config.Groups[0].CCFlags = append(config.Groups[0].CCFlags, "-nostdlibinc") + } + return config } diff --git a/internal/crosscompile/compile/rtlib/rt_test.go b/internal/crosscompile/compile/rtlib/rt_test.go index 5f5f254861..fb9da0a99f 100644 --- a/internal/crosscompile/compile/rtlib/rt_test.go +++ b/internal/crosscompile/compile/rtlib/rt_test.go @@ -1,6 +1,7 @@ package rtlib import ( + "slices" "strings" "testing" ) @@ -120,6 +121,19 @@ func TestGetCompilerRTConfig(t *testing.T) { } } +func TestCompilerRTWasmDoesNotUseHostLibcHeaders(t *testing.T) { + for _, target := range []string{"wasm32-unknown-wasi", "wasm32-unknown-unknown", "wasm64-unknown-unknown"} { + config := GetCompilerRTCompileConfig("/test/base", target) + if len(config.Groups) != 1 || !slices.Contains(config.Groups[0].CCFlags, "-nostdlibinc") { + t.Errorf("compiler-rt %s flags = %v, want -nostdlibinc", target, config.Groups[0].CCFlags) + } + } + native := GetCompilerRTCompileConfig("/test/base", "riscv32-unknown-elf") + if slices.Contains(native.Groups[0].CCFlags, "-nostdlibinc") { + t.Fatalf("native compiler-rt flags unexpectedly disable target libc headers: %v", native.Groups[0].CCFlags) + } +} + func TestGetCompilerRTConfig_DifferentTargets(t *testing.T) { targets := []string{ "riscv32-unknown-elf", diff --git a/internal/crosscompile/crosscompile.go b/internal/crosscompile/crosscompile.go index 274ef91b5d..3a21c29932 100644 --- a/internal/crosscompile/crosscompile.go +++ b/internal/crosscompile/crosscompile.go @@ -165,6 +165,52 @@ func getESPClangRoot(forceEspClang bool) (clangRoot string, err error) { return } +// getLLVM22Toolchain resolves the host LLVM tools used for WebAssembly. LLGo +// emits LLVM 22 IR, so handing that IR to the target-specific ESP LLVM 19 +// bundle is not a compatibility path: LLVM 22 attributes such as +// captures(none) and target_mem0 are rejected by its parser. The command entry +// installs the validated LLVM 22 bin directory in PATH; repeat that validation +// here so library callers receive the same fail-closed contract. +func getLLVM22Toolchain(linker string) (root, bin, cc, linkerPath string, err error) { + return resolveLLVM22Toolchain(linker, envllvm.SetupPath, exec.LookPath) +} + +func resolveLLVM22Toolchain( + linker string, + setup func() error, + lookup func(string) (string, error), +) (root, bin, cc, linkerPath string, err error) { + if err = setup(); err != nil { + err = fmt.Errorf("select LLVM 22 WebAssembly toolchain: %w", err) + return + } + if cc, err = lookup("clang++"); err != nil { + err = fmt.Errorf("select LLVM 22 WebAssembly compiler: %w", err) + return + } + bin = filepath.Dir(cc) + ar, arErr := lookup("llvm-ar") + if arErr != nil { + err = fmt.Errorf("select LLVM 22 archive tool: %w", arErr) + return + } + if filepath.Clean(filepath.Dir(ar)) != filepath.Clean(bin) { + err = fmt.Errorf( + "LLVM 22 clang++ and llvm-ar must share one bin directory: %q and %q", + cc, ar, + ) + return + } + root = filepath.Dir(bin) + if linker != "" { + if linkerPath, err = lookup(linker); err != nil { + err = fmt.Errorf("select LLVM 22 linker %q: %w", linker, err) + return + } + } + return +} + // getESPClangPlatform returns the platform suffix for ESP Clang downloads func getESPClangPlatform(goos, goarch string) string { switch goos { @@ -272,19 +318,30 @@ func use(goos, goarch string, wasiThreads, forceEspClang bool, level optlevel.Le } llgoRoot := env.LLGoROOT() - // Check for ESP Clang support for target-based builds - clangRoot, err := getESPClangRoot(forceEspClang) - if err != nil { - return - } - - // Set ClangRoot and CC if clang is available - export.ClangRoot = clangRoot - if clangRoot != "" { - export.CC = filepath.Join(clangRoot, "bin", "clang++") + // WebAssembly IR is emitted by LLVM 22 and must be consumed by that same + // baseline. ESP Clang remains a target backend for non-wasm embedded builds; + // it is never an IR compatibility fallback for wasm. + var clangRoot string + if goarch == "wasm" { + var clangBin string + clangRoot, clangBin, export.CC, _, err = getLLVM22Toolchain("") + if err != nil { + return + } + export.ClangBinPath = clangBin } else { - export.CC = "clang++" + clangRoot, err = getESPClangRoot(forceEspClang) + if err != nil { + return + } + if clangRoot != "" { + export.CC = filepath.Join(clangRoot, "bin", "clang++") + export.ClangBinPath = filepath.Join(clangRoot, "bin") + } else { + export.CC = "clang++" + } } + export.ClangRoot = clangRoot if runtime.GOOS == goos && runtime.GOARCH == goarch { export.DebugInfo = nativeDebugInfoPolicy(goos) @@ -560,15 +617,25 @@ func UseTarget(targetName string, level optlevel.Level, ltoMode lto.Mode) (expor return export, fmt.Errorf("target '%s' has incompatible libc/toolchain configuration: %w", targetName, err) } - // Check for ESP Clang support for target-based builds - clangRoot, err := getESPClangRoot(true) - if err != nil { - return + // The in-process frontend and every WebAssembly IR consumer share LLVM 22. + // Non-wasm embedded targets may still require the ESP backend for target + // instruction support, but it must never parse LLGo's LLVM 22 wasm IR. + var clangRoot string + if strings.HasPrefix(target, "wasm") { + clangRoot, export.ClangBinPath, export.CC, export.Linker, err = + getLLVM22Toolchain(config.Linker) + if err != nil { + return + } + } else { + clangRoot, err = getESPClangRoot(true) + if err != nil { + return + } + export.CC = filepath.Join(clangRoot, "bin", "clang++") + export.ClangBinPath = filepath.Join(clangRoot, "bin") } - - // Set ClangRoot and CC if clang is available export.ClangRoot = clangRoot - export.CC = filepath.Join(clangRoot, "bin", "clang++") // Convert target config to Export - only export necessary fields export.BuildTags = config.BuildTags @@ -609,12 +676,15 @@ func UseTarget(targetName string, level optlevel.Level, ltoMode lto.Mode) (expor envs := buildEnvMap(env.LLGoROOT()) // Convert LLVMTarget, CPU, Features to CCFLAGS/LDFLAGS. Some wasm-ld - // distributions expose the lld ICF switch while others (including the ESP - // LLVM 19 build) reject it. Keep the Go pc-identity policy explicit whenever - // the selected linker advertises the option; older wasm-ld defaults to no + // distributions expose the lld ICF switch while legacy target-specific + // distributions reject it. Keep the Go pc-identity policy explicit whenever + // the selected linker advertises the option; those wasm-ld builds default to no // ICF, so omitting the unsupported switch preserves the same semantics. ldflags := []string{"-S"} - targetLinker := filepath.Join(clangRoot, "bin", config.Linker) + targetLinker := export.Linker + if targetLinker == "" { + targetLinker = filepath.Join(clangRoot, "bin", config.Linker) + } if config.Linker != "wasm-ld" || linkerSupportsICF(targetLinker) { ldflags = append(ldflags, "--icf=none") } @@ -733,7 +803,7 @@ func UseTarget(targetName string, level optlevel.Level, ltoMode lto.Mode) (expor } // Handle Linker - keep it for external usage - if config.Linker != "" { + if config.Linker != "" && export.Linker == "" { export.Linker = filepath.Join(clangRoot, "bin", config.Linker) } if config.LinkerScript != "" { diff --git a/internal/crosscompile/crosscompile_test.go b/internal/crosscompile/crosscompile_test.go index 37e431283c..8771722817 100644 --- a/internal/crosscompile/crosscompile_test.go +++ b/internal/crosscompile/crosscompile_test.go @@ -49,7 +49,7 @@ func TestLinkerHelpSupportsICF(t *testing.T) { help string want bool }{ - {name: "esp-llvm19-no-icf", help: "--import-memory\n--no-entry\n--export=\n", want: false}, + {name: "legacy-wasm-ld-no-icf", help: "--import-memory\n--no-entry\n--export=\n", want: false}, {name: "lld-equals-form", help: "--icf={none,safe,all} Perform identical code folding\n", want: true}, {name: "lld-separated-form", help: "--icf Perform identical code folding\n", want: true}, } { @@ -61,6 +61,40 @@ func TestLinkerHelpSupportsICF(t *testing.T) { } } +func TestLLVM22WasmToolchainUsesConfiguredPath(t *testing.T) { + setupCalls := 0 + setup := func() error { + setupCalls++ + return nil + } + root := filepath.Join(t.TempDir(), "llvm-22") + bin := filepath.Join(root, "bin") + linkBin := filepath.Join(t.TempDir(), "lld-22", "bin") + tools := map[string]string{ + "clang++": filepath.Join(bin, "clang++"), + "llvm-ar": filepath.Join(bin, "llvm-ar"), + "wasm-ld": filepath.Join(linkBin, "wasm-ld"), + } + lookup := func(name string) (string, error) { + if path := tools[name]; path != "" { + return path, nil + } + return "", os.ErrNotExist + } + + gotRoot, gotBin, gotCC, gotLinker, err := resolveLLVM22Toolchain("wasm-ld", setup, lookup) + if err != nil { + t.Fatal(err) + } + if setupCalls != 1 || gotRoot != root || gotBin != bin || + gotCC != tools["clang++"] || gotLinker != tools["wasm-ld"] { + t.Fatalf( + "LLVM 22 wasm tools = setup:%d root:%q bin:%q cc:%q linker:%q", + setupCalls, gotRoot, gotBin, gotCC, gotLinker, + ) + } +} + func TestValidateWasmBuiltinsTargetCompatibility(t *testing.T) { for _, tt := range []struct { name string diff --git a/internal/llvmproof/executor_leaf.go b/internal/llvmproof/executor_leaf.go index 026e864366..fbab441f91 100644 --- a/internal/llvmproof/executor_leaf.go +++ b/internal/llvmproof/executor_leaf.go @@ -50,9 +50,9 @@ type ExecutorLeafProof struct { ClosureSHA256 string } -// LLVM C's opcode enum has kept these values stable since before the supported -// LLVM 19 floor. The Go binding currently omits symbolic Opcode constants for -// them even though it exposes their builders. +// LLVM C's opcode enum uses these values in the supported LLVM 22 release. The +// Go binding currently omits symbolic Opcode constants for them even though it +// exposes their builders. const ( llvmOpcodeFence llvm.Opcode = 55 llvmOpcodeAtomicCmpXchg llvm.Opcode = 56 @@ -155,15 +155,27 @@ func ProveExecutorLeaf(module llvm.Module, symbol string) (ExecutorLeafProof, er names = append(names, name) } sort.Strings(names) + closureSHA256 := executorLeafClosureSHA256(module, names, module.DataLayout()) + return ExecutorLeafProof{ + Symbol: symbol, + Signature: root.GlobalValueType().String(), + TargetTriple: module.Target(), + DataLayout: module.DataLayout(), + CallClosure: names, + ClosureSHA256: closureSHA256, + }, nil +} + +func executorLeafClosureSHA256(module llvm.Module, names []string, dataLayout string) string { var frozen strings.Builder fmt.Fprintf( &frozen, "%d:%s\n%d:%s\n", len(module.Target()), module.Target(), - len(module.DataLayout()), module.DataLayout(), + len(dataLayout), dataLayout, ) for _, name := range names { - function := closure[name] + function := module.NamedFunction(name) fmt.Fprintf( &frozen, "%d:%s\n%d:%s\n", @@ -186,14 +198,162 @@ func ProveExecutorLeaf(module llvm.Module, symbol string) (ExecutorLeafProof, er } } sum := sha256.Sum256([]byte(frozen.String())) - return ExecutorLeafProof{ - Symbol: symbol, - Signature: root.GlobalValueType().String(), - TargetTriple: module.Target(), - DataLayout: module.DataLayout(), - CallClosure: names, - ClosureSHA256: hex.EncodeToString(sum[:]), - }, nil + return hex.EncodeToString(sum[:]) +} + +// ProveExecutorLeafForDataLayout rebinds a structural executor-leaf proof to +// another target-data spelling only when every LLVM type actually reachable +// from that closure has identical ABI size, alignment, and aggregate offsets. +// This is intentionally closure-local: LLVM 22 wasm Clang may omit an i128 +// alignment entry used by LLGo's target machine, but a void debug-trap leaf is +// unaffected while a closure which mentions i128 remains rejected. +func ProveExecutorLeafForDataLayout( + module llvm.Module, + symbol string, + dataLayout string, +) (ExecutorLeafProof, error) { + proof, err := ProveExecutorLeaf(module, symbol) + if err != nil { + return ExecutorLeafProof{}, err + } + if dataLayout == proof.DataLayout { + return proof, nil + } + if dataLayout == "" { + return ExecutorLeafProof{}, fmt.Errorf( + "LLVM executor-leaf proof for %q: alternate data layout is empty", symbol, + ) + } + if err := executorLeafClosureDataLayoutsCompatible( + module, proof.CallClosure, dataLayout, + ); err != nil { + return ExecutorLeafProof{}, fmt.Errorf( + "LLVM executor-leaf proof for %q: alternate data layout: %w", symbol, err, + ) + } + proof.DataLayout = dataLayout + proof.ClosureSHA256 = executorLeafClosureSHA256( + module, proof.CallClosure, dataLayout, + ) + return proof, nil +} + +func executorLeafClosureDataLayoutsCompatible( + module llvm.Module, + closure []string, + dataLayout string, +) error { + if module.IsNil() || module.DataLayout() == "" || dataLayout == "" { + return fmt.Errorf("requires two non-empty target data layouts") + } + current := llvm.NewTargetData(module.DataLayout()) + alternate := llvm.NewTargetData(dataLayout) + defer current.Dispose() + defer alternate.Dispose() + if current.ByteOrder() != alternate.ByteOrder() || + current.PointerSize() != alternate.PointerSize() { + return fmt.Errorf("byte order or default pointer width differs") + } + + seen := make(map[llvm.Type]bool) + var compareType func(llvm.Type) error + compareSized := func(typ llvm.Type) error { + if current.TypeAllocSize(typ) != alternate.TypeAllocSize(typ) || + current.ABITypeAlignment(typ) != alternate.ABITypeAlignment(typ) { + return fmt.Errorf("type %q has different ABI layout", typ.String()) + } + return nil + } + compareType = func(typ llvm.Type) error { + if typ.IsNil() || seen[typ] { + return nil + } + seen[typ] = true + switch typ.TypeKind() { + case llvm.VoidTypeKind, llvm.LabelTypeKind, + llvm.MetadataTypeKind, llvm.TokenTypeKind: + return nil + case llvm.FunctionTypeKind: + if err := compareType(typ.ReturnType()); err != nil { + return err + } + for _, parameter := range typ.ParamTypes() { + if err := compareType(parameter); err != nil { + return err + } + } + return nil + case llvm.StructTypeKind: + elements := typ.StructElementTypes() + // A named zero-field body can be opaque through the C API. Reject it + // rather than asking TargetData to size a potentially unsized type. + if typ.StructName() != "" && len(elements) == 0 { + return fmt.Errorf("named zero-field type %q is not layout-provable", typ.String()) + } + for index, element := range elements { + if err := compareType(element); err != nil { + return err + } + if current.ElementOffset(typ, index) != alternate.ElementOffset(typ, index) { + return fmt.Errorf("type %q field %d has a different ABI offset", typ.String(), index) + } + } + return compareSized(typ) + case llvm.ArrayTypeKind, llvm.VectorTypeKind: + if err := compareType(typ.ElementType()); err != nil { + return err + } + return compareSized(typ) + case llvm.IntegerTypeKind, llvm.FloatTypeKind, llvm.DoubleTypeKind, + llvm.X86_FP80TypeKind, llvm.FP128TypeKind, llvm.PPC_FP128TypeKind, + llvm.PointerTypeKind: + return compareSized(typ) + default: + return fmt.Errorf("type %q has unsupported kind %d", typ.String(), typ.TypeKind()) + } + } + + for _, name := range closure { + function := module.NamedFunction(name) + if function.IsNil() { + return fmt.Errorf("closure function %q is absent", name) + } + if err := compareType(function.GlobalValueType()); err != nil { + return fmt.Errorf("function %q signature: %w", name, err) + } + for _, parameter := range function.Params() { + if err := compareType(parameter.Type()); err != nil { + return fmt.Errorf("function %q parameter: %w", name, err) + } + } + for block := function.FirstBasicBlock(); !block.IsNil(); block = llvm.NextBasicBlock(block) { + for instruction := block.FirstInstruction(); !instruction.IsNil(); instruction = llvm.NextInstruction(instruction) { + if err := compareType(instruction.Type()); err != nil { + return fmt.Errorf("function %q instruction result: %w", name, err) + } + for operand := 0; operand < instruction.OperandsCount(); operand++ { + if err := compareType(instruction.Operand(operand).Type()); err != nil { + return fmt.Errorf("function %q instruction operand: %w", name, err) + } + } + switch instruction.InstructionOpcode() { + case llvm.Alloca: + if err := compareType(instruction.AllocatedType()); err != nil { + return fmt.Errorf("function %q alloca: %w", name, err) + } + case llvm.GetElementPtr: + if err := compareType(instruction.GEPSourceElementType()); err != nil { + return fmt.Errorf("function %q GEP: %w", name, err) + } + case llvm.Call: + if err := compareType(instruction.CalledFunctionType()); err != nil { + return fmt.Errorf("function %q call: %w", name, err) + } + } + } + } + } + return nil } // proveNoPointerRetention rejects a definition that can publish a diff --git a/internal/llvmproof/executor_leaf_test.go b/internal/llvmproof/executor_leaf_test.go index cf2b2e8f2e..1167ec6589 100644 --- a/internal/llvmproof/executor_leaf_test.go +++ b/internal/llvmproof/executor_leaf_test.go @@ -83,6 +83,41 @@ trap: } } +func TestProveExecutorLeafForDataLayoutIsClosureLocal(t *testing.T) { + const ( + clangWasm = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20" + llgoWasm = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-i128:128-n32:64-S128-ni:1:10:20" + ) + terminal := parseExecutorLeafModule(t, ` +target datalayout = "`+clangWasm+`" +target triple = "wasm32-unknown-unknown" +declare void @llvm.debugtrap() +define void @leaf() { +entry: + call void @llvm.debugtrap() + ret void +}`) + proof, err := ProveExecutorLeafForDataLayout(terminal, "leaf", llgoWasm) + if err != nil { + t.Fatal(err) + } + if proof.DataLayout != llgoWasm || len(proof.ClosureSHA256) != 64 { + t.Fatalf("rebound terminal proof = %+v", proof) + } + + wide := parseExecutorLeafModule(t, ` +target datalayout = "`+clangWasm+`" +target triple = "wasm32-unknown-unknown" +define i128 @leaf(i128 %value) { +entry: + ret i128 %value +}`) + if _, err := ProveExecutorLeafForDataLayout(wide, "leaf", llgoWasm); err == nil || + !strings.Contains(err.Error(), "different ABI layout") { + t.Fatalf("i128 alternate-layout proof error = %v", err) + } +} + func TestProveExecutorLeafFailsClosed(t *testing.T) { tests := []struct { name string diff --git a/internal/lto/lto.go b/internal/lto/lto.go index 06967124d4..b0a1760ebe 100644 --- a/internal/lto/lto.go +++ b/internal/lto/lto.go @@ -65,7 +65,7 @@ func (p PassPlugin) LinkerFlags(goos string) ([]string, error) { return nil, nil } if goos == "darwin" { - return nil, fmt.Errorf("LTO pass plugins are not supported on darwin by LLVM 19 ld64.lld or Apple ld64") + return nil, fmt.Errorf("LTO pass plugins are not supported on darwin by LLVM 22 ld64.lld or Apple ld64") } return []string{"-Wl,--load-pass-plugin=" + p.Path}, nil } diff --git a/ltoplugin/CMakeLists.txt b/ltoplugin/CMakeLists.txt index 67e9c3ce1e..a025454c6f 100644 --- a/ltoplugin/CMakeLists.txt +++ b/ltoplugin/CMakeLists.txt @@ -1,11 +1,11 @@ cmake_minimum_required(VERSION 3.20) -project(LLGOLTOPlugin LANGUAGES CXX) +project(LLGOLTOPlugin LANGUAGES C CXX) find_package(LLVM REQUIRED CONFIG) -if(NOT LLVM_VERSION_MAJOR EQUAL 19) - message(FATAL_ERROR "LLGo LTO plugin requires LLVM 19.x, found ${LLVM_PACKAGE_VERSION}") +if(NOT LLVM_VERSION_MAJOR EQUAL 22) + message(FATAL_ERROR "LLGo LTO plugin requires LLVM 22.x, found ${LLVM_PACKAGE_VERSION}") endif() message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") diff --git a/ltoplugin/LLGOLTOPlugin.cpp b/ltoplugin/LLGOLTOPlugin.cpp index 340e995b82..006c6415ce 100644 --- a/ltoplugin/LLGOLTOPlugin.cpp +++ b/ltoplugin/LLGOLTOPlugin.cpp @@ -2,7 +2,7 @@ #include "llvm/Config/llvm-config.h" #include "llvm/Passes/PassBuilder.h" -#include "llvm/Passes/PassPlugin.h" +#include "llvm/Plugins/PassPlugin.h" #include "llvm/Support/Compiler.h" using namespace llvm; diff --git a/ltoplugin/README.md b/ltoplugin/README.md index 399a7511dc..c3ac863ae3 100644 --- a/ltoplugin/README.md +++ b/ltoplugin/README.md @@ -3,11 +3,11 @@ This directory contains the optional LLVM new pass manager plugin used by LLGo full LTO builds. It is not required to build or use LLGo. -Build with the same LLVM 19 toolchain used by LLGo: +Build with the same LLVM 22 toolchain used by LLGo: ```sh cmake -S ltoplugin -B ltoplugin/build \ - -DLLVM_DIR=/path/to/llvm-19/lib/cmake/llvm \ + -DLLVM_DIR=/path/to/llvm-22/lib/cmake/llvm \ -DCMAKE_BUILD_TYPE=Release cmake --build ltoplugin/build ``` @@ -24,7 +24,7 @@ The plugin registers `llgo-lto-pre-globaldce` and also inserts that pass through LLVM's full LTO early extension point, so loading the plugin is enough for the pass to run before the normal full LTO optimization pipeline proceeds. -LLGo forwards the plugin path through lld's `--load-pass-plugin` option. LLVM 19 +LLGo forwards the plugin path through lld's `--load-pass-plugin` option. LLVM 22 `ld64.lld` and Apple `ld64` do not expose an equivalent new pass manager LTO plugin loading option for Mach-O links, so LLGo rejects `-lto-pass-plugin` for Darwin targets until the linker side grows that support. diff --git a/runtime/coro_doorbell_source_test.go b/runtime/coro_doorbell_source_test.go index fb384332c8..e6424d63f4 100644 --- a/runtime/coro_doorbell_source_test.go +++ b/runtime/coro_doorbell_source_test.go @@ -43,13 +43,13 @@ func TestCoroDoorbellUsesExactBoundedForeignLeaves(t *testing.T) { coroDoorbellGoSource, "nativeCDoorbellOpen", "nativeCDoorbellRead", - "nativeCDoorbellWrite", "nativeCDoorbellClose", ) for _, required := range []string{ `LLGoFiles = "_wrap/doorbell.c"`, "//go:linkname nativeCDoorbellOpen C.__llgo_coro_doorbell_open_v1", "//go:linkname nativeCDoorbellRead C.__llgo_coro_doorbell_read_v1", + "//llgo:coro contract foreign.v1 scope=declaration progress=executor-safe affinity=any-thread reentry=none memory=borrow-until-return\n//go:linkname nativeCDoorbellWrite C.__llgo_coro_doorbell_write_v1", "//go:linkname nativeCDoorbellWrite C.__llgo_coro_doorbell_write_v1", "//go:linkname nativeCDoorbellClose C.__llgo_coro_doorbell_close_v1", "unpackNativeDoorbellResult", diff --git a/runtime/coro_program_capability_source_test.go b/runtime/coro_program_capability_source_test.go new file mode 100644 index 0000000000..6d754e60e2 --- /dev/null +++ b/runtime/coro_program_capability_source_test.go @@ -0,0 +1,126 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +import ( + "strings" + "testing" +) + +// TestRuntimeCoroProgramWorkerCapabilityIsEndToEndOptional is an architecture +// gate, not a spelling inventory. It keeps the compiler-emitted closed-program +// fact authoritative from bootstrap validation through native source binding +// and physical worker startup. A partial implementation at any one of those +// layers would retain the worker subsystem or create four idle pthreads for a +// program which has no physical worker operation. +func TestRuntimeCoroProgramWorkerCapabilityIsEndToEndOptional(t *testing.T) { + bootstrapPath := "internal/coro/bootstrap.go" + bootstrap := readRuntimePollFile(t, bootstrapPath) + for _, required := range []string{ + "ProgramCapabilityWorkerV2 ProgramCapabilitiesV2 = 1 << iota", + "const known = ProgramCapabilityWorkerV2", + "lo, hi = mixProgramDigestV2(lo, hi, uint64(bootstrap.Flags))", + "func ResolveProgramCapabilitiesV2(", + "capabilities := ProgramCapabilitiesV2(current.bootstrap.Flags)", + } { + if !strings.Contains(bootstrap, required) { + t.Errorf("%s lacks program capability contract %q", bootstrapPath, required) + } + } + + programPath := "internal/runtime/coro_program.go" + program := readRuntimePollFile(t, programPath) + resolve := strings.Index(program, "capabilities, capabilityCode := coro.ResolveProgramCapabilitiesV2(program)") + publish := strings.Index(program, "coroProgramCapabilitiesV2State = capabilities") + bind := strings.Index(program, "!coroProgramBindExecutorV1()") + if resolve < 0 || publish <= resolve || bind <= publish { + t.Errorf("%s does not resolve and publish capabilities before source binding", programPath) + } + if !strings.Contains(program, "return coroProgramCapabilitiesV2State.Valid() && coroProgramCapabilitiesV2State.Worker()") { + t.Errorf("%s lacks the validated worker capability query", programPath) + } + + for _, path := range []string{ + "internal/runtime/coro_executor_driver_timer_llgo.go", + "internal/runtime/coro_executor_driver_worker_llgo.go", + } { + driver := readRuntimePollFile(t, path) + normalized := strings.Join(strings.Fields(driver), " ") + for _, required := range []string{ + "var worker *coro.WorkerOperationSource", + "coroProgramWorkerCapabilityV2()", + "worker = &coroProgramWorkerSourceV1State", + "Worker: worker", + } { + if !strings.Contains(normalized, required) { + t.Errorf("%s lacks optional worker binding marker %q", path, required) + } + } + if strings.Contains(normalized, "Worker: &coroProgramWorkerSourceV1State") { + t.Errorf("%s restored unconditional program worker binding", path) + } + } + + fleetProgramPath := "internal/runtime/coro_native_fleet_program_llgo.go" + fleetProgram := strings.Join(strings.Fields(readRuntimePollFile(t, fleetProgramPath)), " ") + for _, required := range []string{ + "if coroProgramWorkerCapabilityV2() { worker = &coroProgramWorkerSourceV1State }", + "Worker: worker", + } { + if !strings.Contains(fleetProgram, required) { + t.Errorf("%s lacks optional adopted-worker marker %q", fleetProgramPath, required) + } + } + + fleetCorePath := "internal/runtime/coro_native_fleet.go" + fleetCore := strings.Join(strings.Fields(readRuntimePollFile(t, fleetCorePath)), " ") + for _, required := range []string{ + "func coroNativeFleetBindDomainV1(state *coroNativeFleetStateV1, index uint32, workerEnabled bool) bool", + "if workerEnabled { worker = &domain.worker }", + "workerEnabled := program == nil || program.sources.Worker != nil", + "coroNativeFleetBindDomainV1(state, index, workerEnabled)", + } { + if !strings.Contains(fleetCore, required) { + t.Errorf("%s lacks fleet worker optionality marker %q", fleetCorePath, required) + } + } + + workerPath := "internal/runtime/coro_worker_native_llgo.go" + worker := strings.Join(strings.Fields(readRuntimePollFile(t, workerPath)), " ") + for _, required := range []string{ + "func coroNativeWorkerPoolStartFleetV1() bool", + "if !coroProgramWorkerCapabilityV2() { return coroNativeWorkerPoolCanReleaseV1() }", + } { + if !strings.Contains(worker, required) { + t.Errorf("%s lacks no-worker physical-start marker %q", workerPath, required) + } + } + + // TLS setup happens before the scheduler can service a worker operation. + // Its terminal caller owns diagnostics, so this leaf must remain free of + // stdio which effect analysis would correctly classify as may-block. + pthreadPath := "internal/runtime/g_pthread.go" + pthread := readRuntimePollFile(t, pthreadPath) + if strings.Contains(pthread, "c.Fprintf(") { + t.Errorf("%s restored pre-scheduler stdio and false worker demand", pthreadPath) + } + if !strings.Contains(pthread, "The caller owns the terminal diagnostic") { + t.Errorf("%s lost the audited terminal-diagnostic ownership", pthreadPath) + } +} diff --git a/runtime/coro_runnable_distribution_source_test.go b/runtime/coro_runnable_distribution_source_test.go index 28d79dedd5..3a93df6fc0 100644 --- a/runtime/coro_runnable_distribution_source_test.go +++ b/runtime/coro_runnable_distribution_source_test.go @@ -19,6 +19,9 @@ package runtime import ( + "go/ast" + "go/parser" + "go/token" "strings" "testing" ) @@ -67,6 +70,39 @@ func TestCoroRunnableDistributionUsesDemandAndExactMailbox(t *testing.T) { } } +func TestCoroDirectChannelPublisherDoesNotReadOwnerCursor(t *testing.T) { + const name = "PublishExecutorDirectChannelCompletion" + source := readRuntimePollFile(t, "internal/coro/direct_channel_completion.go") + file, err := parser.ParseFile(token.NewFileSet(), "direct_channel_completion.go", source, 0) + if err != nil { + t.Fatal(err) + } + var publisher *ast.FuncDecl + for _, declaration := range file.Decls { + function, ok := declaration.(*ast.FuncDecl) + if ok && function.Name.Name == name { + publisher = function + break + } + } + if publisher == nil { + t.Fatalf("runtime core lacks %s", name) + } + ast.Inspect(publisher.Body, func(node ast.Node) bool { + selector, ok := node.(*ast.SelectorExpr) + if !ok || selector.Sel.Name != "directChannelTail" { + return true + } + if receiver, ok := selector.X.(*ast.Ident); ok && receiver.Name == "driver" { + t.Error("direct-channel producer reads the owner-only inbox cursor") + } + return true + }) + if !strings.Contains(source, "preemptLoadPointer(&driver.directChannelHead)") { + t.Error("direct-channel producer lacks its atomic publication cursor gate") + } +} + func TestCoroRunnableDistributionHasOneGenericTargetPath(t *testing.T) { paths := []string{ "internal/runtime/coro_native_fleet.go", @@ -123,6 +159,12 @@ func TestCoroNativeFleetUsesFixedTopologyLogicalQuotaAndScalarPeerABI(t *testing owner := readRuntimePollFile(t, "internal/runtime/coro_native_fleet_owner_llgo.go") for _, required := range []string{ "[coroNativeFleetDomainCapacityV1 - 1]coroNativeFleetPhysicalOwnerV1", + "policyEpoch uint32", + "func coroNativeFleetPhysicalOwnerDesiredPeersV1(limit, capacity uint32)", + "func coroNativeFleetPhysicalOwnersEnsureLockedV1(", + "state.started < desired", + "func coroNativeFleetSetExecutionLimitV1(limit uint32)", + "coroNativeAtomicStoreV1(&state.policyEpoch, epoch+1)", "coroNativeMStartPhysicalOwnerV1(owner, slot)", "func __llgo_coro_native_fleet_owner_v2(slot uint32) uint32", "coroNativeMRunReplacementOwnerV1(slot)", @@ -182,6 +224,7 @@ func TestCoroNativeFleetUsesFixedTopologyLogicalQuotaAndScalarPeerABI(t *testing "coroNativeFleetV1State.execution.TryAcquire(route)", "coroNativeFleetV1State.execution.Release(route)", "func CoroGOMAXPROCS(n int) int", + "coroNativeFleetSetExecutionLimitV1(next)", "coroNativeFleetRingExecutionWaitersV1(waiters uint32)", "coroNativeFleetV1State.execution.WaiterMask()", } { @@ -190,12 +233,50 @@ func TestCoroNativeFleetUsesFixedTopologyLogicalQuotaAndScalarPeerABI(t *testing } } + distribution := readRuntimePollFile(t, "internal/runtime/coro_ready_distribution_fleet_llgo.go") + for _, required := range []string{ + "func coroTargetBeginRunSliceV1(", + "target.policyEpoch = epoch", + "func coroTargetRefreshRunSliceV1(target coroRunTargetCapabilityV1)", + "epoch != target.policyEpoch", + } { + if !strings.Contains(distribution, required) { + t.Errorf("native fleet distribution lacks stable-slice policy marker %q", required) + } + } + + runSlice := readRuntimePollFile(t, "internal/runtime/coro_run_slice.go") + for _, required := range []string{ + "if next.Kind == coro.ActionYield {", + "coroTargetRefreshRunSliceV1(target)", + "coroTargetReadyDistributionV1(target)", + } { + if !strings.Contains(runSlice, required) { + t.Errorf("runtime run slice lacks yield-synchronized policy marker %q", required) + } + } + if strings.Count(runSlice, "coroTargetRefreshRunSliceV1(target)") != 1 { + t.Error("runtime run slice must refresh mutable placement policy only at the explicit yield boundary") + } + + gomaxprocs := readRuntimePollFile(t, "internal/lib/runtime/gomaxprocs_coro_llgo.go") + for _, required := range []string{ + "previous := llruntime.CoroGOMAXPROCS(n)", + "if n > 0 && previous != n {", + "coroSchedulerYield()", + } { + if !strings.Contains(gomaxprocs, required) { + t.Errorf("public coroutine GOMAXPROCS wrapper lacks policy handoff marker %q", required) + } + } + target := readRuntimePollFile(t, "internal/runtime/coro_target_native_fleet_llgo.go") for _, required := range []string{ "coroNativeInitialExecutionLimitV1()", "coroTargetStartPhysicalThreadCapacityV1()", "coroNativeFleetStartProgramV1(coroNativeFleetDomainCapacityV1)", "coroNativeFleetV1State.execution.Start(limit)", + "coroNativeFleetPhysicalOwnersStartV1(limit)", "coroNativeFleetV1State.execution.Seal()", "coroNativeFleetV1State.execution.Retire()", "coroNativeMStartCleanFactoryV1()", diff --git a/runtime/coro_scheduler_capability_source_test.go b/runtime/coro_scheduler_capability_source_test.go index b89f7af3e8..cddb7de906 100644 --- a/runtime/coro_scheduler_capability_source_test.go +++ b/runtime/coro_scheduler_capability_source_test.go @@ -57,3 +57,60 @@ func TestCoroLLVMHandleControlIsSchedulerOwnerRawHostStackOnly(t *testing.T) { } } } + +func TestCoroBoundedRunSliceAuditsImmutableDriverOncePerHostEntry(t *testing.T) { + read := func(path string) string { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(raw) + } + corePath := "internal/coro/run_slice.go" + core := read(corePath) + for _, marker := range []string{ + "func nextExecutorRunStepAtValidated(", + "type ExecutorRunSliceCapability struct", + "func BeginExecutorRunSlice(", + "func (capability *ExecutorRunSliceCapability) owner(", + "func (capability *ExecutorRunSliceCapability) Next(", + "func (capability *ExecutorRunSliceCapability) NextBeforeTime(", + "func (capability *ExecutorRunSliceCapability) NextAt(", + "return nextExecutorRunStepAtValidated(driver, now, withDeadline, false)", + "func (capability *ExecutorRunSliceCapability) NextActionCombined()", + "func (capability *ExecutorRunSliceCapability) NextAtCombined(now int64)", + } { + if !strings.Contains(core, marker) { + t.Errorf("%s lacks bounded run-slice capability marker %q", corePath, marker) + } + } + + programPath := "internal/runtime/coro_sched.go" + program := read(programPath) + for _, marker := range []string{ + "run, runOK := coro.BeginExecutorRunSlice(driver)", + "run.NextActionCombined()", + "coroProgramNextRunStepV1(driver, &run, combineDispatch)", + } { + if !strings.Contains(program, marker) { + t.Errorf("%s lacks bounded program-run marker %q", programPath, marker) + } + } + + fleetPath := "internal/runtime/coro_run_slice.go" + fleet := read(fleetPath) + for _, marker := range []string{ + "run, runOK := coro.BeginExecutorRunSlice(driver)", + "run.NextActionCombined()", + "step, nextOK = run.NextAtCombined(now)", + "step, nextOK = run.NextAt(now)", + } { + if !strings.Contains(fleet, marker) { + t.Errorf("%s lacks bounded fleet-run marker %q", fleetPath, marker) + } + } + if strings.Contains(fleet, "coro.NextExecutorRunStepAt(driver, now)") { + t.Errorf("%s repeats the full immutable driver audit inside its bounded loop", fleetPath) + } +} diff --git a/runtime/coro_task_allocation_source_test.go b/runtime/coro_task_allocation_source_test.go index d0bd6a8efd..221a72898d 100644 --- a/runtime/coro_task_allocation_source_test.go +++ b/runtime/coro_task_allocation_source_test.go @@ -52,6 +52,12 @@ func TestCoroSpawnFusesTaskAndRuntimeContextAllocation(t *testing.T) { strings.Contains(spawn, "coroBindRuntimeContext(child, parent, false)") { t.Fatal("spawn path retained a separately allocated runtime sidecar") } + allocation := strings.Index(spawn, "raw := coroalloc.AllocTask(allocationSize)") + initialize := strings.Index(spawn, "child, _, actualSize, allocationOK := coroTaskAllocationAt(raw)") + if allocation < 0 || initialize < 0 || allocation >= initialize || + strings.Contains(spawn[allocation:initialize], "coro.Zero(") { + t.Fatal("spawn path repeats the allocator's zero-filled storage contract") + } context := readRuntimePollFile(t, "internal/runtime/coro_task_context.go") for _, required := range []string{ @@ -66,6 +72,59 @@ func TestCoroSpawnFusesTaskAndRuntimeContextAllocation(t *testing.T) { } } +func TestCoroAllocatorOwnsZeroFillAndRetirementSanitization(t *testing.T) { + allocator := readRuntimePollFile(t, "internal/coroalloc/allocator.go") + for _, marker := range []string{ + "const backendAllocationsAreZeroed = true", + "allocates one zero-filled, explicitly owned", + "if !backendAllocationsAreZeroed || !Ready()", + } { + if marker == "const backendAllocationsAreZeroed = true" { + continue + } + if !strings.Contains(allocator, marker) { + t.Errorf("coroutine allocator lacks zero-fill contract marker %q", marker) + } + } + for _, path := range []string{ + "internal/coroalloc/backend_gc.go", + "internal/coroalloc/backend_nogc.go", + "internal/coroalloc/backend_webassembly.go", + "internal/coroalloc/backend_tinygogc.go", + } { + source := readRuntimePollFile(t, path) + if !strings.Contains(source, "const backendAllocationsAreZeroed = true") { + t.Errorf("%s lacks zero-filled backend contract", path) + } + } + for _, path := range []string{ + "internal/coroalloc/backend_nogc.go", + "internal/coroalloc/backend_webassembly.go", + } { + if source := readRuntimePollFile(t, path); !strings.Contains(source, "return c.Calloc(1, size)") || + strings.Contains(source, "return c.Malloc(size)") { + t.Errorf("%s does not implement the zero-filled libc contract", path) + } + } + tiny := readRuntimePollFile(t, "internal/runtime/tinygogc/rooted.go") + if !strings.Contains(tiny, "c.Memset(ptr, 0, size)") || + strings.Index(tiny, "c.Memset(ptr, 0, size)") > strings.Index(tiny, "unlinkRootedAllocation(root)") { + t.Fatal("tinygogc does not sanitize a rooted allocation before unlink") + } + frameCore := readRuntimePollFile(t, "internal/coro/frame.go") + register := strings.Index(frameCore, "func RegisterFrame(") + publish := strings.Index(frameCore, "func FrameFromStorage(") + if register < 0 || publish < 0 || register >= publish || + strings.Contains(frameCore[register:publish], "Zero(raw, total)") { + t.Fatal("frame registration repeats the allocator's zero-fill contract") + } + frameRuntime := readRuntimePollFile(t, "internal/runtime/coro_frame.go") + free := strings.Index(frameRuntime, "func __llgo_coro_frame_free_v1(") + if free < 0 || strings.Contains(frameRuntime[free:], "coro.Zero(raw, total)") { + t.Fatal("frame release sanitizes outside the selected allocator backend") + } +} + func TestCoroLogicalContextBorrowsPhysicalMPOnlyWhileRunning(t *testing.T) { contextLayout := readRuntimePollFile(t, "internal/runtime/runtime_context.go") for _, required := range []string{ diff --git a/runtime/internal/coro/bootstrap.go b/runtime/internal/coro/bootstrap.go index 8111c39ddb..2d7712a7a7 100644 --- a/runtime/internal/coro/bootstrap.go +++ b/runtime/internal/coro/bootstrap.go @@ -32,6 +32,24 @@ const ( RootFactoryVersionV1 uint32 = 1 ) +// ProgramCapabilitiesV2 is the compiler-proven set of optional physical +// services used by the final program. It is carried in ProgramBootstrapV2.Flags +// and is deliberately independent of the target's larger capability set. +type ProgramCapabilitiesV2 uint32 + +const ( + ProgramCapabilityWorkerV2 ProgramCapabilitiesV2 = 1 << iota +) + +func (capabilities ProgramCapabilitiesV2) Valid() bool { + const known = ProgramCapabilityWorkerV2 + return capabilities&^known == 0 +} + +func (capabilities ProgramCapabilitiesV2) Worker() bool { + return capabilities&ProgramCapabilityWorkerV2 != 0 +} + // ProgramStepKindV1 identifies how one compiler-emitted bootstrap step must be // entered. It is data only: this package never invokes either pointer kind. type ProgramStepKindV1 uint32 @@ -545,6 +563,7 @@ func programStepsDigestV2( ) (uint64, uint64, ProgramValidationCodeV2) { lo := uint64(14695981039346656037) hi := uint64(0x6eed0e9da4d94a4f) + lo, hi = mixProgramDigestV2(lo, hi, uint64(bootstrap.Flags)) lo, hi = mixProgramDigestV2(lo, hi, uint64(bootstrap.StepCount)) lo, hi = mixProgramDigestV2(lo, hi, bootstrap.HashLo) lo, hi = mixProgramDigestV2(lo, hi, bootstrap.HashHi) @@ -622,7 +641,7 @@ func ValidateRunnableProgramV2( if bootstrap.Version != ProgramBootstrapVersionV2 { return ProgramViewV2{}, ProgramValidationBootstrapVersionV2 } - if bootstrap.Flags != 0 { + if !ProgramCapabilitiesV2(bootstrap.Flags).Valid() { return ProgramViewV2{}, ProgramValidationBootstrapFlagsV2 } if bootstrap.HashLo != manifest.HashLo || bootstrap.HashHi != manifest.HashHi { @@ -668,6 +687,28 @@ func ValidateRunnableProgramV2( }, ProgramValidationOKV2 } +// ResolveProgramCapabilitiesV2 returns the optional-service demand from an +// opaque validated view. Like ResolveProgramStepV2 it revalidates mutable test +// storage and compares the complete digest before publishing any fact. +func ResolveProgramCapabilitiesV2( + program ProgramViewV2, +) (ProgramCapabilitiesV2, ProgramValidationCodeV2) { + if program.magic != validatedProgramMagicV2 { + return 0, ProgramValidationInvalidViewV2 + } + current, code := ValidateRunnableProgramV2(program.manifest, program.factory) + if code != ProgramValidationOKV2 || current.bootstrap != program.bootstrap || + current.steps != program.steps || current.stepCount != program.stepCount || + current.digestLo != program.digestLo || current.digestHi != program.digestHi { + return 0, ProgramValidationInvalidViewV2 + } + capabilities := ProgramCapabilitiesV2(current.bootstrap.Flags) + if !capabilities.Valid() { + return 0, ProgramValidationBootstrapFlagsV2 + } + return capabilities, ProgramValidationOKV2 +} + // ResolveProgramStepV2 returns one action from an opaque validated view. It // never calls the plain target or coroutine factory. func ResolveProgramStepV2(program ProgramViewV2, index uintptr) (ResolvedProgramStepV2, ProgramValidationCodeV2) { diff --git a/runtime/internal/coro/bootstrap_test.go b/runtime/internal/coro/bootstrap_test.go index 4a383c5ed7..25d437ee9f 100644 --- a/runtime/internal/coro/bootstrap_test.go +++ b/runtime/internal/coro/bootstrap_test.go @@ -239,6 +239,26 @@ func TestValidateAndResolveEveryHeterogeneousProgramV2(t *testing.T) { } } +func TestProgramCapabilitiesV2AreValidatedAndDigestBound(t *testing.T) { + f := newProgramBootstrapTestFixtureV2(0b01010) + f.bootstrap.Flags = uint32(ProgramCapabilityWorkerV2) + view := requireProgramViewV2(t, f) + capabilities, code := ResolveProgramCapabilitiesV2(view) + if code != ProgramValidationOKV2 || !capabilities.Worker() { + t.Fatalf("resolved program capabilities = (%#x, %d), want worker", capabilities, code) + } + f.bootstrap.Flags = 0 + if _, code := ResolveProgramCapabilitiesV2(view); code != ProgramValidationInvalidViewV2 { + t.Fatalf("mutated program capabilities code = %d, want invalid view", code) + } + + unknown := newProgramBootstrapTestFixtureV2(0b01010) + unknown.bootstrap.Flags = uint32(ProgramCapabilityWorkerV2) << 1 + if _, code := ValidateRunnableProgramV2(&unknown.manifest, unsafe.Pointer(&unknown.bootstrapFactory)); code != ProgramValidationBootstrapFlagsV2 { + t.Fatalf("unknown program capability code = %d, want bootstrap flags", code) + } +} + func TestValidateRunnableProgramV2RejectsMalformedTable(t *testing.T) { tests := []struct { name string @@ -250,7 +270,7 @@ func TestValidateRunnableProgramV2RejectsMalformedTable(t *testing.T) { {"nil bootstrap", ProgramValidationNilBootstrapV2, func(f *programBootstrapTestFixtureV2) { f.manifest.Bootstrap = nil }}, {"bootstrap v1", ProgramValidationBootstrapVersionV2, func(f *programBootstrapTestFixtureV2) { f.bootstrap.Version = uint32(1) }}, {"unknown bootstrap version", ProgramValidationBootstrapVersionV2, func(f *programBootstrapTestFixtureV2) { f.bootstrap.Version = 3 }}, - {"bootstrap flags", ProgramValidationBootstrapFlagsV2, func(f *programBootstrapTestFixtureV2) { f.bootstrap.Flags = 1 }}, + {"bootstrap flags", ProgramValidationBootstrapFlagsV2, func(f *programBootstrapTestFixtureV2) { f.bootstrap.Flags = 2 }}, {"bootstrap hash", ProgramValidationBootstrapHashV2, func(f *programBootstrapTestFixtureV2) { f.bootstrap.HashHi++ }}, {"four steps", ProgramValidationStepCountV2, func(f *programBootstrapTestFixtureV2) { f.bootstrap.StepCount = 4 }}, {"nil steps", ProgramValidationStepCountPointerV2, func(f *programBootstrapTestFixtureV2) { f.bootstrap.Steps = nil }}, diff --git a/runtime/internal/coro/channel_claim_core_test.go b/runtime/internal/coro/channel_claim_core_test.go index d4e0af6460..ff8aece0e3 100644 --- a/runtime/internal/coro/channel_claim_core_test.go +++ b/runtime/internal/coro/channel_claim_core_test.go @@ -453,6 +453,18 @@ func externallyCommitChannelCandidateAtRoute( fixture *channelClaimCoreFixture, index int, route RouteID, +) { + externallyCommitChannelCandidateAtRouteWithResult( + t, fixture, index, route, ResumeSmallInvalid, + ) +} + +func externallyCommitChannelCandidateAtRouteWithResult( + t *testing.T, + fixture *channelClaimCoreFixture, + index int, + route RouteID, + small uint8, ) { t.Helper() admission, acquired := fixture.source.acquireExternalCommit(fixture.ids[index]) @@ -466,7 +478,7 @@ func externallyCommitChannelCandidateAtRoute( if !beginExternalSelectClaimEffect(fixture.claim) { t.Fatal("begin externally committed channel effect") } - if result := admission.publishExternallyCommittedAtRoute(route); result != ChannelOperationPosted { + if result := admission.publishExternallyCommittedAtRoute(route, small); result != ChannelOperationPosted { t.Fatalf("publish externally committed channel candidate = %d", result) } if !publishExternalSelectClaim(fixture.claim) { @@ -516,8 +528,10 @@ func TestOwnerLocalChannelCompletionSkipsExternalSourceEpoch(t *testing.T) { yieldRunningDriverTask(t, fixture.p, producer, producerAction) var complete ExecutorPollProgress + reductions := 0 for reduction := 0; reduction < 64; reduction++ { step, advanced := NextExecutorRunStep(fixture.driver) + reductions++ if !advanced || step.Kind != ExecutorRunStepSource || step.Poll.Used != 1 || !step.Poll.AtomicResolve || fixture.driver.poll != (executorPollTransaction{}) { t.Fatalf("owner-local reduction %d = (%+v,%t), poll=%+v", reduction, step, advanced, fixture.driver.poll) @@ -537,6 +551,9 @@ func TestOwnerLocalChannelCompletionSkipsExternalSourceEpoch(t *testing.T) { complete, fixture.driver.local, fixture.task.g.park, fixture.source.Pending(), fixture.registry.ObserveRequested(fixture.handle)) } + if reductions > 4 { + t.Fatalf("owner-local direct completion used %d scheduler reductions, want at most 4", reductions) + } if !EnterExecutorRunCompatibility(fixture.driver) { t.Fatal("leave owner-local bounded runner") @@ -553,6 +570,812 @@ func TestOwnerLocalChannelCompletionSkipsExternalSourceEpoch(t *testing.T) { runtime.KeepAlive(producer.frame.memory) } +func TestOwnerLocalDirectChannelCleanupFinishesAfterMaterialize(t *testing.T) { + var ( + packet ResumePacket + plan ResumeCleanupPlan + token byte + ) + fixture := newChannelClaimCoreFixtureBeforeResume( + t, + "channel-owner-local-direct-cleanup", + []uint32{1}, + true, + 0, + func(fixture *channelClaimCoreFixture) { + if !BindWaitSetResumeCleanup( + &fixture.wait, + &packet, + &plan, + ResumeCleanupBinding{ + Kind: ResumeCleanupChannelDirect, + Context: unsafe.Pointer(&token), + Entries: unsafe.Pointer(&fixture.ids[0]), + Claim: fixture.claim, + Count: 1, + RuntimeCount: 1, + Stride: unsafe.Sizeof(OperationID{}), + }, + ) { + t.Fatal("bind owner-local direct cleanup") + } + }, + ) + + producer := newYieldingTestG(t, "channel-owner-local-direct-producer") + if !Enqueue(fixture.p, producer.g) { + t.Fatal("enqueue owner-local direct producer") + } + if g, ok := NextRunnable(fixture.p); !ok || g != producer.g { + t.Fatalf("dequeue owner-local direct producer = (%p,%t)", g, ok) + } + producerAction := beginWaitTestResume(t, fixture.p, producer) + externallyCommitChannelCandidateAtRoute(t, fixture, 0, RouteID(1)) + local, localOK := TryPublishOwnerLocalChannelCompletion( + producer.g, + fixture.source, + fixture.ids[0], + ) + if !localOK || !local { + t.Fatalf("publish owner-local direct completion = (%t,%t)", local, localOK) + } + yieldRunningDriverTask(t, fixture.p, producer, producerAction) + + begin, beginOK := NextExecutorRunStep(fixture.driver) + if !beginOK || begin.Kind != ExecutorRunStepSource || begin.Poll.Complete || + !begin.Poll.AtomicResolve || begin.Poll.Used != 1 || begin.Poll.ApplyVisits != 1 || + plan.phase != resumeCleanupRuntime || fixture.task.g.park.phase != parkConsumed || + !fixture.driver.local.resolve.directChannel { + t.Fatalf("owner-local direct begin = (%+v,%t), plan=%+v park=%+v", + begin, beginOK, plan, fixture.task.g.park) + } + assertDirectSelector := func(want bool) { + t.Helper() + _, _, _, selectorOK := ownerLocalDirectChannelCleanupHeader(&fixture.driver.local.resolve) + _, pending := pendingResumeCleanupStepForCursor(&fixture.driver.local.resolve) + headerOK := validOwnerLocalCompletionHeader(&fixture.driver.local, fixture.p) + if selectorOK != want || pending != want || headerOK != want { + t.Fatalf("direct runtime selector = certificate:%t pending:%t header:%t, want %t", + selectorOK, pending, headerOK, want) + } + } + assertDirectState := func(want bool) { + t.Helper() + _, _, _, stateOK := ownerLocalDirectChannelCleanupState(&fixture.driver.local.resolve) + if stateOK != want { + t.Fatalf("direct runtime effect audit = %t, want %t", stateOK, want) + } + } + assertDirectSelector(true) + assertDirectState(true) + // Cursor and typed context fields guard selection itself. Corruption in an + // immutable packet or ParkState is deliberately deferred to the one complete + // audit at the effectful retirement boundary. + savedNext := fixture.driver.local.resolve.nextWait + fixture.driver.local.resolve.nextWait = &fixture.wait + assertDirectSelector(false) + assertDirectState(false) + fixture.driver.local.resolve.nextWait = savedNext + savedContext := plan.context + plan.context = nil + assertDirectSelector(false) + assertDirectState(false) + plan.context = savedContext + savedPacketCase := packet.caseID + packet.caseID = 1 + assertDirectSelector(true) + assertDirectState(false) + packet.caseID = savedPacketCase + savedExpected := fixture.task.g.park.expected + fixture.task.g.park.expected = 2 + assertDirectSelector(true) + assertDirectState(false) + fixture.task.g.park.expected = savedExpected + assertDirectSelector(true) + assertDirectState(true) + materialize, materializeOK := NextExecutorRunStep(fixture.driver) + if !materializeOK || materialize.Kind != ExecutorRunStepMaterialize || + materialize.Cleanup.Kind != ResumeCleanupChannelDirect || + materialize.Cleanup.Context != unsafe.Pointer(&token) || + materialize.Cleanup.Index != 0 || materialize.Cleanup.WinnerCase != 1 || + materialize.Cleanup.Outcome != ParkOutcomeCompleted || + !CommitResumeCleanupStep(materialize.Cleanup, 3) { + t.Fatalf("owner-local direct materialize = (%+v,%t)", materialize, materializeOK) + } + if _, _, _, stateOK := ownerLocalDirectChannelCleanupState(&fixture.driver.local.resolve); !stateOK { + t.Fatal("direct confirmation certificate rejected materialized status") + } + slot, slotOK := channelOperationSlotFor(fixture.source, fixture.ids[0]) + if !slotOK { + t.Fatal("lookup owner-local direct closing source") + } + preemptStore(&slot.mailbox, uint32(channelMailboxReady)) + if _, _, _, _, direct := ownerLocalDirectChannelPlan( + fixture.driver, + &fixture.driver.local.resolve, + ); direct { + t.Fatal("direct finish accepted a non-empty source mailbox") + } + preemptStore(&slot.mailbox, uint32(channelMailboxEmpty)) + if _, _, _, _, direct := ownerLocalDirectChannelPlan( + fixture.driver, + &fixture.driver.local.resolve, + ); !direct { + t.Fatal("direct finish rejected restored exact source generation") + } + + // The first source reduction after the one typed runtime hook must perform + // confirm, claim release, result transfer, recycle, packet finalization, and + // promotion together. A generic cleanup fallback needs several reductions, + // so this exact boundary is the performance regression gate. + finish, finishOK := NextExecutorRunStep(fixture.driver) + if !finishOK || finish.Kind != ExecutorRunStepSource || !finish.Poll.Complete || + !finish.Poll.AtomicResolve || finish.Poll.Used != 1 || finish.Poll.Promoted != 1 || + finish.Poll.ApplyVisits != 0 { + t.Fatalf("owner-local direct finish = (%+v,%t), plan=%+v local=%+v", + finish, finishOK, plan, fixture.driver.local) + } + if !CommitExecutorRunSourceDistribution(fixture.driver, false) { + t.Fatal("commit owner-local direct source distribution") + } + if packet.state != resumePacketMaterialized || packet.outcome != ParkOutcomeCompleted || + packet.caseID != 1 || packet.result != ResumeResultChannel || packet.small != 3 || + plan != (ResumeCleanupPlan{}) || fixture.wait != (WaitSetRecord{}) || + fixture.ids[0] != (OperationID{}) || + !emptyOwnerLocalCompletion(&fixture.driver.local) || + !channelOperationSourceEmpty(fixture.source, fixture.p) { + t.Fatalf("owner-local direct cleanup retained state: packet=%+v plan=%+v wait=%+v ids=%+v local=%+v empty=%t", + packet, plan, fixture.wait, fixture.ids, fixture.driver.local, + channelOperationSourceEmpty(fixture.source, fixture.p)) + } + + if !EnterExecutorRunCompatibility(fixture.driver) { + t.Fatal("leave owner-local direct bounded runner") + } + if g, ok := NextRunnable(fixture.p); !ok || g != producer.g { + t.Fatalf("dequeue yielded owner-local direct producer = (%p,%t)", g, ok) + } + finishWaitTestTask(t, fixture.p, producer, beginWaitTestResume(t, fixture.p, producer)) + if g, ok := NextRunnable(fixture.p); !ok || g != fixture.task.g { + t.Fatalf("dequeue owner-local direct peer = (%p,%t)", g, ok) + } + action := beginWaitTestResume(t, fixture.p, fixture.task) + outcome, caseID, cancel, result, small, taken := TakeResumePacket( + fixture.task.g, + fixture.ticket, + &packet, + nil, + ) + if !taken || outcome != ParkOutcomeCompleted || caseID != 1 || + cancel != TaskCancelNone || result != ResumeResultChannel || small != 3 { + t.Fatalf("take owner-local direct packet = (%d,%d,%d,%d,%d,%t)", + outcome, caseID, cancel, result, small, taken) + } + yieldRunningDriverTask(t, fixture.p, fixture.task, action) + closeTestExecutorDriver(t, fixture.driver) + finishReadyDriverTasks(t, fixture.p, map[*G]*yieldingTestG{ + fixture.task.g: fixture.task, + }) + if !fixture.source.CanRelease() || !fixture.registry.CanRelease() { + t.Fatal("owner-local direct cleanup retained source/registry") + } + runtime.KeepAlive(producer.frame.memory) + runtime.KeepAlive(fixture.task.frame.memory) +} + +func TestOwnerLocalDirectChannelCommittedResultCompletesInline(t *testing.T) { + var ( + packet ResumePacket + plan ResumeCleanupPlan + token byte + ) + fixture := newChannelClaimCoreFixtureBeforeResume( + t, + "channel-owner-local-direct-result", + []uint32{1}, + true, + 0, + func(fixture *channelClaimCoreFixture) { + if !BindWaitSetResumeCleanup( + &fixture.wait, + &packet, + &plan, + ResumeCleanupBinding{ + Kind: ResumeCleanupChannelDirect, + Context: unsafe.Pointer(&token), + Entries: unsafe.Pointer(&fixture.ids[0]), + Claim: fixture.claim, + Count: 1, + RuntimeCount: 1, + Stride: unsafe.Sizeof(OperationID{}), + }, + ) { + t.Fatal("bind owner-local direct result cleanup") + } + // Production installs this certificate through + // PrepareSingleChannelParkCleanup. This fixture assembles the same + // frozen binding explicitly so it can exercise the source core. + fixture.wait.directChannel = true + }, + ) + + producer := newYieldingTestG(t, "channel-owner-local-direct-result-producer") + if !Enqueue(fixture.p, producer.g) { + t.Fatal("enqueue owner-local direct result producer") + } + if g, ok := NextRunnable(fixture.p); !ok || g != producer.g { + t.Fatalf("dequeue owner-local direct result producer = (%p,%t)", g, ok) + } + producerAction := beginWaitTestResume(t, fixture.p, producer) + slot, slotOK := channelOperationSlotFor(fixture.source, fixture.ids[0]) + if !slotOK { + t.Fatal("find owner-local direct result slot") + } + var transaction ChannelExternalCommit + beginResult, prepared := BeginChannelOwnerLocalDirectCommit( + &transaction, + fixture.source, + fixture.ids[0], + fixture.claim, + producer.g, + fixture.driver, + ) + if beginResult != ChannelExternalCommitBeginPrepared || !prepared || + !transaction.ownerLocalUnadmitted || transaction.endpoint.held || + preemptLoad(&slot.inflight) != 0 || preemptLoad(&slot.externalLease) != 0 { + t.Fatalf("begin owner-local direct result = transaction:%+v claim:%d inflight:%#x lease:%#x", + transaction, selectClaimLoad(fixture.claim), preemptLoad(&slot.inflight), + preemptLoad(&slot.externalLease)) + } + if !transaction.BeginEffect() || transaction.CommitOwnerLocalDirectWithResult(3) != ChannelOwnerLocalCompletedInline { + t.Fatalf("commit owner-local direct result = transaction:%+v claim:%d", + transaction, selectClaimLoad(fixture.claim)) + } + if preemptLoad(&slot.inflight)&producerAdmissionCountMask != 0 || + preemptLoad(&slot.externalLease) != 0 { + t.Fatalf("owner-local direct result paid generic admission: inflight=%#x lease=%#x", + preemptLoad(&slot.inflight), preemptLoad(&slot.externalLease)) + } + if packet.state != resumePacketMaterialized || packet.outcome != ParkOutcomeCompleted || + packet.caseID != 1 || packet.result != ResumeResultChannel || packet.small != 3 || + plan != (ResumeCleanupPlan{}) || fixture.wait != (WaitSetRecord{}) || + fixture.ids[0] != (OperationID{}) || !emptyOwnerLocalCompletion(&fixture.driver.local) || + !channelOperationSourceEmpty(fixture.source, fixture.p) || + fixture.p.readyHead != fixture.task.g || fixture.p.readyTail != fixture.task.g { + t.Fatalf("inline owner-local direct result retained state: packet=%+v plan=%+v wait=%+v ids=%+v local=%+v ready=(%p,%p)", + packet, plan, fixture.wait, fixture.ids, fixture.driver.local, + fixture.p.readyHead, fixture.p.readyTail) + } + yieldRunningDriverTask(t, fixture.p, producer, producerAction) + + if !EnterExecutorRunCompatibility(fixture.driver) { + t.Fatal("leave owner-local direct result bounded runner") + } + if g, ok := NextRunnable(fixture.p); !ok || g != fixture.task.g { + t.Fatalf("dequeue inline owner-local direct result peer = (%p,%t)", g, ok) + } + action := beginWaitTestResume(t, fixture.p, fixture.task) + outcome, caseID, cancel, result, small, taken := TakeResumePacket( + fixture.task.g, + fixture.ticket, + &packet, + nil, + ) + if !taken || outcome != ParkOutcomeCompleted || caseID != 1 || + cancel != TaskCancelNone || result != ResumeResultChannel || small != 3 { + t.Fatalf("take owner-local direct result packet = (%d,%d,%d,%d,%d,%t)", + outcome, caseID, cancel, result, small, taken) + } + yieldRunningDriverTask(t, fixture.p, fixture.task, action) + if g, ok := NextRunnable(fixture.p); !ok || g != producer.g { + t.Fatalf("dequeue yielded owner-local direct result producer = (%p,%t)", g, ok) + } + finishWaitTestTask(t, fixture.p, producer, beginWaitTestResume(t, fixture.p, producer)) + closeTestExecutorDriver(t, fixture.driver) + finishReadyDriverTasks(t, fixture.p, map[*G]*yieldingTestG{ + fixture.task.g: fixture.task, + }) + if !fixture.source.CanRelease() || !fixture.registry.CanRelease() { + t.Fatal("owner-local direct result retained source/registry") + } + runtime.KeepAlive(producer.frame.memory) + runtime.KeepAlive(fixture.task.frame.memory) +} + +func TestOwnerLocalDirectChannelInlinePreflightIsAtomic(t *testing.T) { + var ( + packet ResumePacket + plan ResumeCleanupPlan + token byte + ) + fixture := newChannelClaimCoreFixtureBeforeResume( + t, + "channel-owner-local-direct-preflight", + []uint32{1}, + true, + 0, + func(fixture *channelClaimCoreFixture) { + if !BindWaitSetResumeCleanup( + &fixture.wait, + &packet, + &plan, + ResumeCleanupBinding{ + Kind: ResumeCleanupChannelDirect, + Context: unsafe.Pointer(&token), + Entries: unsafe.Pointer(&fixture.ids[0]), + Claim: fixture.claim, + Count: 1, + RuntimeCount: 1, + Stride: unsafe.Sizeof(OperationID{}), + }, + ) { + t.Fatal("bind owner-local direct preflight cleanup") + } + fixture.wait.directChannel = true + }, + ) + + producer := newYieldingTestG(t, "channel-owner-local-direct-preflight-producer") + if !Enqueue(fixture.p, producer.g) { + t.Fatal("enqueue owner-local direct preflight producer") + } + if g, ok := NextRunnable(fixture.p); !ok || g != producer.g { + t.Fatalf("dequeue owner-local direct preflight producer = (%p,%t)", g, ok) + } + producerAction := beginWaitTestResume(t, fixture.p, producer) + slot, slotOK := channelOperationSlotFor(fixture.source, fixture.ids[0]) + if !slotOK { + t.Fatal("find owner-local direct preflight slot") + } + var transaction ChannelExternalCommit + beginResult, prepared := BeginChannelOwnerLocalDirectCommit( + &transaction, + fixture.source, + fixture.ids[0], + fixture.claim, + producer.g, + fixture.driver, + ) + if beginResult != ChannelExternalCommitBeginPrepared || !prepared || + !transaction.BeginEffect() { + t.Fatalf("begin owner-local direct preflight = (%d,%t,%+v)", + beginResult, prepared, transaction) + } + + // Corrupt immutable packet scratch after the prepared capability. Commit + // has already crossed the physical effect when the inline preflight sees + // this, so it must fail closed without partially detaching, recycling, or + // promoting any owner state. + packet.small = 9 + if result := transaction.CommitOwnerLocalDirectWithResult(3); result != ChannelOwnerLocalCommitInvalid || + transaction.phase != channelExternalCommitPairBroken || + preemptLoad(&slot.state) != uint32(producerSourceClosing) || + !producerSourceSlotQuiesced(&slot.producerSourceSlot) || + slot.record.phase != operationActive || slot.record.resolutionApplied || slot.claim != fixture.claim || + selectClaimLoad(fixture.claim) != selectClaimClaimed || + fixture.wait.state != waitSetRecordActive || fixture.wait.g.park.phase != parkParked || + fixture.p.parkWaitHead != &fixture.wait || fixture.p.parkWaitTail != &fixture.wait || + fixture.p.readyHead != nil || fixture.p.readyTail != nil { + t.Fatalf("failed inline preflight partially mutated state: result=%d transaction=%+v slot=%+v wait=%+v ready=(%p,%p)", + result, transaction, *slot, fixture.wait, fixture.p.readyHead, fixture.p.readyTail) + } + + packet.small = ResumeSmallInvalid + handled, recovered := completeOwnerLocalDirectChannelInline( + transaction.ownerLocalDriver, + transaction.ownerLocalWait, + transaction.ownerLocalAdmission, + transaction.endpoint.source, + transaction.endpoint.slot, + transaction.endpoint.id, + 3, + ) + if !handled || !recovered { + t.Fatalf("recover owner-local direct preflight = (%t,%t)", handled, recovered) + } + transaction = ChannelExternalCommit{} + if packet.state != resumePacketMaterialized || packet.result != ResumeResultChannel || packet.small != 3 || + plan != (ResumeCleanupPlan{}) || fixture.wait != (WaitSetRecord{}) || + fixture.ids[0] != (OperationID{}) || !channelOperationSourceEmpty(fixture.source, fixture.p) || + fixture.p.readyHead != fixture.task.g || fixture.p.readyTail != fixture.task.g { + t.Fatalf("recovered inline preflight retained state: packet=%+v plan=%+v wait=%+v ids=%+v ready=(%p,%p)", + packet, plan, fixture.wait, fixture.ids, fixture.p.readyHead, fixture.p.readyTail) + } + yieldRunningDriverTask(t, fixture.p, producer, producerAction) + + if !EnterExecutorRunCompatibility(fixture.driver) { + t.Fatal("leave owner-local direct preflight bounded runner") + } + if g, ok := NextRunnable(fixture.p); !ok || g != fixture.task.g { + t.Fatalf("dequeue owner-local direct preflight peer = (%p,%t)", g, ok) + } + action := beginWaitTestResume(t, fixture.p, fixture.task) + if outcome, caseID, cancel, result, small, taken := TakeResumePacket( + fixture.task.g, + fixture.ticket, + &packet, + nil, + ); !taken || outcome != ParkOutcomeCompleted || caseID != 1 || cancel != TaskCancelNone || + result != ResumeResultChannel || small != 3 { + t.Fatalf("take recovered owner-local direct preflight packet = (%d,%d,%d,%d,%d,%t)", + outcome, caseID, cancel, result, small, taken) + } + yieldRunningDriverTask(t, fixture.p, fixture.task, action) + if g, ok := NextRunnable(fixture.p); !ok || g != producer.g { + t.Fatalf("dequeue yielded owner-local direct preflight producer = (%p,%t)", g, ok) + } + finishWaitTestTask(t, fixture.p, producer, beginWaitTestResume(t, fixture.p, producer)) + closeTestExecutorDriver(t, fixture.driver) + finishReadyDriverTasks(t, fixture.p, map[*G]*yieldingTestG{ + fixture.task.g: fixture.task, + }) + if !fixture.source.CanRelease() || !fixture.registry.CanRelease() { + t.Fatal("owner-local direct preflight retained source/registry") + } + runtime.KeepAlive(producer.frame.memory) + runtime.KeepAlive(fixture.task.frame.memory) +} + +func TestOwnerLocalDirectChannelReadyRaceUpgradesAdmission(t *testing.T) { + var ( + packet ResumePacket + plan ResumeCleanupPlan + token byte + ) + fixture := newChannelClaimCoreFixtureBeforeResume( + t, + "channel-owner-local-ready-race", + []uint32{1}, + true, + 0, + func(fixture *channelClaimCoreFixture) { + if !BindWaitSetResumeCleanup( + &fixture.wait, + &packet, + &plan, + ResumeCleanupBinding{ + Kind: ResumeCleanupChannelDirect, + Context: unsafe.Pointer(&token), + Entries: unsafe.Pointer(&fixture.ids[0]), + Claim: fixture.claim, + Count: 1, + RuntimeCount: 1, + Stride: unsafe.Sizeof(OperationID{}), + }, + ) { + t.Fatal("bind ready-race direct result cleanup") + } + fixture.wait.directChannel = true + }, + ) + + producer := newYieldingTestG(t, "channel-owner-local-ready-race-producer") + if !Enqueue(fixture.p, producer.g) { + t.Fatal("enqueue ready-race producer") + } + if g, ok := NextRunnable(fixture.p); !ok || g != producer.g { + t.Fatalf("dequeue ready-race producer = (%p,%t)", g, ok) + } + producerAction := beginWaitTestResume(t, fixture.p, producer) + slot, slotOK := channelOperationSlotFor(fixture.source, fixture.ids[0]) + if !slotOK { + t.Fatal("find ready-race channel slot") + } + var transaction ChannelExternalCommit + if result, prepared := BeginChannelOwnerLocalDirectCommit( + &transaction, + fixture.source, + fixture.ids[0], + fixture.claim, + producer.g, + fixture.driver, + ); result != ChannelExternalCommitBeginPrepared || !prepared || + !transaction.ownerLocalUnadmitted || transaction.endpoint.held { + t.Fatalf("begin ready-race direct transaction = (%d,%t,%+v)", result, prepared, transaction) + } + if result := fixture.source.PostReady(fixture.ids[0]); result != ChannelOperationPosted { + t.Fatalf("post ready-race source = %d", result) + } + if !transaction.BeginEffect() || + transaction.CommitOwnerLocalDirectWithResult(3) != ChannelOwnerLocalCommitFallback { + t.Fatalf("ready-race direct fallback = %+v", transaction) + } + if transaction.ownerLocalUnadmitted || !transaction.endpoint.held || + preemptLoad(&slot.inflight) != 1 || preemptLoad(&slot.externalLease)&1 == 0 || + selectClaimLoad(fixture.claim) != selectClaimCommitting { + t.Fatalf("ready-race admission upgrade = transaction:%+v inflight:%#x lease:%#x claim:%d", + transaction, preemptLoad(&slot.inflight), preemptLoad(&slot.externalLease), + selectClaimLoad(fixture.claim)) + } + if !transaction.CommitAtRouteWithResult(fixture.driver.route, 3) || + transaction != (ChannelExternalCommit{}) || preemptLoad(&slot.inflight) != 0 || + preemptLoad(&slot.externalLease)&1 != 0 || selectClaimLoad(fixture.claim) != selectClaimClaimed { + t.Fatalf("commit ready-race fallback = transaction:%+v inflight:%#x lease:%#x claim:%d", + transaction, preemptLoad(&slot.inflight), preemptLoad(&slot.externalLease), + selectClaimLoad(fixture.claim)) + } + + local, localOK := TryPublishOwnerLocalChannelCompletionCurrent( + producer.g, + fixture.driver, + fixture.source, + fixture.ids[0], + ) + if !localOK || !local { + t.Fatalf("publish ready-race owner-local fallback = (%t,%t)", local, localOK) + } + yieldRunningDriverTask(t, fixture.p, producer, producerAction) + + finish, finishOK := NextExecutorRunStep(fixture.driver) + if !finishOK || finish.Kind != ExecutorRunStepSource || !finish.Poll.Complete || + !finish.Poll.AtomicResolve || finish.Poll.ApplyVisits != 1 || finish.Poll.Promoted != 1 || + !CommitExecutorRunSourceDistribution(fixture.driver, false) { + t.Fatalf("ready-race owner-local finish = (%+v,%t)", finish, finishOK) + } + if !EnterExecutorRunCompatibility(fixture.driver) { + t.Fatal("leave ready-race bounded runner") + } + if g, ok := NextRunnable(fixture.p); !ok || g != producer.g { + t.Fatalf("dequeue yielded ready-race producer = (%p,%t)", g, ok) + } + finishWaitTestTask(t, fixture.p, producer, beginWaitTestResume(t, fixture.p, producer)) + if g, ok := NextRunnable(fixture.p); !ok || g != fixture.task.g { + t.Fatalf("dequeue ready-race peer = (%p,%t)", g, ok) + } + action := beginWaitTestResume(t, fixture.p, fixture.task) + outcome, caseID, cancel, result, small, taken := TakeResumePacket( + fixture.task.g, + fixture.ticket, + &packet, + nil, + ) + if !taken || outcome != ParkOutcomeCompleted || caseID != 1 || cancel != TaskCancelNone || + result != ResumeResultChannel || small != 3 { + t.Fatalf("take ready-race packet = (%d,%d,%d,%d,%d,%t)", + outcome, caseID, cancel, result, small, taken) + } + yieldRunningDriverTask(t, fixture.p, fixture.task, action) + closeTestExecutorDriver(t, fixture.driver) + finishReadyDriverTasks(t, fixture.p, map[*G]*yieldingTestG{ + fixture.task.g: fixture.task, + }) + if !fixture.source.CanRelease() || !fixture.registry.CanRelease() { + t.Fatal("ready-race fallback retained source/registry") + } + runtime.KeepAlive(producer.frame.memory) + runtime.KeepAlive(fixture.task.frame.memory) +} + +func TestOwnerLocalDirectChannelJoinsProducerAdmittedBeforePhysicalCommit(t *testing.T) { + var ( + packet ResumePacket + plan ResumeCleanupPlan + token byte + ) + fixture := newChannelClaimCoreFixtureBeforeResume( + t, + "channel-owner-local-admitted-producer", + []uint32{1}, + true, + 0, + func(fixture *channelClaimCoreFixture) { + if !BindWaitSetResumeCleanup( + &fixture.wait, + &packet, + &plan, + ResumeCleanupBinding{ + Kind: ResumeCleanupChannelDirect, + Context: unsafe.Pointer(&token), + Entries: unsafe.Pointer(&fixture.ids[0]), + Claim: fixture.claim, + Count: 1, + RuntimeCount: 1, + Stride: unsafe.Sizeof(OperationID{}), + }, + ) { + t.Fatal("bind admitted-producer direct cleanup") + } + fixture.wait.directChannel = true + }, + ) + + producer := newYieldingTestG(t, "channel-owner-local-admitted-producer-current") + if !Enqueue(fixture.p, producer.g) { + t.Fatal("enqueue admitted-producer current") + } + if g, ok := NextRunnable(fixture.p); !ok || g != producer.g { + t.Fatalf("dequeue admitted-producer current = (%p,%t)", g, ok) + } + producerAction := beginWaitTestResume(t, fixture.p, producer) + slot, slotOK := channelOperationSlotFor(fixture.source, fixture.ids[0]) + if !slotOK { + t.Fatal("find admitted-producer channel slot") + } + var transaction ChannelExternalCommit + if result, prepared := BeginChannelOwnerLocalDirectCommit( + &transaction, + fixture.source, + fixture.ids[0], + fixture.claim, + producer.g, + fixture.driver, + ); result != ChannelExternalCommitBeginPrepared || !prepared || + !transaction.ownerLocalUnadmitted || transaction.endpoint.held { + t.Fatalf("begin admitted-producer direct transaction = (%d,%t,%+v)", result, prepared, transaction) + } + if acquired := acquireProducerSourceGeneration( + &slot.producerSourceSlot, + fixture.ids[0].Generation, + ); acquired != producerSourceAcquired { + t.Fatalf("hold source producer before physical commit = %d", acquired) + } + if !transaction.BeginEffect() || + transaction.CommitOwnerLocalDirectWithResult(3) != ChannelOwnerLocalCommitted { + t.Fatalf("commit with admitted producer = %+v", transaction) + } + if transaction != (ChannelExternalCommit{}) || + preemptLoad(&slot.state) != uint32(producerSourceClosing) || + preemptLoad(&slot.inflight) != producerAdmissionClosed|1 || + fixture.driver.local.head != &fixture.wait || fixture.driver.local.tail != &fixture.wait || + fixture.wait.work != waitSetWorkQueued || slot.record.phase != operationActive || + selectClaimLoad(fixture.claim) != selectClaimClaimed { + t.Fatalf("admitted producer did not defer inline detach: transaction=%+v state=%d inflight=%#x local=(%p,%p) wait=%+v record=%+v claim=%d", + transaction, preemptLoad(&slot.state), preemptLoad(&slot.inflight), + fixture.driver.local.head, fixture.driver.local.tail, fixture.wait, + slot.record, selectClaimLoad(fixture.claim)) + } + posted := fixture.source.postReadyAdmitted(slot, fixture.ids[0]) + released := producerAdmissionReleaseChecked(&slot.inflight) + if posted != ChannelOperationPostClosed || !released || + preemptLoad(&slot.inflight) != producerAdmissionClosed { + t.Fatalf("finish admitted producer after seal = posted:%d inflight:%#x", + posted, preemptLoad(&slot.inflight)) + } + yieldRunningDriverTask(t, fixture.p, producer, producerAction) + + complete := false + for reduction := 0; reduction < 16; reduction++ { + step, advanced := NextExecutorRunStep(fixture.driver) + if !advanced { + t.Fatalf("admitted-producer reduction %d did not advance", reduction) + } + switch step.Kind { + case ExecutorRunStepSource: + if !step.Poll.AtomicResolve || step.Poll.Used != 1 { + t.Fatalf("admitted-producer source reduction %d = %+v", reduction, step.Poll) + } + if step.Poll.Complete { + if !CommitExecutorRunSourceDistribution(fixture.driver, false) { + t.Fatal("commit admitted-producer source distribution") + } + complete = true + } + case ExecutorRunStepMaterialize: + if step.Cleanup.Kind != ResumeCleanupChannelDirect || + !CommitResumeCleanupStep(step.Cleanup, 3) { + t.Fatalf("materialize admitted-producer cleanup %d = %+v", reduction, step.Cleanup) + } + default: + t.Fatalf("admitted-producer reduction %d kind = %d", reduction, step.Kind) + } + if complete { + break + } + } + if !complete || packet.state != resumePacketMaterialized || packet.outcome != ParkOutcomeCompleted || + packet.caseID != 1 || packet.result != ResumeResultChannel || packet.small != 3 { + t.Fatalf("admitted-producer cleanup incomplete: complete=%t packet=%+v", complete, packet) + } + if !EnterExecutorRunCompatibility(fixture.driver) { + t.Fatal("leave admitted-producer bounded runner") + } + if g, ok := NextRunnable(fixture.p); !ok || g != producer.g { + t.Fatalf("dequeue admitted-producer current after yield = (%p,%t)", g, ok) + } + finishWaitTestTask(t, fixture.p, producer, beginWaitTestResume(t, fixture.p, producer)) + if g, ok := NextRunnable(fixture.p); !ok || g != fixture.task.g { + t.Fatalf("dequeue admitted-producer peer = (%p,%t)", g, ok) + } + action := beginWaitTestResume(t, fixture.p, fixture.task) + outcome, caseID, cancel, result, small, taken := TakeResumePacket( + fixture.task.g, + fixture.ticket, + &packet, + nil, + ) + if !taken || outcome != ParkOutcomeCompleted || caseID != 1 || cancel != TaskCancelNone || + result != ResumeResultChannel || small != 3 { + t.Fatalf("take admitted-producer packet = (%d,%d,%d,%d,%d,%t)", + outcome, caseID, cancel, result, small, taken) + } + yieldRunningDriverTask(t, fixture.p, fixture.task, action) + closeTestExecutorDriver(t, fixture.driver) + finishReadyDriverTasks(t, fixture.p, map[*G]*yieldingTestG{ + fixture.task.g: fixture.task, + }) + if !fixture.source.CanRelease() || !fixture.registry.CanRelease() { + t.Fatal("admitted-producer cleanup retained source/registry") + } + runtime.KeepAlive(producer.frame.memory) + runtime.KeepAlive(fixture.task.frame.memory) +} + +func TestOwnerLocalChannelCompletionConsumesInitialAffectedVisit(t *testing.T) { + fixture := newChannelClaimCoreFixture(t, "channel-owner-local-initial", []uint32{152}, true, 0) + if fixture.wait.work != waitSetWorkQueued || fixture.p.affectedWaitHead != &fixture.wait || + fixture.p.affectedWaitTail != &fixture.wait { + t.Fatalf("initial affected visit = wait:%+v queue:(%p,%p)", + fixture.wait, fixture.p.affectedWaitHead, fixture.p.affectedWaitTail) + } + + producer := newYieldingTestG(t, "channel-owner-local-initial-producer") + if !Enqueue(fixture.p, producer.g) { + t.Fatal("enqueue initial-visit producer") + } + if g, ok := NextRunnable(fixture.p); !ok || g != producer.g { + t.Fatalf("dequeue initial-visit producer = (%p,%t)", g, ok) + } + producerAction := beginWaitTestResume(t, fixture.p, producer) + externallyCommitChannelCandidateAtRoute(t, fixture, 0, RouteID(1)) + + local, ok := TryPublishOwnerLocalChannelCompletion(producer.g, fixture.source, fixture.ids[0]) + if !ok || !local || fixture.p.affectedWaitHead != nil || fixture.p.affectedWaitTail != nil || + fixture.driver.local.head != &fixture.wait || fixture.driver.local.tail != &fixture.wait || + fixture.wait.work != waitSetWorkQueued || fixture.wait.workNext != nil || fixture.source.Pending() || + fixture.registry.ObserveRequested(fixture.handle) { + t.Fatalf("consume initial affected visit = (%t,%t) affected:(%p,%p) local:(%p,%p) wait:%+v pending:%t request:%t", + local, ok, fixture.p.affectedWaitHead, fixture.p.affectedWaitTail, + fixture.driver.local.head, fixture.driver.local.tail, fixture.wait, fixture.source.Pending(), + fixture.registry.ObserveRequested(fixture.handle)) + } + yieldRunningDriverTask(t, fixture.p, producer, producerAction) + + var complete ExecutorPollProgress + for reduction := 0; reduction < 8; reduction++ { + step, advanced := NextExecutorRunStep(fixture.driver) + if !advanced { + t.Fatalf("initial-visit local reduction %d failed", reduction) + } + switch step.Kind { + case ExecutorRunStepSource: + if !step.Poll.AtomicResolve || step.Poll.Used != 1 { + t.Fatalf("initial-visit source reduction %d = %+v", reduction, step.Poll) + } + complete = step.Poll + case ExecutorRunStepMaterialize: + // The core fixture binds no runtime-owned typed cleanup plan. + t.Fatalf("unexpected initial-visit materialize reduction %d", reduction) + default: + t.Fatalf("unexpected initial-visit reduction %d kind %d", reduction, step.Kind) + } + if complete.Complete { + if !CommitExecutorRunSourceDistribution(fixture.driver, false) { + t.Fatal("commit initial-visit local distribution") + } + break + } + } + if !complete.Complete || complete.Promoted != 1 || !emptyOwnerLocalCompletion(&fixture.driver.local) || + fixture.task.g.park.phase != parkReady || fixture.source.Pending() { + t.Fatalf("initial-visit owner-local completion = %+v local:%+v park:%+v pending:%t", + complete, fixture.driver.local, fixture.task.g.park, fixture.source.Pending()) + } + + if !EnterExecutorRunCompatibility(fixture.driver) { + t.Fatal("leave initial-visit bounded runner") + } + if g, runnable := NextRunnable(fixture.p); !runnable || g != producer.g { + t.Fatalf("dequeue yielded initial-visit producer = (%p,%t)", g, runnable) + } + finishWaitTestTask(t, fixture.p, producer, beginWaitTestResume(t, fixture.p, producer)) + decision := takeChannelClaimCoreDecision(t, fixture) + if decision.outcome != ParkOutcomeCompleted || decision.caseID != 152 || !decision.lease.Valid() { + t.Fatalf("initial-visit peer decision = %+v", decision) + } + releaseChannelClaimCoreFixture(t, fixture, decision) + runtime.KeepAlive(producer.frame.memory) +} + func TestSelectClaimLayoutPairAcquisitionAndFrozenSourceID(t *testing.T) { if unsafe.Sizeof(SelectClaim{}) != 4 || unsafe.Alignof(SelectClaim{}) != 4 { t.Fatalf("SelectClaim layout = size:%d align:%d", unsafe.Sizeof(SelectClaim{}), unsafe.Alignof(SelectClaim{})) @@ -1159,7 +1982,8 @@ func TestChannelReadyTryCommitRetryBudgetYieldsEpochAndPreservesReady(t *testing var progress ExecutorPollProgress for step := 0; step < 10000; step++ { runStep, ok := NextExecutorRunStep(fixture.driver) - if !ok || runStep.Kind != ExecutorRunStepSource || runStep.Poll.Used != 1 { + if !ok || runStep.Kind != ExecutorRunStepSource || runStep.Poll.Used == 0 || + runStep.Poll.Used > executorRunSourceBatchQuantum { t.Fatalf("bounded retry source step %d = (%+v,%t)", step, runStep, ok) } progress = runStep.Poll diff --git a/runtime/internal/coro/channel_operation_source.go b/runtime/internal/coro/channel_operation_source.go index 129fd90b44..82c9053671 100644 --- a/runtime/internal/coro/channel_operation_source.go +++ b/runtime/internal/coro/channel_operation_source.go @@ -68,32 +68,45 @@ const ( channelPhysicalStateBits = 2 channelPhysicalStateMask = uint32(1< channelPhysicalCommitted || route != 0 && !route.Valid() || - route != 0 && state != channelPhysicalCommitted { + (route != 0 || small != ResumeSmallInvalid) && state != channelPhysicalCommitted { return 0, false } - return uint32(state) | uint32(route)<> channelPhysicalRouteShift) + route := RouteID(word >> channelPhysicalRouteShift & operationRouteMask) + small := uint8(word >> channelPhysicalSmallShift) if word&^channelPhysicalWordMask != 0 || state > channelPhysicalCommitted || - route != 0 && (!route.Valid() || state != channelPhysicalCommitted) { + route != 0 && !route.Valid() || + (route != 0 || small != ResumeSmallInvalid) && state != channelPhysicalCommitted { return channelPhysicalState(^uint32(0)) } return state } -func channelPhysicalCompletionRoute(word uint32) (RouteID, bool) { +func channelPhysicalCompletion(word uint32) (RouteID, uint8, bool) { if channelPhysicalStateOf(word) != channelPhysicalCommitted { - return 0, false + return 0, ResumeSmallInvalid, false } - return RouteID(word >> channelPhysicalRouteShift), true + return RouteID(word >> channelPhysicalRouteShift & operationRouteMask), + uint8(word >> channelPhysicalSmallShift), true +} + +func channelPhysicalCompletionRoute(word uint32) (RouteID, bool) { + route, _, ok := channelPhysicalCompletion(word) + return route, ok } type channelExternalState uint32 @@ -123,6 +136,20 @@ type channelOperationSlot struct { claim *SelectClaim } +// ChannelDirectReservation is an owner-stack capability retained only across +// the fallible frame/capacity preflight and the no-fail direct park commit. It +// never enters a coroutine frame or producer ABI. Stable source configuration +// and owner serialization make the selected reusable slot immutable during +// that interval. Its fields are deliberately private: runtime adapters may +// carry the value but cannot forge a source, owner, or slot identity. +type ChannelDirectReservation struct { + source *ChannelOperationSource + owner *P + slot *channelOperationSlot + index uint32 + capacity uint32 +} + // ChannelOperationPage is stable target-provided storage. Producer ingress // still resolves it exclusively through the existing two-word OperationID; // neither this page pointer nor a frame pointer crosses an ingress ABI. @@ -537,7 +564,54 @@ func channelOperationCommitDomainCompatible( return true } -func (source *ChannelOperationSource) ReserveAndAttachWait( +func (source *ChannelOperationSource) reserveAndAttachWaitSlot( + state *ParkState, + ticket ParkTicket, + wait *WaitSetRecord, + caseID uint32, + claim *SelectClaim, + slot *channelOperationSlot, + index uint32, + capacity uint32, +) (OperationID, bool) { + if slot == nil || index >= capacity || capacity != ChannelOperationConfiguredCapacity(source) { + return OperationID{}, false + } + generation, begun := beginProducerSourceSlot(&slot.producerSourceSlot) + if !begun { + return OperationID{}, false + } + if !raiseSourceScanLimit(&source.scanLimit, index, capacity) { + return OperationID{}, false + } + id, ok := MakeOperationIDAtRoute(OperationSourceChannel, source.route, index+1, generation) + if !ok || !PrepareOperationAtGeneration(&slot.record, id) { + _ = resetProducerSourceSlot(&slot.producerSourceSlot, generation) + return OperationID{}, false + } + if !DeclareOperationCommitMode(&slot.record, OperationCommitReadyThenTryCommit) || + !AttachParkWaitOperation(state, ticket, wait, &slot.record, caseID) { + if !AbortReservedOperation(&slot.record, id) || + !resetProducerSourceSlot(&slot.producerSourceSlot, generation) { + return OperationID{}, false + } + return OperationID{}, false + } + slot.claim = claim + if claim != nil { + preemptStore(&slot.external, uint32(channelExternalReserved)) + } + if !activateProducerSourceSlot(&slot.producerSourceSlot, generation) { + return OperationID{}, false + } + source.reserveCursor = index + 1 + if source.reserveCursor == capacity { + source.reserveCursor = 0 + } + return id, true +} + +func (source *ChannelOperationSource) reserveAndAttachWait( p *P, state *ParkState, ticket ParkTicket, @@ -545,7 +619,8 @@ func (source *ChannelOperationSource) ReserveAndAttachWait( caseID uint32, claim *SelectClaim, ) (OperationID, bool) { - if !validChannelOperationOwner(source, p) || claim != nil && selectClaimLoad(claim) != selectClaimOpen || + if !validChannelOperationOwner(source, p) || + claim != nil && selectClaimLoad(claim) != selectClaimOpen || !channelOperationCommitDomainCompatible(source, state, ticket, wait, claim) { return OperationID{}, false } @@ -564,58 +639,107 @@ func (source *ChannelOperationSource) ReserveAndAttachWait( claim != nil && !channelOperationExternalReservable(slot) { continue } - generation, begun := beginProducerSourceSlot(&slot.producerSourceSlot) - if !begun { - return OperationID{}, false - } - if !raiseSourceScanLimit(&source.scanLimit, index, ChannelOperationConfiguredCapacity(source)) { - return OperationID{}, false - } - id, ok := MakeOperationIDAtRoute(OperationSourceChannel, source.route, index+1, generation) - if !ok || !PrepareOperationAtGeneration(&slot.record, id) { - _ = resetProducerSourceSlot(&slot.producerSourceSlot, generation) - return OperationID{}, false - } - if !DeclareOperationCommitMode(&slot.record, OperationCommitReadyThenTryCommit) || - !AttachParkWaitOperation(state, ticket, wait, &slot.record, caseID) { - if !AbortReservedOperation(&slot.record, id) || - !resetProducerSourceSlot(&slot.producerSourceSlot, generation) { - return OperationID{}, false - } - return OperationID{}, false - } - slot.claim = claim - if claim != nil { - preemptStore(&slot.external, uint32(channelExternalReserved)) - } - if !activateProducerSourceSlot(&slot.producerSourceSlot, generation) { - return OperationID{}, false + return source.reserveAndAttachWaitSlot( + state, ticket, wait, caseID, claim, slot, index, capacity, + ) + } + return OperationID{}, false +} + +func (source *ChannelOperationSource) ReserveAndAttachWait( + p *P, + state *ParkState, + ticket ParkTicket, + wait *WaitSetRecord, + caseID uint32, + claim *SelectClaim, +) (OperationID, bool) { + return source.reserveAndAttachWait(p, state, ticket, wait, caseID, claim) +} + +func (source *ChannelOperationSource) preflightDirectReservationOwned(p *P) (ChannelDirectReservation, bool) { + capacity := ChannelOperationConfiguredCapacity(source) + start := source.reserveCursor + if start >= capacity { + start = 0 + } + for offset := uint32(0); offset < capacity; offset++ { + index := start + offset + if index >= capacity { + index -= capacity } - source.reserveCursor = index + 1 - if source.reserveCursor == capacity { - source.reserveCursor = 0 + slot, ok := channelOperationSlotAt(source, index) + if ok && channelOperationReusableSlot(source, slot, index) && + preemptLoad(&slot.generation) != ^uint32(0) && + channelOperationExternalReservable(slot) { + return ChannelDirectReservation{ + source: source, owner: p, slot: slot, index: index, capacity: capacity, + }, true } - return id, true } - return OperationID{}, false + return ChannelDirectReservation{}, false } -// ExposeExternalCommit is the final owner-side publication before a typed -// hchan node becomes reachable. PrepareParkSet has already frozen the exact -// ParkState/WaitSet/frame relation and installed pendingParkSet; after this -// release publication the compiler/runtime path may only publish the node and -// execute llvm.coro.suspend. Rejection leaves Reserved unchanged, so no peer -// can mistake a partial preparation for a committable endpoint. -func (source *ChannelOperationSource) ExposeExternalCommit( +func (source *ChannelOperationSource) PreflightDirectReservation(p *P) (ChannelDirectReservation, bool) { + if !validChannelOperationOwner(source, p) { + return ChannelDirectReservation{}, false + } + return source.preflightDirectReservationOwned(p) +} + +// validDirectReservationHeader authenticates only the private identity carried +// across the direct park's no-suspend interval. PreflightDirectReservation +// already audited the owner binding and reusable slot before constructing this +// value; the consuming slot primitive still checks capacity and atomically +// begins the exact generation. Repeating the complete source-owner proof at +// every intervening helper would turn this capability back into a hint. +func validDirectReservationHeader( + source *ChannelOperationSource, + p *P, + reservation ChannelDirectReservation, +) bool { + return source != nil && p != nil && reservation.source == source && + reservation.owner == p && reservation.slot != nil && + reservation.capacity != 0 && reservation.index < reservation.capacity +} + +func (source *ChannelOperationSource) reserveAndAttachDirectWait( + p *P, + state *ParkState, + ticket ParkTicket, + wait *WaitSetRecord, + caseID uint32, + claim *SelectClaim, + reservation ChannelDirectReservation, +) (OperationID, bool) { + // The compiler-owned direct-channel recipe supplies fresh frame storage for + // one exact ParkSet. Its private claim cannot alias another live generation, + // so the preflight capability replaces both the shared-domain scan and a + // second reusable-slot scan. The caller consumes the capability in the same + // no-suspend interval; do not repeat its owner proof here. + if !validDirectReservationHeader(source, p, reservation) || claim == nil || + selectClaimLoad(claim) != selectClaimOpen || state == nil || + state.phase != parkPreparing || state.ticket != ticket || state.expected != 1 || + state.attached != 0 || state.head != nil || + !validPreparingWaitSetRecord(wait, state, ticket) { + return OperationID{}, false + } + return source.reserveAndAttachWaitSlot( + state, ticket, wait, caseID, claim, + reservation.slot, reservation.index, reservation.capacity, + ) +} + +func (source *ChannelOperationSource) exposeExternalCommitSlot( p *P, g *G, id OperationID, ticket ParkTicket, wait *WaitSetRecord, claim *SelectClaim, + slot *channelOperationSlot, ) bool { - slot, ok := channelOperationSlotFor(source, id) - if !ok || !validChannelOperationOwner(source, p) || g == nil || claim == nil || + if p == nil || slot == nil || g == nil || claim == nil || preemptLoad(&slot.generation) != id.Generation || preemptLoad(&slot.state) != uint32(producerSourceActive) || preemptLoad(&slot.external) != uint32(channelExternalReserved) || slot.claim != claim || selectClaimLoad(claim) != selectClaimOpen || g.runP != p || p.current != g || !p.inResume || @@ -635,6 +759,51 @@ func (source *ChannelOperationSource) ExposeExternalCommit( ) } +// exposeExternalCommitDirect consumes the same private reservation which +// selected and began this slot. OperationID construction fixes its route and +// one-based local index, so the terminal publication needs no catalog lookup +// or repeated owner audit. All mutable post-build state is still checked by +// exposeExternalCommitSlot immediately before the release CAS. +func (source *ChannelOperationSource) exposeExternalCommitDirect( + g *G, + id OperationID, + ticket ParkTicket, + wait *WaitSetRecord, + claim *SelectClaim, + reservation ChannelDirectReservation, +) bool { + p := reservation.owner + if !validDirectReservationHeader(source, p, reservation) || !id.Valid() || + id.Source() != OperationSourceChannel || id.Route() != source.route || + id.LocalSlot() != reservation.index+1 { + return false + } + return source.exposeExternalCommitSlot( + p, g, id, ticket, wait, claim, reservation.slot, + ) +} + +// ExposeExternalCommit is the final owner-side publication before a typed +// hchan node becomes reachable. PrepareParkSet has already frozen the exact +// ParkState/WaitSet/frame relation and installed pendingParkSet; after this +// release publication the compiler/runtime path may only publish the node and +// execute llvm.coro.suspend. Rejection leaves Reserved unchanged, so no peer +// can mistake a partial preparation for a committable endpoint. +func (source *ChannelOperationSource) ExposeExternalCommit( + p *P, + g *G, + id OperationID, + ticket ParkTicket, + wait *WaitSetRecord, + claim *SelectClaim, +) bool { + slot, ok := channelOperationSlotFor(source, id) + if !ok || !validChannelOperationOwner(source, p) { + return false + } + return source.exposeExternalCommitSlot(p, g, id, ticket, wait, claim, slot) +} + // AbortSelectPreparation atomically excludes external claimers, aborts one // owner preparation transaction, and terminalizes its Channel commit domain. // This is not a general cancellation or resolver entry: wait must still be @@ -916,17 +1085,20 @@ func (admission *channelExternalCommitAdmission) releaseWithoutCommit() bool { // stored Claimed. Executor requests happen only after releaseCommitted on both // endpoints. func (admission *channelExternalCommitAdmission) publishExternallyCommitted() ChannelOperationPostResult { - return admission.publishExternallyCommittedAtRoute(0) + return admission.publishExternallyCommittedAtRoute(0, ResumeSmallInvalid) } -func (admission *channelExternalCommitAdmission) publishExternallyCommittedAtRoute(route RouteID) ChannelOperationPostResult { +func (admission *channelExternalCommitAdmission) publishExternallyCommittedAtRoute( + route RouteID, + small uint8, +) ChannelOperationPostResult { if admission == nil || !admission.held || admission.posted || admission.broken || admission.source == nil || admission.slot == nil || !admission.id.Valid() || admission.token == 0 || admission.token&1 == 0 || preemptLoad(&admission.slot.externalLease) != admission.token || route != 0 && !route.Valid() { return ChannelOperationPostInvalid } source, slot, id := admission.source, admission.slot, admission.id - result := source.publishExternallyCommittedHeld(slot, id, route) + result := source.publishExternallyCommittedHeld(slot, id, route, small) if result == ChannelOperationPosted { admission.posted = true } else { @@ -1223,20 +1395,23 @@ func (pair *channelExternalCommitPair) abort() bool { // A post-effect invariant failure retains the transaction/admissions and must // fail-stop; rollback is never legal in Effect or Broken. func (pair *channelExternalCommitPair) commit() bool { - return pair.commitAtRoute(0) + return pair.commitAtRoute(0, ResumeSmallInvalid, ResumeSmallInvalid) } -func (pair *channelExternalCommitPair) commitAtRoute(route RouteID) bool { +func (pair *channelExternalCommitPair) commitAtRoute( + route RouteID, + firstSmall, secondSmall uint8, +) bool { if pair == nil || pair.self != pair || pair.phase != channelExternalCommitPairEffect || route != 0 && !route.Valid() { return false } - firstResult := pair.endpointA.publishExternallyCommittedAtRoute(route) + firstResult := pair.endpointA.publishExternallyCommittedAtRoute(route, firstSmall) if firstResult != ChannelOperationPosted { pair.phase = channelExternalCommitPairBroken return false } - secondResult := pair.endpointB.publishExternallyCommittedAtRoute(route) + secondResult := pair.endpointB.publishExternallyCommittedAtRoute(route, secondSmall) if secondResult != ChannelOperationPosted { pair.phase = channelExternalCommitPairBroken return false @@ -1331,14 +1506,30 @@ func (pair *ChannelExternalCommitPair) Abort() bool { } func (pair *ChannelExternalCommitPair) Commit() bool { - return pair != nil && pair.transaction.commit() + return pair != nil && pair.transaction.commitAtRoute( + 0, ResumeSmallInvalid, ResumeSmallInvalid, + ) } // CommitAtRoute is Commit with an advisory producer-locality hint. It is used // only by a currently running logical Go task; foreign and asynchronous // producers retain Commit's route-zero behavior. func (pair *ChannelExternalCommitPair) CommitAtRoute(route RouteID) bool { - return pair != nil && pair.transaction.commitAtRoute(route) + return pair != nil && pair.transaction.commitAtRoute( + route, ResumeSmallInvalid, ResumeSmallInvalid, + ) +} + +// CommitAtRouteWithResults publishes the two optional closed runtime result +// tags with the physical pair commit. The tags are opaque to the source core; +// zero independently keeps an endpoint on the general typed-materialization +// path, while a fused direct winner may consume its non-zero tag after the +// producer admission join. +func (pair *ChannelExternalCommitPair) CommitAtRouteWithResults( + route RouteID, + firstSmall, secondSmall uint8, +) bool { + return pair != nil && pair.transaction.commitAtRoute(route, firstSmall, secondSmall) } // ChannelExternalCommit is the single-endpoint counterpart of the pair @@ -1355,11 +1546,16 @@ func (pair *ChannelExternalCommitPair) CommitAtRoute(route RouteID) bool { // heap allocation and that Begin -> BeginEffect -> typed effect -> Commit is a // NoSuspend/NoPanic span. type ChannelExternalCommit struct { - self *ChannelExternalCommit - endpoint channelExternalCommitAdmission - claim *SelectClaim - phase channelExternalCommitPairPhase - _ [7]byte + self *ChannelExternalCommit + endpoint channelExternalCommitAdmission + claim *SelectClaim + ownerLocalCurrent *G + ownerLocalDriver *ExecutorDriver + ownerLocalWait *WaitSetRecord + phase channelExternalCommitPairPhase + ownerLocalAdmission ownerLocalCompletionAdmission + ownerLocalUnadmitted bool + _ [5]byte } // ChannelExternalCommitBeginResult distinguishes ordinary stale/contention @@ -1381,11 +1577,28 @@ const ( ChannelExternalCommitBeginInvariantFailure ) +// availableChannelExternalCommitOutput checks the two authoritative linear +// state fields. Invalid owns no endpoint or frame capability, so its suffix is +// scratch and is overwritten in full by Begin. Prepared, Effect, Broken, and +// every copied live transaction retain either self or a non-invalid phase and +// cannot be reused as output. +func availableChannelExternalCommitOutput(out *ChannelExternalCommit) bool { + return out != nil && out.self == nil && out.phase == channelExternalCommitPairInvalid +} + func releaseChannelExternalCommitWithoutClaim(transaction *ChannelExternalCommit) bool { if transaction == nil || transaction.self != transaction || transaction.phase != channelExternalCommitPairPrepared || transaction.claim == nil { return false } + if transaction.ownerLocalUnadmitted { + if transaction.endpoint.held || transaction.endpoint.posted || transaction.endpoint.broken { + transaction.phase = channelExternalCommitPairBroken + return false + } + *transaction = ChannelExternalCommit{} + return true + } if !transaction.endpoint.releaseWithoutCommit() { transaction.phase = channelExternalCommitPairBroken return false @@ -1406,7 +1619,107 @@ func BeginChannelExternalCommit( id OperationID, claim *SelectClaim, ) ChannelExternalCommitBeginResult { - if out == nil || *out != (ChannelExternalCommit{}) || source == nil || !id.Valid() || claim == nil { + return beginChannelExternalCommit(out, source, id, claim, nil, nil, false) +} + +// BeginChannelOwnerLocalDirectCommit first attempts the same-P direct +// preparation proof without taking an endpoint admission. prepared is true +// only when that capability replaced the complete general endpoint audit; +// false with a Prepared result remains a valid common mailbox transaction. +func BeginChannelOwnerLocalDirectCommit( + out *ChannelExternalCommit, + source *ChannelOperationSource, + id OperationID, + claim *SelectClaim, + current *G, + driver *ExecutorDriver, +) (result ChannelExternalCommitBeginResult, prepared bool) { + if result, prepared, handled := beginChannelOwnerLocalDirectUnadmitted( + out, source, id, claim, current, driver, + ); handled { + return result, prepared + } + result = beginChannelExternalCommit(out, source, id, claim, current, driver, true) + return result, result == ChannelExternalCommitBeginPrepared && out != nil && + current != nil && driver != nil && + out.ownerLocalCurrent == current && out.ownerLocalDriver == driver +} + +// beginChannelOwnerLocalDirectUnadmitted consumes the compiler's exact +// current-G capability before taking the generic producer lifetime lease. A +// direct one-case waiter has exactly one hchan node, that node has already been +// detached under its hchan lock, and its owner P cannot resolve or recycle the +// frame while another G is current on that same P. An atomic-prefix readiness +// producer may still race; Commit upgrades to the full admission protocol if +// its publication appears before the effect is recorded. +// +// handled=false means no shared state remains changed and the caller may use +// the general transaction. handled=true returns a complete public begin +// result and must not be retried through another admission in this call. +func beginChannelOwnerLocalDirectUnadmitted( + out *ChannelExternalCommit, + source *ChannelOperationSource, + id OperationID, + claim *SelectClaim, + current *G, + driver *ExecutorDriver, +) (result ChannelExternalCommitBeginResult, prepared, handled bool) { + if !availableChannelExternalCommitOutput(out) || source == nil || !id.Valid() || claim == nil || + current == nil || driver == nil || driver.magic != executorDriverMagic || + driver.state != executorDriverActive || driver.p == nil || driver.p.executor != driver || + driver.p.current != current || current.runP != driver.p || + source != driver.sources.channel || source.owner != driver.p || + source.route != driver.route || id.Route() != driver.route { + return ChannelExternalCommitBeginInvalid, false, false + } + slot, ok := channelOperationSlotFor(source, id) + if !ok || preemptLoad(&slot.generation) != id.Generation || + preemptLoad(&slot.state) != uint32(producerSourceActive) || + preemptLoad(&slot.external) != uint32(channelExternalExposed) || slot.claim != claim { + return ChannelExternalCommitBeginInvalid, false, false + } + switch state := selectClaimOwnerAcquire(claim); state { + case selectClaimOpen: + case selectClaimAcquiring, selectClaimCommitting, selectClaimContended: + return ChannelExternalCommitBeginClaimContended, false, true + case selectClaimClaimed: + return ChannelExternalCommitBeginClaimResolved, false, true + default: + return ChannelExternalCommitBeginInvariantFailure, false, true + } + *out = ChannelExternalCommit{ + self: out, + endpoint: channelExternalCommitAdmission{ + source: source, + slot: slot, + id: id, + }, + claim: claim, + phase: channelExternalCommitPairPrepared, + ownerLocalUnadmitted: true, + ownerLocalAdmission: ownerLocalCompletionRejected, + } + if out.prepareOwnerLocalDirectAfterSource(current, driver, source, slot, id) { + return ChannelExternalCommitBeginPrepared, true, true + } + if !selectClaimOwnerReleasePending(claim) { + out.phase = channelExternalCommitPairBroken + return ChannelExternalCommitBeginInvariantFailure, false, true + } + *out = ChannelExternalCommit{} + return ChannelExternalCommitBeginInvalid, false, false +} + +func beginChannelExternalCommit( + out *ChannelExternalCommit, + source *ChannelOperationSource, + id OperationID, + claim *SelectClaim, + current *G, + driver *ExecutorDriver, + tryOwnerLocal bool, +) ChannelExternalCommitBeginResult { + if !availableChannelExternalCommitOutput(out) || source == nil || !id.Valid() || claim == nil { return ChannelExternalCommitBeginInvalid } endpoint, acquired := source.acquireExternalCommit(id) @@ -1440,6 +1753,9 @@ func BeginChannelExternalCommit( out.phase = channelExternalCommitPairBroken return ChannelExternalCommitBeginInvariantFailure } + if tryOwnerLocal && out.prepareOwnerLocalDirectHeld(current, driver) { + return ChannelExternalCommitBeginPrepared + } if !validChannelExternalEndpointHeld(&out.endpoint, claim) { if !out.Abort() { return ChannelExternalCommitBeginInvariantFailure @@ -1464,6 +1780,253 @@ func (transaction *ChannelExternalCommit) BeginEffect() bool { return true } +// ChannelOwnerLocalCommitResult reports whether an already prepared direct +// same-P completion was committed, remained untouched for the common durable +// source path, or encountered an invariant failure after the physical effect. +type ChannelOwnerLocalCommitResult uint8 + +const ( + ChannelOwnerLocalCommitInvalid ChannelOwnerLocalCommitResult = iota + ChannelOwnerLocalCommitFallback + ChannelOwnerLocalCommitted + // ChannelOwnerLocalCompletedInline means the core also materialized and + // promoted the one-case peer before returning. The typed hchan adapter must + // clear its now-recycled frame node instead of retaining the old source ID + // for a later owner-local reduction. + ChannelOwnerLocalCompletedInline +) + +// PrepareOwnerLocalDirect proves that the currently running G owns the exact +// executor and direct one-channel continuation named by transaction. It does +// not publish a channel result. The returned capability is valid only across +// the caller's existing NoSuspend/NoPanic hchan critical section. +// +// The fast lane intentionally accepts only an untouched physical source slot. +// A readiness producer racing after this proof first changes physical away +// from Idle; CommitOwnerLocalDirectWithResult then observes that failed CAS +// before making any source mutation and falls back to the ordinary sticky +// mailbox protocol. +func (transaction *ChannelExternalCommit) PrepareOwnerLocalDirect( + current *G, + driver *ExecutorDriver, +) bool { + return transaction.prepareOwnerLocalDirectHeld(current, driver) +} + +func (transaction *ChannelExternalCommit) prepareOwnerLocalDirectHeld( + current *G, + driver *ExecutorDriver, +) bool { + if transaction == nil || transaction.self != transaction || + transaction.phase != channelExternalCommitPairPrepared || transaction.claim == nil || + transaction.ownerLocalCurrent != nil || transaction.ownerLocalDriver != nil || + transaction.ownerLocalWait != nil || + transaction.ownerLocalAdmission != ownerLocalCompletionRejected || + current == nil || driver == nil || driver.magic != executorDriverMagic || + driver.state != executorDriverActive || driver.p == nil || driver.p.executor != driver || + driver.p.current != current || current.runP != driver.p { + return false + } + endpoint := &transaction.endpoint + source, slot, id := endpoint.source, endpoint.slot, endpoint.id + if source == nil || slot == nil || !id.Valid() || source != driver.sources.channel || + source.owner != driver.p || source.route != driver.route || id.Route() != driver.route { + return false + } + return transaction.prepareOwnerLocalDirectAfterSource(current, driver, source, slot, id) +} + +// prepareOwnerLocalDirectAfterSource is the shared proof tail after the caller +// has authenticated transaction/current/driver and the exact source slot. The +// compiler-task ingress already owns that capability; the generic transaction +// reaches it through prepareOwnerLocalDirectHeld. Keeping source-independent +// scheduler and frozen-frame checks here prevents those paths from diverging. +func (transaction *ChannelExternalCommit) prepareOwnerLocalDirectAfterSource( + current *G, + driver *ExecutorDriver, + source *ChannelOperationSource, + slot *channelOperationSlot, + id OperationID, +) bool { + if preemptLoad(&slot.physical) != uint32(channelPhysicalIdle) || + preemptLoad(&slot.mailbox) != uint32(channelMailboxEmpty) { + return false + } + ready, readyOK := channelOperationReadyAt(source, id.LocalSlot()-1) + record, wait := &slot.record, slot.record.link.wait + plan, planOK := directChannelBoundResumeHeader(wait) + // The current-P capability keeps the owner resolver excluded after claim + // acquisition. validCommittedDirectChannelPark audited the complete + // ParkState/record/link/cleanup graph before this waiter became active, and + // only this same P can mutate its owner-only suffix. Check the source-facing + // identities which select this transaction; atomic producer state remains + // checked separately before and after the physical effect. + if !readyOK || ready || wait == nil || !planOK || + plan.claim != transaction.claim || plan.entries == nil || *(*OperationID)(plan.entries) != id || + record.id != id || record.phase != operationActive || record.link.wait != wait { + return false + } + admission := ownerLocalDirectCompletionAdmissionForCurrent(driver, wait) + if admission == ownerLocalCompletionRejected { + admission = ownerLocalCompletionAdmissionForCurrent(driver, wait) + } + if admission == ownerLocalCompletionRejected || !RequestPreempt(current) { + return false + } + transaction.ownerLocalCurrent = current + transaction.ownerLocalDriver = driver + transaction.ownerLocalWait = wait + transaction.ownerLocalAdmission = admission + return true +} + +func clearChannelExternalCommitOwnerLocal(transaction *ChannelExternalCommit) { + transaction.ownerLocalCurrent = nil + transaction.ownerLocalDriver = nil + transaction.ownerLocalWait = nil + transaction.ownerLocalAdmission = ownerLocalCompletionRejected +} + +// acquireChannelExternalCommitFallback upgrades an owner-local transaction +// only when a concurrent atomic-prefix producer made the mailbox protocol +// necessary. Claim ownership and the current P keep the frame stable while the +// ordinary admission is acquired; no physical source effect has been +// published by this transaction yet. +func acquireChannelExternalCommitFallback(transaction *ChannelExternalCommit) bool { + if transaction == nil || transaction.self != transaction || !transaction.ownerLocalUnadmitted || + transaction.claim == nil || transaction.endpoint.held || transaction.endpoint.posted || + transaction.endpoint.broken { + return false + } + source, slot, id := transaction.endpoint.source, transaction.endpoint.slot, transaction.endpoint.id + if source == nil || slot == nil || !id.Valid() || slot.claim != transaction.claim { + return false + } + endpoint, acquired := source.acquireExternalCommit(id) + if acquired != channelExternalCommitAcquired { + return false + } + if endpoint.slot != slot || endpoint.source != source || endpoint.id != id || + endpoint.slot.claim != transaction.claim { + _ = endpoint.releaseWithoutCommit() + return false + } + transaction.endpoint = endpoint + transaction.ownerLocalUnadmitted = false + return true +} + +// CommitOwnerLocalDirectWithResult publishes one direct result without +// round-tripping through the external mailbox, ready leaf/page, pending +// summary, and subsequent drain protocol. The exact ordinary one-case shape +// may be resolved and promoted inline; every other valid shape enters the +// scheduler-owned completion FIFO. The peer is never executed while the hchan +// lock may still be held. +func (transaction *ChannelExternalCommit) CommitOwnerLocalDirectWithResult( + small uint8, +) ChannelOwnerLocalCommitResult { + if transaction == nil || transaction.self != transaction || + transaction.phase != channelExternalCommitPairEffect || transaction.claim == nil || + small == ResumeSmallInvalid || transaction.ownerLocalCurrent == nil || + transaction.ownerLocalDriver == nil || transaction.ownerLocalWait == nil || + transaction.ownerLocalAdmission == ownerLocalCompletionRejected { + return ChannelOwnerLocalCommitInvalid + } + current, driver, wait := transaction.ownerLocalCurrent, + transaction.ownerLocalDriver, transaction.ownerLocalWait + endpoint := &transaction.endpoint + source, slot, id := endpoint.source, endpoint.slot, endpoint.id + if driver.p == nil || driver.p.executor != driver || driver.p.current != current || + current.runP != driver.p || source == nil || slot == nil || + source != driver.sources.channel || source.owner != driver.p || + source.route != driver.route || id.Route() != driver.route || + slot.record.link.wait != wait { + transaction.phase = channelExternalCommitPairBroken + return ChannelOwnerLocalCommitInvalid + } + ready, readyOK := channelOperationReadyAt(source, id.LocalSlot()-1) + if !readyOK { + transaction.phase = channelExternalCommitPairBroken + return ChannelOwnerLocalCommitInvalid + } + if ready || preemptLoad(&slot.mailbox) != uint32(channelMailboxEmpty) { + if transaction.ownerLocalUnadmitted && !acquireChannelExternalCommitFallback(transaction) { + transaction.phase = channelExternalCommitPairBroken + return ChannelOwnerLocalCommitInvalid + } + clearChannelExternalCommitOwnerLocal(transaction) + return ChannelOwnerLocalCommitFallback + } + committed, committedOK := makeChannelPhysicalWord( + channelPhysicalCommitted, + driver.route, + small, + ) + if !committedOK { + transaction.phase = channelExternalCommitPairBroken + return ChannelOwnerLocalCommitInvalid + } + if !preemptCompareAndSwap(&slot.physical, uint32(channelPhysicalIdle), committed) { + physical := channelPhysicalStateOf(preemptLoad(&slot.physical)) + if physical == channelPhysicalReady || physical == channelPhysicalRetryBudget { + if transaction.ownerLocalUnadmitted && !acquireChannelExternalCommitFallback(transaction) { + transaction.phase = channelExternalCommitPairBroken + return ChannelOwnerLocalCommitInvalid + } + clearChannelExternalCommitOwnerLocal(transaction) + return ChannelOwnerLocalCommitFallback + } + transaction.phase = channelExternalCommitPairBroken + return ChannelOwnerLocalCommitInvalid + } + // From the successful Idle -> Committed CAS onward, PostReady can only + // observe Duplicate and cannot mutate the empty mailbox or ready bitmap. + // Every remaining transition was prevalidated while the select claim and + // current-P capability excluded owner resolution. A generic transaction + // additionally retains its producer lifetime admission through Claimed. + releaseAdmission := !transaction.ownerLocalUnadmitted + if releaseAdmission { + endpoint.posted = true + } + if PublishExternallyCommittedReadyThenCandidate(&slot.record, id) != OperationCompletionPublished || + !publishExternalSelectClaim(transaction.claim) { + transaction.phase = channelExternalCommitPairBroken + return ChannelOwnerLocalCommitInvalid + } + // Seal producer ingress before releasing an ordinary endpoint admission or + // touching the parked frame. This closes the narrow Duplicate-producer + // window between the physical commit and inline ApplyOne: an already + // admitted producer is joined by the owner-local fallback, and no new one + // can enter after this point. + closeResult := source.beginCloseSlot(driver.p, id) + if (closeResult != ChannelOperationCloseStarted && closeResult != ChannelOperationAlreadyClosing && + closeResult != ChannelOperationAlreadyQuiesced) || + releaseAdmission && !endpoint.releaseCommitted() { + transaction.phase = channelExternalCommitPairBroken + return ChannelOwnerLocalCommitInvalid + } + handled, inlineOK := completeOwnerLocalDirectChannelInline( + driver, + wait, + transaction.ownerLocalAdmission, + source, + slot, + id, + small, + ) + if !inlineOK { + transaction.phase = channelExternalCommitPairBroken + return ChannelOwnerLocalCommitInvalid + } + if handled { + *transaction = ChannelExternalCommit{} + return ChannelOwnerLocalCompletedInline + } + appendOwnerLocalCompletionUnchecked(driver, wait, transaction.ownerLocalAdmission) + *transaction = ChannelExternalCommit{} + return ChannelOwnerLocalCommitted +} + // Abort releases a Prepared transaction in claim-before-admission order. func (transaction *ChannelExternalCommit) Abort() bool { if transaction == nil || transaction.self != transaction || @@ -1488,12 +2051,29 @@ func (transaction *ChannelExternalCommit) Commit() bool { // does not participate in the transaction proof and can be ignored by every // single-owner target. func (transaction *ChannelExternalCommit) CommitAtRoute(route RouteID) bool { + return transaction.commitAtRouteWithResult(route, ResumeSmallInvalid) +} + +// CommitAtRouteWithResult is the runtime-facing direct result publication. +// A zero result is reserved for the general typed cleanup path. +func (transaction *ChannelExternalCommit) CommitAtRouteWithResult( + route RouteID, + small uint8, +) bool { + return small != ResumeSmallInvalid && transaction.commitAtRouteWithResult(route, small) +} + +func (transaction *ChannelExternalCommit) commitAtRouteWithResult(route RouteID, small uint8) bool { if transaction == nil || transaction.self != transaction || transaction.phase != channelExternalCommitPairEffect || transaction.claim == nil || route != 0 && !route.Valid() { return false } - if transaction.endpoint.publishExternallyCommittedAtRoute(route) != ChannelOperationPosted || + if transaction.ownerLocalUnadmitted && !acquireChannelExternalCommitFallback(transaction) { + transaction.phase = channelExternalCommitPairBroken + return false + } + if transaction.endpoint.publishExternallyCommittedAtRoute(route, small) != ChannelOperationPosted || !publishExternalSelectClaim(transaction.claim) || !transaction.endpoint.releaseCommitted() { transaction.phase = channelExternalCommitPairBroken @@ -1518,11 +2098,32 @@ func TryPublishOwnerLocalChannelCompletion( source *ChannelOperationSource, id OperationID, ) (published, ok bool) { - driver, _, route, currentOK := CurrentExecutorDriver(current) - if !currentOK || source == nil || source != driver.sources.channel || - source.owner != driver.p || source.route != route || id.Route() != route { + driver, _, _, currentOK := CurrentExecutorDriver(current) + if !currentOK { return false, true } + return TryPublishOwnerLocalChannelCompletionCurrent(current, driver, source, id) +} + +// TryPublishOwnerLocalChannelCompletionCurrent consumes the transient driver +// capability returned together with current by CurrentExecutorDriver. The +// caller must not cross a suspension between those calls. Rechecking only the +// reciprocal current/P/source identities here preserves that capability while +// avoiding a second complete managed-resume proof in every same-P hchan +// completion. +func TryPublishOwnerLocalChannelCompletionCurrent( + current *G, + driver *ExecutorDriver, + source *ChannelOperationSource, + id OperationID, +) (published, ok bool) { + if current == nil || driver == nil || source == nil || driver.magic != executorDriverMagic || + driver.state != executorDriverActive || driver.p == nil || driver.p.executor != driver || + driver.p.current != current || current.runP != driver.p || source != driver.sources.channel || + source.owner != driver.p || source.route != driver.route || id.Route() != driver.route { + return false, true + } + route := driver.route slot, slotOK := channelOperationSlotFor(source, id) if !slotOK || preemptLoad(&slot.generation) != id.Generation || producerSourceLifecycle(preemptLoad(&slot.state)) != producerSourceActive && @@ -1532,11 +2133,12 @@ func TryPublishOwnerLocalChannelCompletion( record, claim := &slot.record, slot.claim wait := record.link.wait completionRoute, committed := channelPhysicalCompletionRoute(preemptLoad(&slot.physical)) + localAdmission := ownerLocalCompletionAdmissionFor(driver, wait) // Current operations which have not yet become active, an already queued // wait, and a route-zero/cross-owner completion are valid external-path // cases. Do not disturb their sticky mailbox or pending bit. if !committed || completionRoute != route || wait == nil || - !canAppendOwnerLocalCompletion(driver, wait) { + localAdmission == ownerLocalCompletionRejected { return false, true } ready, readyOK := channelOperationReadyAt(source, id.LocalSlot()-1) @@ -1587,7 +2189,7 @@ func TryPublishOwnerLocalChannelCompletion( _ = restoreChannelMailboxDrain(source, slot, channelMailboxForced) return false, false } - appendOwnerLocalCompletionUnchecked(driver, wait) + appendOwnerLocalCompletionUnchecked(driver, wait, localAdmission) if !finishChannelMailboxDrain(source, slot, channelMailboxForced) || !refreshChannelOperationReadyPage(source, page) { return false, false @@ -1602,6 +2204,7 @@ func (source *ChannelOperationSource) publishExternallyCommittedHeld( slot *channelOperationSlot, id OperationID, route RouteID, + small uint8, ) ChannelOperationPostResult { if preemptLoad(&slot.generation) != id.Generation || preemptLoad(&slot.external) != uint32(channelExternalExposed) || route != 0 && !route.Valid() { @@ -1611,7 +2214,7 @@ func (source *ChannelOperationSource) publishExternallyCommittedHeld( if state != producerSourceActive && state != producerSourceClosing { return ChannelOperationPostClosed } - committed, committedOK := makeChannelPhysicalWord(channelPhysicalCommitted, route) + committed, committedOK := makeChannelPhysicalWord(channelPhysicalCommitted, route, small) if !committedOK { return ChannelOperationPostInvalid } @@ -1986,32 +2589,26 @@ func (source *ChannelOperationSource) BeginClose(p *P, id OperationID) ChannelOp return source.beginCloseSlot(p, id) } -// ApplyOne seals producer ingress and joins every admitted frame access before -// detach. A source-only Ready producer touches just the atomic prefix, but an -// external hchan transaction uses this same admission as the lifetime lease -// for its queue-node claim pointer. ConfirmQuiesced remains the later backend -// strong join and physical cleanup boundary, mirroring ManualOperationSource. -func (source *ChannelOperationSource) ApplyOne(p *P, id OperationID, record *OperationRecord) OperationApplyResult { - slot, ok := channelOperationSlotFor(source, id) - if !ok || !validChannelOperationOwner(source, p) || preemptLoad(&slot.generation) != id.Generation || - &slot.record != record || !record.Matches(id) || record.phase != operationActive { - return OperationApplyInvalid - } - disposition, terminal := OperationDispositionOf(record, id) - if !terminal || record.link.park == nil || record.link.wait == nil || record.link.operation != record || - record.link.ticket == (ParkTicket{}) { - return OperationApplyInvalid - } - closeResult := source.beginCloseSlot(p, id) - if closeResult != ChannelOperationCloseStarted && closeResult != ChannelOperationAlreadyClosing && - closeResult != ChannelOperationAlreadyQuiesced { - return OperationApplyInvalid - } +// applyClosedChannelOperationSlot is the shared detach tail after producer +// ingress has been sealed. The ordinary source reducer enters through +// ApplyOne; an exact owner-local direct commit consumes its already-closed +// capability without repeating slot lookup, owner proof, or close. +func applyClosedChannelOperationSlot( + slot *channelOperationSlot, + id OperationID, + record *OperationRecord, + disposition OperationDisposition, +) OperationApplyResult { // Admission covers every hchan access to the frame-local claim, not just // atomic source fields. A held external transaction may have published its // mailbox but not yet stored both claims Claimed. Never acknowledge, detach, // clear the claim pointer, or make the G promotable until the sealed source // joins that transaction. + if slot == nil || preemptLoad(&slot.generation) != id.Generation || + preemptLoad(&slot.state) != uint32(producerSourceClosing) || + &slot.record != record || !record.Matches(id) || record.phase != operationActive { + return OperationApplyInvalid + } if !producerSourceSlotQuiesced(&slot.producerSourceSlot) { return OperationApplyRetryBudget } @@ -2045,12 +2642,35 @@ func (source *ChannelOperationSource) ApplyOne(p *P, id OperationID, record *Ope return OperationApplyDetached } +// ApplyOne seals producer ingress and joins every admitted frame access before +// detach. A source-only Ready producer touches just the atomic prefix, but an +// external hchan transaction uses this same admission as the lifetime lease +// for its queue-node claim pointer. ConfirmQuiesced remains the later backend +// strong join and physical cleanup boundary, mirroring ManualOperationSource. +func (source *ChannelOperationSource) ApplyOne(p *P, id OperationID, record *OperationRecord) OperationApplyResult { + slot, ok := channelOperationSlotFor(source, id) + if !ok || !validChannelOperationOwner(source, p) || preemptLoad(&slot.generation) != id.Generation || + &slot.record != record || !record.Matches(id) || record.phase != operationActive { + return OperationApplyInvalid + } + disposition, terminal := OperationDispositionOf(record, id) + if !terminal || record.link.park == nil || record.link.wait == nil || record.link.operation != record || + record.link.ticket == (ParkTicket{}) { + return OperationApplyInvalid + } + closeResult := source.beginCloseSlot(p, id) + if closeResult != ChannelOperationCloseStarted && closeResult != ChannelOperationAlreadyClosing && + closeResult != ChannelOperationAlreadyQuiesced { + return OperationApplyInvalid + } + return applyClosedChannelOperationSlot(slot, id, record, disposition) +} + // ConfirmQuiesced accepts the hchan/backend strong join. The admission word // additionally proves every source shim admitted before Apply's seal returned; // a late sticky mailbox must first be classified by a later publish epoch. -func (source *ChannelOperationSource) ConfirmQuiesced(p *P, id OperationID) bool { - slot, ok := channelOperationSlotFor(source, id) - if !ok || !validChannelOperationOwner(source, p) || preemptLoad(&slot.generation) != id.Generation || +func confirmChannelOperationQuiescedSlot(slot *channelOperationSlot, id OperationID) bool { + if slot == nil || preemptLoad(&slot.generation) != id.Generation || preemptLoad(&slot.state) != uint32(producerSourceClosing) || !producerSourceSlotQuiesced(&slot.producerSourceSlot) || preemptLoad(&slot.mailbox) != uint32(channelMailboxEmpty) || slot.claim != nil || preemptLoad(&slot.externalLease)&1 != 0 || @@ -2095,6 +2715,12 @@ func (source *ChannelOperationSource) ConfirmQuiesced(p *P, id OperationID) bool return markProducerSourceQuiesced(&slot.producerSourceSlot) } +func (source *ChannelOperationSource) ConfirmQuiesced(p *P, id OperationID) bool { + slot, ok := channelOperationSlotFor(source, id) + return ok && validChannelOperationOwner(source, p) && + confirmChannelOperationQuiescedSlot(slot, id) +} + // ResetSelectClaim is the resume/compiler-owner reuse boundary. A select claim // remains Claimed through logical resolution and every Channel detach; it may // return to Open only after this source no longer retains that frame pointer. @@ -2116,14 +2742,29 @@ func (source *ChannelOperationSource) ResetSelectClaim(p *P, claim *SelectClaim) return preemptCompareAndSwap(&claim.state, selectClaimClaimed, selectClaimOpen) } +// ResetSelectClaimAfterConfirmed is the typed-cleanup counterpart of +// ResetSelectClaim. The caller has already confirmed every frozen operation +// ID in the wait-set, and ConfirmQuiesced clears each exact slot.claim only +// after producer admission reaches zero. The binding-time ID audit plus this +// per-generation confirmation is a stronger, O(case count) certificate than +// rescanning the source's configured catalog here. +func (source *ChannelOperationSource) ResetSelectClaimAfterConfirmed( + p *P, + claim *SelectClaim, + confirmed uint32, + want uint32, +) bool { + return validChannelOperationOwner(source, p) && p.channelSource == source && + claim != nil && confirmed == want && want != 0 && + preemptCompareAndSwap(&claim.state, selectClaimClaimed, selectClaimOpen) +} + // CompletionRoute returns the optional producer-locality hint retained by one // terminal winning channel operation. The source owner may inspect it only // after detach and before Recycle clears the phase-overlaid physical word. // Route zero is a valid answer and means that no migration preference exists. -func (source *ChannelOperationSource) CompletionRoute(p *P, id OperationID) (RouteID, bool) { - slot, ok := channelOperationSlotFor(source, id) - if !ok || !validChannelOperationOwner(source, p) || - preemptLoad(&slot.generation) != id.Generation || !slot.record.Matches(id) || +func channelOperationCompletionRouteSlot(slot *channelOperationSlot, id OperationID) (RouteID, bool) { + if slot == nil || preemptLoad(&slot.generation) != id.Generation || !slot.record.Matches(id) || slot.record.phase != operationDetached || !slot.record.resolutionApplied { return 0, false } @@ -2134,13 +2775,27 @@ func (source *ChannelOperationSource) CompletionRoute(p *P, id OperationID) (Rou return channelPhysicalCompletionRoute(preemptLoad(&slot.physical)) } +func (source *ChannelOperationSource) CompletionRoute(p *P, id OperationID) (RouteID, bool) { + slot, ok := channelOperationSlotFor(source, id) + if !ok || !validChannelOperationOwner(source, p) { + return 0, false + } + return channelOperationCompletionRouteSlot(slot, id) +} + +func takeChannelOperationResultSlot(slot *channelOperationSlot, lease OperationResultLease) bool { + id, ok := lease.ID() + return ok && slot != nil && preemptLoad(&slot.generation) == id.Generation && + TakeOperationResult(&slot.record, lease) +} + func (source *ChannelOperationSource) TakeResult(p *P, lease OperationResultLease) bool { id, ok := lease.ID() if !ok || !validChannelOperationOwner(source, p) { return false } slot, ok := channelOperationSlotFor(source, id) - return ok && preemptLoad(&slot.generation) == id.Generation && TakeOperationResult(&slot.record, lease) + return ok && takeChannelOperationResultSlot(slot, lease) } func (source *ChannelOperationSource) DiscardResult(p *P, lease OperationResultLease) bool { @@ -2152,10 +2807,9 @@ func (source *ChannelOperationSource) DiscardResult(p *P, lease OperationResultL return ok && preemptLoad(&slot.generation) == id.Generation && DiscardOperationResult(&slot.record, lease) } -func (source *ChannelOperationSource) Recycle(p *P, id OperationID) bool { - slot, ok := channelOperationSlotFor(source, id) +func recycleChannelOperationSlot(source *ChannelOperationSource, slot *channelOperationSlot, id OperationID) bool { ready, readyOK := channelOperationReadyAt(source, id.LocalSlot()-1) - if !ok || !validChannelOperationOwner(source, p) || preemptLoad(&slot.generation) != id.Generation || + if slot == nil || preemptLoad(&slot.generation) != id.Generation || preemptLoad(&slot.state) != uint32(producerSourceQuiesced) || !producerSourceSlotQuiesced(&slot.producerSourceSlot) || preemptLoad(&slot.mailbox) != uint32(channelMailboxEmpty) || preemptLoad(&slot.external) != 0 || preemptLoad(&slot.externalLease)&1 != 0 || slot.claim != nil || !readyOK || ready || @@ -2180,6 +2834,42 @@ func (source *ChannelOperationSource) Recycle(p *P, id OperationID) bool { return true } +func (source *ChannelOperationSource) Recycle(p *P, id OperationID) bool { + slot, ok := channelOperationSlotFor(source, id) + return ok && validChannelOperationOwner(source, p) && + recycleChannelOperationSlot(source, slot, id) +} + +// finishOwnerLocalDirectChannelResult collapses the confirmed one-channel +// result tail while retaining the same source lifecycle gates. The direct +// cleanup cursor has already authenticated this generation and descriptor; +// resolve the stable slot once, then confirm, release the private claim, take +// the exact result lease, read its locality hint, and recycle in that order. +// Generic select continues to use the independently bounded operations above. +func (source *ChannelOperationSource) finishOwnerLocalDirectChannelResult( + p *P, + slot *channelOperationSlot, + id OperationID, + claim *SelectClaim, + lease OperationResultLease, +) (RouteID, bool) { + leaseID, leaseOK := lease.ID() + if slot == nil || !leaseOK || leaseID != id || claim == nil || + !validChannelOperationOwner(source, p) || + preemptLoad(&slot.generation) != id.Generation || + !confirmChannelOperationQuiescedSlot(slot, id) || + !preemptCompareAndSwap(&claim.state, selectClaimClaimed, selectClaimOpen) || + !takeChannelOperationResultSlot(slot, lease) { + return 0, false + } + preferred, routeOK := channelOperationCompletionRouteSlot(slot, id) + if !routeOK || preferred != 0 && !preferred.Valid() || + !recycleChannelOperationSlot(source, slot, id) { + return 0, false + } + return preferred, true +} + func channelOperationSourceEmpty(source *ChannelOperationSource, owner *P) bool { if source == nil || !routedProducerHeaderEmpty(&source.routedProducerSource, owner) || !channelOperationReadyCatalogEmpty(source) { diff --git a/runtime/internal/coro/channel_park_owner.go b/runtime/internal/coro/channel_park_owner.go index 6fae32e23c..36bfd95113 100644 --- a/runtime/internal/coro/channel_park_owner.go +++ b/runtime/internal/coro/channel_park_owner.go @@ -33,6 +33,235 @@ func CurrentExecutorChannelDriver(g *G) (*ExecutorDriver, ExecutorHandle, RouteI return driver, handle, route, true } +// CurrentExecutorChannelParkContext resolves the complete transient channel +// preparation capability in one authenticated owner lookup. Keeping the +// driver, P, ParkState, and source in one return prevents typed adapters from +// proving the same SuspendPark/FrameSuspended relation a second time through +// CurrentExecutorChannelParkOwner immediately afterwards. +func CurrentExecutorChannelParkContext( + g *G, +) (*ExecutorDriver, ExecutorHandle, RouteID, *P, *ParkState, *ChannelOperationSource, bool) { + driver, handle, route, ok := CurrentExecutorChannelDriver(g) + if !ok { + return nil, ExecutorHandle{}, 0, nil, nil, nil, false + } + return driver, handle, route, driver.p, &g.park, driver.sources.channel, true +} + +// CurrentExecutorChannelDirectReservation resolves the current direct-channel +// owner and selects its reusable slot under the same authenticated boundary. +// current distinguishes an invalid G/P/source relation from ordinary catalog +// exhaustion; the typed adapter may grow stable source storage only for the +// latter, then retry through the general owner-audited preflight. +func CurrentExecutorChannelDirectReservation( + g *G, +) ( + route RouteID, + p *P, + source *ChannelOperationSource, + reservation ChannelDirectReservation, + current bool, + reserved bool, +) { + driver, _, route, ok := CurrentExecutorChannelDriver(g) + if !ok { + return 0, nil, nil, ChannelDirectReservation{}, false, false + } + p, source = driver.p, driver.sources.channel + reservation, reserved = source.preflightDirectReservationOwned(p) + return route, p, source, reservation, true, reserved +} + +type CurrentChannelParkPreparationState uint8 + +const ( + CurrentChannelParkPreparationInvalid CurrentChannelParkPreparationState = iota + CurrentChannelParkPreparationNeedsCapacity + CurrentChannelParkPreparationPrepared +) + +// CurrentChannelParkPreparation is the closed result of one compiler-owned +// direct-channel park transaction. NeedsCapacity carries only the authenticated +// owner/source pair so the runtime can attach stable source storage and retry +// without publishing a waiter. Prepared additionally carries the exact ticket +// and operation which were bound into the frame-local resume packet. +type CurrentChannelParkPreparation struct { + Owner *P + Source *ChannelOperationSource + Ticket ParkTicket + Operation OperationID + Route RouteID + State CurrentChannelParkPreparationState +} + +// PrepareCurrentChannelParkCleanup performs the generated one-channel park as +// one owner transaction. The compiler supplies the exact G/frame header and +// zero-filled frame storage; this function authenticates that boundary once, +// reserves and attaches one source slot, publishes the parked frame, and binds +// its cleanup packet without immediately re-walking the graph it just built. +// +// reservation is consumed only when hasReservation is true. A first call with +// no reusable slot returns NeedsCapacity without mutating scheduler or frame +// state; the runtime may grow stable storage, obtain a reservation, and retry +// in the same no-suspend interval. +func PrepareCurrentChannelParkCleanup( + g *G, + handle unsafe.Pointer, + header *HeaderV1, + wait *WaitSetRecord, + claim *SelectClaim, + packet *ResumePacket, + plan *ResumeCleanupPlan, + context unsafe.Pointer, + entry *OperationID, + reservation ChannelDirectReservation, + hasReservation bool, + caseID uint32, + seed uint32, +) CurrentChannelParkPreparation { + driver, _, route, current := currentExecutorParkDriver(g) + if !current || driver.sources.channel == nil || + !validChannelOperationOwner(driver.sources.channel, driver.p) || + driver.sources.channel.route != route { + return CurrentChannelParkPreparation{} + } + p, source, frame := driver.p, driver.sources.channel, g.active + base := CurrentChannelParkPreparation{Owner: p, Source: source, Route: route} + if frame == nil || frame.handle != handle || frame.header != header || + wait == nil || claim == nil || packet == nil || plan == nil || context == nil || entry == nil || + *wait != (WaitSetRecord{}) || *claim != (SelectClaim{}) || + *packet != (ResumePacket{}) || *plan != (ResumeCleanupPlan{}) || + *entry != (OperationID{}) || caseID == 0 { + return CurrentChannelParkPreparation{} + } + if hasReservation { + if !validDirectReservationHeader(source, p, reservation) { + return CurrentChannelParkPreparation{} + } + } else { + var reserved bool + reservation, reserved = source.preflightDirectReservationOwned(p) + if !reserved { + base.State = CurrentChannelParkPreparationNeedsCapacity + return base + } + } + + // A stop which won before this no-suspend interval retains the generic + // cancellation-aware transaction. It is cold and may repeat its defensive + // proof; the ordinary compiler path below owns the compact certificate. + if g.park.taskCancelKind != TaskCancelNone || g.park.taskCancelPhase != taskCancelIdle { + ticket, id, ok := prepareSingleChannelParkCleanup( + g, handle, header, source, wait, claim, packet, plan, context, entry, + &reservation, caseID, seed, + ) + if !ok || id.Route() != route { + return CurrentChannelParkPreparation{} + } + base.Ticket, base.Operation = ticket, id + base.State = CurrentChannelParkPreparationPrepared + return base + } + if g.pending.kind != pendingNone || g.spawnChild != nil || g.waiting || + !validReusableSingleParkState(&g.park) || + g.park.attached != 0 || g.park.head != nil { + return CurrentChannelParkPreparation{} + } + + ticket, ok := nextParkTicket(g.park.ticket) + if !ok { + return CurrentChannelParkPreparation{} + } + + // The private reservation was selected under this exact owner P and cannot + // be changed by another owner during the no-suspend compiler hook. Prove + // every scalar which could make OperationID construction or scan publication + // fail before sealing the physical generation. After begin succeeds, the + // remaining writes form one closed, no-fail suffix followed by the two + // release publications which make the generation and hchan endpoint visible. + slot, index, capacity := reservation.slot, reservation.index, reservation.capacity + if slot == nil || capacity != ChannelOperationConfiguredCapacity(source) || + index >= capacity || index >= operationLocalMask || source.scanLimit > capacity { + return CurrentChannelParkPreparation{} + } + previousGeneration := preemptLoad(&slot.generation) + if previousGeneration == ^uint32(0) { + return CurrentChannelParkPreparation{} + } + id, idOK := MakeOperationIDAtRoute( + OperationSourceChannel, source.route, index+1, previousGeneration+1, + ) + if !idOK { + return CurrentChannelParkPreparation{} + } + generation, begun := beginProducerSourceSlot(&slot.producerSourceSlot) + if !begun { + return CurrentChannelParkPreparation{} + } + if generation != id.Generation { + _ = resetProducerSourceSlot(&slot.producerSourceSlot, generation) + return CurrentChannelParkPreparation{} + } + + parkSeed := seed ^ ticket.generation*0x9e3779b9 ^ ticket.epoch*0x85ebca6b + record := &slot.record + *record = OperationRecord{id: id, phase: operationActive} + setOperationCandidate(record, OperationCommitReadyThenTryCommit, OperationCommitIdle, false) + record.link = ParkLink{ + park: &g.park, + wait: wait, + operation: record, + ticket: ticket, + caseID: caseID, + rank: parkCaseRank(parkSeed, caseID), + } + g.park = ParkState{ + ticket: ticket, + phase: parkParked, + expected: 1, + attached: 1, + head: &record.link, + } + *wait = WaitSetRecord{ + g: g, + ticket: ticket, + state: waitSetRecordCommitted, + directChannel: true, + } + *entry = id + installWaitSetResumeCleanup(wait, packet, plan, ResumeCleanupBinding{ + Kind: ResumeCleanupChannelDirect, + Context: context, + Entries: unsafe.Pointer(entry), + Claim: claim, + Count: 1, + RuntimeCount: 1, + Stride: unsafe.Sizeof(OperationID{}), + }) + frame.parkWait = wait + g.pending = pendingTransition{kind: pendingParkSet, from: frame} + slot.claim = claim + preemptStore(&slot.external, uint32(channelExternalReserved)) + if next := index + 1; next > source.scanLimit { + source.scanLimit = next + } + source.reserveCursor = index + 1 + if source.reserveCursor == capacity { + source.reserveCursor = 0 + } + if !activateProducerSourceSlot(&slot.producerSourceSlot, generation) || + !preemptCompareAndSwap( + &slot.external, + uint32(channelExternalReserved), + uint32(channelExternalExposed), + ) { + return CurrentChannelParkPreparation{} + } + base.Ticket, base.Operation = ticket, id + base.State = CurrentChannelParkPreparationPrepared + return base +} + // CurrentExecutorChannelParkOwner returns transient owner-only objects needed // by the typed hchan adapter to build a direct or multi-case park. None of the // returned pointers may cross suspension except the ChannelOperationSource @@ -83,6 +312,327 @@ func PrepareSingleChannelPark( claim *SelectClaim, caseID uint32, seed uint32, +) (ParkTicket, OperationID, bool) { + // The ordinary compiler-generated direct park is a one-candidate + // transaction, not a degenerate select. Build its exact relation in one + // owner-only interval so generic Seal/sort/Prepare/Commit layers do not each + // re-audit the same single ParkLink. A pending task stop retains the generic + // path below because CommitParkSet must translate it into logical cancel. + if g != nil && g.park.taskCancelKind == TaskCancelNone && + g.park.taskCancelPhase == taskCancelIdle { + return prepareSingleChannelParkOrdinary( + g, handle, header, source, wait, claim, nil, caseID, seed, + ) + } + return prepareSingleChannelParkGeneric( + g, handle, header, source, wait, claim, caseID, seed, + ) +} + +// PrepareSingleChannelParkCleanup is the compiler/runtime direct-channel ABI. +// It combines the one-candidate logical/source preparation with installation +// of its frame-local typed cleanup descriptor. entry, packet, and plan are +// stable fields in the same LLVM coroutine frame as wait and claim; no pointer +// escapes to a producer. The public general BindWaitSetResumeCleanup remains +// available for select and independently assembled source sets. +func PrepareSingleChannelParkCleanup( + g *G, + handle unsafe.Pointer, + header *HeaderV1, + source *ChannelOperationSource, + wait *WaitSetRecord, + claim *SelectClaim, + packet *ResumePacket, + plan *ResumeCleanupPlan, + context unsafe.Pointer, + entry *OperationID, + caseID uint32, + seed uint32, +) (ParkTicket, OperationID, bool) { + return prepareSingleChannelParkCleanup( + g, handle, header, source, wait, claim, packet, plan, context, entry, + nil, caseID, seed, + ) +} + +// PrepareSingleChannelParkCleanupReserved consumes the exact owner-only slot +// capability returned by PreflightDirectReservation. No suspension or owner +// transition may occur between the two calls. The ordinary wrapper above is +// retained for adapters which do not need to combine catalog growth with +// direct park preparation. +func PrepareSingleChannelParkCleanupReserved( + g *G, + handle unsafe.Pointer, + header *HeaderV1, + source *ChannelOperationSource, + wait *WaitSetRecord, + claim *SelectClaim, + packet *ResumePacket, + plan *ResumeCleanupPlan, + context unsafe.Pointer, + entry *OperationID, + reservation ChannelDirectReservation, + caseID uint32, + seed uint32, +) (ParkTicket, OperationID, bool) { + return prepareSingleChannelParkCleanup( + g, handle, header, source, wait, claim, packet, plan, context, entry, + &reservation, caseID, seed, + ) +} + +func prepareSingleChannelParkCleanup( + g *G, + handle unsafe.Pointer, + header *HeaderV1, + source *ChannelOperationSource, + wait *WaitSetRecord, + claim *SelectClaim, + packet *ResumePacket, + plan *ResumeCleanupPlan, + context unsafe.Pointer, + entry *OperationID, + reservation *ChannelDirectReservation, + caseID uint32, + seed uint32, +) (ParkTicket, OperationID, bool) { + if packet == nil || plan == nil || context == nil || entry == nil || + *packet != (ResumePacket{}) || *plan != (ResumeCleanupPlan{}) || + *entry != (OperationID{}) { + return ParkTicket{}, OperationID{}, false + } + var ticket ParkTicket + var id OperationID + var ok bool + if reservation != nil && g != nil && g.park.taskCancelKind == TaskCancelNone && + g.park.taskCancelPhase == taskCancelIdle { + ticket, id, ok = prepareSingleChannelParkOrdinary( + g, handle, header, source, wait, claim, reservation, caseID, seed, + ) + } else { + ticket, id, ok = PrepareSingleChannelPark( + g, handle, header, source, wait, claim, caseID, seed, + ) + } + if !ok { + return ParkTicket{}, OperationID{}, false + } + *entry = id + binding := ResumeCleanupBinding{ + Kind: ResumeCleanupChannelDirect, + Context: context, + Entries: unsafe.Pointer(entry), + Claim: claim, + Count: 1, + RuntimeCount: 1, + Stride: unsafe.Sizeof(OperationID{}), + } + // PrepareSingleChannelPark has just frozen this exact one-link relation and + // no typed hchan node is reachable yet. Recheck the scalar frame boundary, + // then install through the common no-fail write half without rescanning the + // source catalog or revalidating the same plan after construction. + if wait == nil || wait.state != waitSetRecordCommitted || wait.work != waitSetWorkIdle || + wait.resume != nil || wait.resumeKind != resumeBindingNone || wait.g != g || + wait.ticket != ticket || g == nil || g.active == nil || g.active.parkWait != wait || + g.pending.kind != pendingParkSet || g.pending.from != g.active || + g.park.phase != parkParked || g.park.ticket != ticket || g.park.expected != 1 || + g.park.attached != 1 || g.park.head == nil || g.park.head.next != nil || + g.park.head.wait != wait || g.park.head.operation == nil || + g.park.head.operation.id != id || g.park.head.caseID != caseID || + selectClaimLoad(claim) != selectClaimOpen { + return ParkTicket{}, OperationID{}, false + } + installWaitSetResumeCleanup(wait, packet, plan, binding) + wait.directChannel = true + return ticket, id, true +} + +// validCommittedDirectChannelPark is the post-llvm.coro.suspend gate for the +// fused direct preparation transaction. The full graph was audited before the +// endpoint became Exposed; between that publication and Resumed, a peer may +// change source atomics and the SelectClaim but cannot mutate this owner-only +// ParkState/OperationRecord suffix. Validate the exact scalar/pointer boundary +// once instead of immediately repeating validParkState and the cleanup plan. +func validCommittedDirectChannelPark(g *G, frame *Frame, wait *WaitSetRecord) bool { + if g == nil || frame == nil || wait == nil || !wait.directChannel || + frame.owner != g || frame.parkWait != wait || wait.g != g || + wait.state != waitSetRecordCommitted || wait.work != waitSetWorkIdle || + wait.activePrev != nil || wait.activeNext != nil || wait.workNext != nil { + return false + } + if wait.resumeKind == resumeBindingDirectChannel { + return validCommittedCompactDirectChannelPark(g, frame, wait) + } + plan := (*ResumeCleanupPlan)(wait.resume) + if !validDirectChannelBoundResumeState(wait, plan) { + return false + } + id := (*OperationID)(plan.entries) + state := &g.park + if state.ticket != wait.ticket || state.phase != parkParked || state.resolving || + state.expected != 1 || state.attached != 1 || state.seed != 0 || state.hasDefault || + state.taskCancelKind != TaskCancelNone || state.taskCancelPhase != taskCancelIdle || + state.cancelKind != ParkCancelNone || state.outcome != ParkOutcomePending || + state.winnerCase != 0 || state.winnerID != (OperationID{}) || + state.winnerRecord != nil || state.head == nil || state.head.previous != nil || + state.head.next != nil || state.head.wait != wait || state.head.park != state || + state.head.ticket != wait.ticket || state.head.operation == nil { + return false + } + record := state.head.operation + return record.id == *id && record.phase == operationActive && + record.disposition == OperationDispositionPending && !record.resolutionApplied && + !record.cancelRequested && !record.quiesced && + operationCandidateMode(record) == OperationCommitReadyThenTryCommit && + operationCandidateState(record) == OperationCommitIdle && + !operationCandidateIsPublished(record) && record.resultState == operationResultEmpty && + record.resultTicket == (ParkTicket{}) && record.link.park == state && + record.link.wait == wait && record.link.operation == record && + record.link.ticket == wait.ticket && record.link.previous == nil && record.link.next == nil +} + +// validReusableSingleParkState is the O(1) preflight for an ordinary +// compiler-owned one-event park. Idle and Delivered have no attached operation +// graph; checking them through validParkState would enter its general N-way +// link walker and large phase switch on every operation. Consumed is a +// compatibility handoff shape with outcome-dependent state and deliberately +// retains the complete validator. +func validReusableSingleParkState(state *ParkState) bool { + if state == nil || state.resolving || state.taskCancelKind != TaskCancelNone || + state.taskCancelPhase != taskCancelIdle { + return false + } + switch state.phase { + case parkIdle: + return *state == (ParkState{}) + case parkDelivered: + return validParkTicket(state.ticket) && state.expected == 0 && state.attached == 0 && + state.seed == 0 && !state.hasDefault && !state.directChannel && state.cancelKind == ParkCancelNone && + state.outcome == ParkOutcomePending && state.winnerCase == 0 && + state.winnerID == (OperationID{}) && state.winnerRecord == nil && state.head == nil + case parkConsumed: + return validParkState(state) + default: + return false + } +} + +// validPreparedDirectChannelParkState is the single-link post-build audit. +// A source has just produced this exact graph under the owner P, and no source +// endpoint is externally reachable yet. This is the complete one-link +// equivalent of validParkState, shared by direct-channel and manual parks. +func validPreparedDirectChannelParkState( + state *ParkState, + wait *WaitSetRecord, + id OperationID, + caseID uint32, +) bool { + if state == nil || wait == nil || !id.Valid() || caseID == 0 || + state.phase != parkParked || !validParkTicket(state.ticket) || state.resolving || + state.expected != 1 || state.attached != 1 || state.seed != 0 || state.hasDefault || + state.taskCancelKind != TaskCancelNone || state.taskCancelPhase != taskCancelIdle || + state.cancelKind != ParkCancelNone || state.outcome != ParkOutcomePending || + state.winnerCase != 0 || state.winnerID != (OperationID{}) || + state.winnerRecord != nil || state.head == nil || state.head.previous != nil || + state.head.next != nil || !validPreparingWaitSetRecord(wait, state, state.ticket) { + return false + } + link := state.head + record := link.operation + return link.park == state && link.wait == wait && record != nil && + link == &record.link && link.ticket == state.ticket && link.caseID == caseID && + record.id == id && record.phase == operationActive && + record.disposition == OperationDispositionPending && !record.resolutionApplied && + operationCandidatePendingResultStorageValid(record) && + operationCandidatePendingForResolution(record) +} + +func prepareSingleChannelParkOrdinary( + g *G, + handle unsafe.Pointer, + header *HeaderV1, + source *ChannelOperationSource, + wait *WaitSetRecord, + claim *SelectClaim, + prepared *ChannelDirectReservation, + caseID uint32, + seed uint32, +) (ParkTicket, OperationID, bool) { + if !ValidG(g) || handle == nil || header == nil || source == nil || wait == nil || claim == nil || + *wait != (WaitSetRecord{}) || *claim != (SelectClaim{}) || caseID == 0 || + !resumeGateTaken(g) || g.runP == nil || g.pending.kind != pendingNone || + g.spawnChild != nil || g.waiting || g.park.taskCancelKind != TaskCancelNone || + g.park.taskCancelPhase != taskCancelIdle || !validReusableSingleParkState(&g.park) || + g.park.attached != 0 || g.park.head != nil { + return ParkTicket{}, OperationID{}, false + } + p := g.runP + frame := findFrame(g, handle) + if frame == nil || frame != g.active || frame.header != header || frame.state != FrameActive || + frame.parkWait != nil || header.SuspendReason != uint16(SuspendPark) || + header.Lifecycle != uint16(FrameSuspended) { + return ParkTicket{}, OperationID{}, false + } + var reservation ChannelDirectReservation + if prepared == nil { + var reservationOK bool + reservation, reservationOK = source.PreflightDirectReservation(p) + if !reservationOK { + return ParkTicket{}, OperationID{}, false + } + } else { + reservation = *prepared + if !validDirectReservationHeader(source, p, reservation) { + return ParkTicket{}, OperationID{}, false + } + } + ticket, ticketOK := nextParkTicket(g.park.ticket) + if !ticketOK { + return ParkTicket{}, OperationID{}, false + } + // Write the complete Preparing value only after every allocation/capacity/ + // frame preflight above has succeeded. + g.park = ParkState{ + ticket: ticket, + phase: parkPreparing, + expected: 1, + seed: seed ^ ticket.generation*0x9e3779b9 ^ ticket.epoch*0x85ebca6b, + } + wait.g = g + wait.ticket = ticket + wait.state = waitSetRecordPreparing + id, ok := source.reserveAndAttachDirectWait( + p, &g.park, ticket, wait, caseID, claim, reservation, + ) + if !ok { + return ParkTicket{}, OperationID{}, false + } + // One candidate is already sorted and unique. Freeze the preparation seed + // into the ordinary Parked visit cursor, then perform the transaction's one + // complete post-build graph audit before publishing frame/scheduler state. + g.park.seed = 0 + g.park.phase = parkParked + if !validPreparedDirectChannelParkState(&g.park, wait, id, caseID) { + return ParkTicket{}, OperationID{}, false + } + wait.state = waitSetRecordCommitted + frame.parkWait = wait + g.pending = pendingTransition{kind: pendingParkSet, from: frame} + if !source.exposeExternalCommitDirect(g, id, ticket, wait, claim, reservation) { + return ParkTicket{}, OperationID{}, false + } + return ticket, id, true +} + +func prepareSingleChannelParkGeneric( + g *G, + handle unsafe.Pointer, + header *HeaderV1, + source *ChannelOperationSource, + wait *WaitSetRecord, + claim *SelectClaim, + caseID uint32, + seed uint32, ) (ParkTicket, OperationID, bool) { if !ValidG(g) || handle == nil || header == nil || source == nil || wait == nil || claim == nil || *wait != (WaitSetRecord{}) || *claim != (SelectClaim{}) || caseID == 0 || diff --git a/runtime/internal/coro/channel_park_owner_test.go b/runtime/internal/coro/channel_park_owner_test.go index 826264d753..33ecf38399 100644 --- a/runtime/internal/coro/channel_park_owner_test.go +++ b/runtime/internal/coro/channel_park_owner_test.go @@ -16,7 +16,56 @@ package coro -import "testing" +import ( + "testing" + "unsafe" +) + +func TestDirectChannelParkStateCertificates(t *testing.T) { + if !validReusableSingleParkState(new(ParkState)) { + t.Fatal("zero idle park was not reusable") + } + ticket := ParkTicket{generation: 1} + delivered := ParkState{ticket: ticket, phase: parkDelivered} + if !validReusableSingleParkState(&delivered) { + t.Fatal("clean delivered park was not reusable") + } + delivered.expected = 1 + if validReusableSingleParkState(&delivered) { + t.Fatal("delivered park with retained expected count was reusable") + } + + id, idOK := MakeOperationID(OperationSourceChannel, 1, 1) + if !idOK { + t.Fatal("make direct certificate operation ID") + } + g := new(G) + wait := WaitSetRecord{ + g: g, ticket: ticket, state: waitSetRecordPreparing, + } + record := OperationRecord{id: id, phase: operationActive} + setOperationCandidate(&record, OperationCommitReadyThenTryCommit, OperationCommitIdle, false) + g.park = ParkState{ + ticket: ticket, phase: parkParked, expected: 1, attached: 1, + } + record.link = ParkLink{ + park: &g.park, wait: &wait, operation: &record, + ticket: ticket, caseID: 1, + } + g.park.head = &record.link + if !validPreparedDirectChannelParkState(&g.park, &wait, id, 1) { + t.Fatal("exact prepared direct park certificate was rejected") + } + record.link.previous = &record.link + if validPreparedDirectChannelParkState(&g.park, &wait, id, 1) { + t.Fatal("cyclic prepared direct park certificate was accepted") + } + record.link.previous = nil + setOperationCandidate(&record, OperationCommitReadyThenTryCommit, OperationCommitCommitted, true) + if validPreparedDirectChannelParkState(&g.park, &wait, id, 1) { + t.Fatal("pre-committed prepared direct park certificate was accepted") + } +} func TestCurrentExecutorChannelOwnerResolvesExactRouteAcrossTwoP(t *testing.T) { registry := new(ExecutorRegistry) @@ -58,6 +107,14 @@ func TestCurrentExecutorChannelOwnerResolvesExactRouteAcrossTwoP(t *testing.T) { t.Fatalf("resolve channel route %d = driver:%p handle:%+v route:%d p:%p park:%p source:%p ok:(%t,%t)", current.route, driver, handle, route, p, park, source, ok, ownerOK) } + directRoute, directP, directSource, reservation, directCurrent, reserved := + CurrentExecutorChannelDirectReservation(current.task.g) + if !directCurrent || !reserved || directRoute != current.route || + directP != current.p || directSource != current.source || + !validDirectReservationHeader(directSource, directP, reservation) { + t.Fatalf("resolve direct channel route %d = route:%d p:%p source:%p current:%t reserved:%t", + current.route, directRoute, directP, directSource, directCurrent, reserved) + } } if fixtures[0].source == fixtures[1].source || fixtures[0].handle == fixtures[1].handle { t.Fatal("channel route fixtures alias physical ownership") @@ -77,6 +134,121 @@ func TestCurrentExecutorChannelOwnerResolvesExactRouteAcrossTwoP(t *testing.T) { } } +func TestCurrentChannelParkPreparationCapacityFailureIsZeroEffect(t *testing.T) { + p := new(P) + driver := new(ExecutorDriver) + registry := new(ExecutorRegistry) + source := new(ChannelOperationSource) + handle := registerTestExecutor(t, registry) + if !BindExecutorSourceCatalog(driver, p, registry, handle, ExecutorSourceCatalog{Channel: source}) { + t.Fatal("bind capacity-gate channel executor") + } + task := newYieldingTestG(t, "channel-capacity-gate") + if !Enqueue(p, task.g) { + t.Fatal("enqueue capacity-gate channel task") + } + if g, ok := NextRunnable(p); !ok || g != task.g { + t.Fatal("dequeue capacity-gate channel task") + } + action := beginWaitTestResume(t, p, task) + task.frame.header.SuspendReason = uint16(SuspendPark) + task.frame.header.Lifecycle = uint16(FrameSuspended) + + capacity := ChannelOperationConfiguredCapacity(source) + for index := uint32(0); index < capacity; index++ { + slot, ok := channelOperationSlotAt(source, index) + if !ok { + t.Fatalf("lookup capacity-gate slot %d", index) + } + preemptStore(&slot.externalLease, ^uint32(0)-1) + } + defer func() { + for index := uint32(0); index < capacity; index++ { + slot, _ := channelOperationSlotAt(source, index) + preemptStore(&slot.externalLease, 0) + } + }() + + var wait WaitSetRecord + var claim SelectClaim + var packet ResumePacket + var plan ResumeCleanupPlan + var entry OperationID + var context byte + beforePark, beforePending := task.g.park, task.g.pending + prepared := PrepareCurrentChannelParkCleanup( + task.g, + task.handle, + task.frame.header, + &wait, + &claim, + &packet, + &plan, + unsafe.Pointer(&context), + &entry, + ChannelDirectReservation{}, + false, + 1, + 91, + ) + if prepared.State != CurrentChannelParkPreparationNeedsCapacity || + prepared.Owner != p || prepared.Source != source || prepared.Route != driver.route || + prepared.Ticket != (ParkTicket{}) || prepared.Operation != (OperationID{}) { + t.Fatalf("capacity-gate result = %+v", prepared) + } + if task.g.park != beforePark || task.g.pending != beforePending || task.g.active.parkWait != nil || + wait != (WaitSetRecord{}) || claim != (SelectClaim{}) || packet != (ResumePacket{}) || + plan != (ResumeCleanupPlan{}) || entry != (OperationID{}) { + t.Fatalf("capacity-gate preparation mutated state: park=%+v pending=%+v frameWait=%p wait=%+v claim=%+v packet=%+v plan=%+v entry=%+v", + task.g.park, task.g.pending, task.g.active.parkWait, wait, claim, packet, plan, entry) + } + + for index := uint32(0); index < capacity; index++ { + slot, _ := channelOperationSlotAt(source, index) + preemptStore(&slot.externalLease, 0) + } + reservation, reserved := source.PreflightDirectReservation(p) + if !reserved { + t.Fatal("preflight stale direct reservation") + } + beforeScan, beforeCursor := source.scanLimit, source.reserveCursor + preemptStore(&reservation.slot.state, uint32(producerSourceInitializing)) + prepared = PrepareCurrentChannelParkCleanup( + task.g, + task.handle, + task.frame.header, + &wait, + &claim, + &packet, + &plan, + unsafe.Pointer(&context), + &entry, + reservation, + true, + 1, + 91, + ) + if prepared != (CurrentChannelParkPreparation{}) || task.g.park != beforePark || + task.g.pending != beforePending || task.g.active.parkWait != nil || + wait != (WaitSetRecord{}) || claim != (SelectClaim{}) || + packet != (ResumePacket{}) || plan != (ResumeCleanupPlan{}) || + entry != (OperationID{}) || source.scanLimit != beforeScan || + source.reserveCursor != beforeCursor { + t.Fatalf("stale-reservation preparation was not zero effect: result=%+v park=%+v pending=%+v frameWait=%p wait=%+v claim=%+v packet=%+v plan=%+v entry=%+v scan=%d cursor=%d", + prepared, task.g.park, task.g.pending, task.g.active.parkWait, wait, claim, + packet, plan, entry, source.scanLimit, source.reserveCursor) + } + preemptStore(&reservation.slot.state, uint32(producerSourceFree)) + task.frame.header.SuspendReason = uint16(SuspendNone) + task.frame.header.Lifecycle = uint16(FrameActive) + yieldRunningDriverTask(t, p, task, action) + closeTestExecutorDriver(t, driver) + finishReadyDriverTasks(t, p, map[*G]*yieldingTestG{task.g: task}) + if !source.CanRelease() || !registry.CanRelease() { + t.Fatal("capacity-gate cleanup retained source/registry") + } +} + func TestSingleChannelParkOwnerTransactionAndFinish(t *testing.T) { p := new(P) driver := new(ExecutorDriver) @@ -100,13 +272,26 @@ func TestSingleChannelParkOwnerTransactionAndFinish(t *testing.T) { task.frame.header.Lifecycle = uint16(FrameSuspended) var wait WaitSetRecord var claim SelectClaim - ticket, id, ok := PrepareSingleChannelPark( + reservation, reserved := source.PreflightDirectReservation(p) + if !reserved { + t.Fatal("preflight single-channel owner reservation") + } + if !validDirectReservationHeader(source, p, reservation) { + t.Fatal("direct reservation lost its authenticated owner/slot identity") + } + malformed := reservation + malformed.index = malformed.capacity + if validDirectReservationHeader(source, p, malformed) { + t.Fatal("direct reservation accepted an out-of-catalog slot identity") + } + ticket, id, ok := prepareSingleChannelParkOrdinary( task.g, task.handle, task.frame.header, source, &wait, &claim, + &reservation, 41, 73, ) diff --git a/runtime/internal/coro/channel_single_commit_test.go b/runtime/internal/coro/channel_single_commit_test.go index 43efdcce11..c3f3fc3b7b 100644 --- a/runtime/internal/coro/channel_single_commit_test.go +++ b/runtime/internal/coro/channel_single_commit_test.go @@ -18,29 +18,35 @@ package coro import "testing" -func TestChannelPhysicalWordCarriesOnlyCommittedRouteHint(t *testing.T) { +func TestChannelPhysicalWordCarriesCommittedRouteAndResultHints(t *testing.T) { for _, state := range []channelPhysicalState{ channelPhysicalIdle, channelPhysicalReady, channelPhysicalRetryBudget, channelPhysicalCommitted, } { - word, ok := makeChannelPhysicalWord(state, 0) + word, ok := makeChannelPhysicalWord(state, 0, ResumeSmallInvalid) if !ok || word != uint32(state) || channelPhysicalStateOf(word) != state { t.Fatalf("route-zero physical word %d = (%#x,%t)", state, word, ok) } } - word, ok := makeChannelPhysicalWord(channelPhysicalCommitted, 7) + word, ok := makeChannelPhysicalWord(channelPhysicalCommitted, 7, 5) if !ok || channelPhysicalStateOf(word) != channelPhysicalCommitted { t.Fatalf("committed route word = (%#x,%t)", word, ok) } if route, valid := channelPhysicalCompletionRoute(word); !valid || route != 7 { t.Fatalf("committed route decode = (%d,%t)", route, valid) } - if _, ok := makeChannelPhysicalWord(channelPhysicalReady, 7); ok { + if route, small, valid := channelPhysicalCompletion(word); !valid || route != 7 || small != 5 { + t.Fatalf("committed completion decode = (%d,%d,%t)", route, small, valid) + } + if _, ok := makeChannelPhysicalWord(channelPhysicalReady, 7, ResumeSmallInvalid); ok { t.Fatal("non-committed physical state accepted a route hint") } - if _, ok := makeChannelPhysicalWord(channelPhysicalCommitted, OperationRouteEncodingCapacity+1); ok { + if _, ok := makeChannelPhysicalWord(channelPhysicalReady, 0, 5); ok { + t.Fatal("non-committed physical state accepted a result hint") + } + if _, ok := makeChannelPhysicalWord(channelPhysicalCommitted, OperationRouteEncodingCapacity+1, 5); ok { t.Fatal("physical word accepted an unencodable route hint") } if state := channelPhysicalStateOf(uint32(7) << channelPhysicalRouteShift); state <= channelPhysicalCommitted { @@ -101,6 +107,57 @@ func TestChannelExternalCommitSingleAbortCopyAndCommit(t *testing.T) { releaseChannelClaimCoreFixture(t, fixture, decision) } +func TestChannelExternalCommitInvalidOutputOverwritesScratchSuffix(t *testing.T) { + fixture := newChannelClaimCoreFixture(t, "channel-single-scratch-output", []uint32{97}, true, 0) + transaction := ChannelExternalCommit{ + endpoint: channelExternalCommitAdmission{ + source: fixture.source, + id: fixture.ids[0], + token: 0xdeadbeef, + posted: true, + broken: true, + }, + claim: new(SelectClaim), + ownerLocalCurrent: fixture.task.g, + ownerLocalDriver: fixture.driver, + ownerLocalWait: &fixture.wait, + ownerLocalAdmission: ownerLocalCompletionAffectedHead, + ownerLocalUnadmitted: true, + } + if !availableChannelExternalCommitOutput(&transaction) { + t.Fatal("invalid transaction suffix became authoritative") + } + if result := BeginChannelExternalCommit( + &transaction, + fixture.source, + fixture.ids[0], + fixture.claim, + ); result != ChannelExternalCommitBeginPrepared || transaction.self != &transaction || + transaction.endpoint.source != fixture.source || transaction.endpoint.id != fixture.ids[0] || + transaction.endpoint.token == 0 || !transaction.endpoint.held || transaction.endpoint.posted || + transaction.endpoint.broken || transaction.claim != fixture.claim || + transaction.ownerLocalCurrent != nil || transaction.ownerLocalDriver != nil || + transaction.ownerLocalWait != nil || transaction.ownerLocalAdmission != ownerLocalCompletionRejected || + transaction.ownerLocalUnadmitted { + t.Fatalf("begin did not overwrite invalid suffix: result=%d transaction=%+v", result, transaction) + } + if availableChannelExternalCommitOutput(&transaction) || !transaction.Abort() || + transaction != (ChannelExternalCommit{}) { + t.Fatalf("live transaction was reusable or did not abort: %+v", transaction) + } + + if result := fixture.source.PostReady(fixture.ids[0]); result != ChannelOperationPosted { + t.Fatalf("post scratch-output cleanup readiness = %d", result) + } + requestChannelClaimCoreFixture(t, fixture) + pollChannelClaimCoreComplete(t, fixture) + decision := takeChannelClaimCoreDecision(t, fixture) + if decision.outcome != ParkOutcomeCompleted || decision.caseID != 97 || !decision.lease.Valid() { + t.Fatalf("scratch-output decision = %+v", decision) + } + releaseChannelClaimCoreFixture(t, fixture, decision) +} + func TestChannelExternalCommitSingleFailureIsAtomic(t *testing.T) { fixture := newChannelClaimCoreFixture(t, "channel-single-failure", []uint32{92}, true, 0) slot, ok := channelOperationSlotFor(fixture.source, fixture.ids[0]) diff --git a/runtime/internal/coro/completion.go b/runtime/internal/coro/completion.go index afeec9b58c..c3c5c9dd59 100644 --- a/runtime/internal/coro/completion.go +++ b/runtime/internal/coro/completion.go @@ -388,3 +388,29 @@ func ConsumeAwaitCompletion(g *G, parentHandle unsafe.Pointer) (CompletionSnapsh *record = CompletionRecord{} return snapshot, true } + +// ConsumeAwaitCompletionCompiler is the ordinary-return lane immediately +// following CommitInlineAwaitDestroyCompiler or the scheduler's checked +// destroy/resume transaction. The active parent and its terminal completion +// record already carry the exact destroyed-child receipt; panic/recover, +// cancellation, Goexit, and uncertain callers retain the complete validator. +func ConsumeAwaitCompletionCompiler(g *G, parentHandle unsafe.Pointer) (CompletionSnapshot, bool) { + if ValidG(g) && parentHandle != nil && resumeGateTaken(g) && + g.pending.kind == pendingNone && g.destroyTarget == nil && + g.spawnChild == nil && compilerReleasableParkState(&g.park) { + parent := g.active + if parent != nil && parent.handle == parentHandle && parent.owner == g && + parent.header != nil && parent.state == FrameActive && + parent.header.G == unsafe.Pointer(g) && + parent.header.SuspendReason == uint16(SuspendNone) && + parent.header.Lifecycle == uint16(FrameActive) { + record := &parent.completion + if record.child != nil && record.status == CompletionReturn && + record.typeWord == nil && record.dataWord == nil { + *record = CompletionRecord{} + return CompletionSnapshot{Status: CompletionReturn}, true + } + } + } + return ConsumeAwaitCompletion(g, parentHandle) +} diff --git a/runtime/internal/coro/critical.go b/runtime/internal/coro/critical.go index 784c3ee31a..4359420277 100644 --- a/runtime/internal/coro/critical.go +++ b/runtime/internal/coro/critical.go @@ -176,7 +176,13 @@ func pollPreemptDepthZero(g *G, charge uint32) (bool, bool) { switch mode { case executorModeBound: driver := p.executor - if !validExecutorDriverForP(driver, p) { + // A compiler safepoint only consumes the immutable current-P binding + // and the executor request gate below. Source catalogs, poll + // transactions, and selected owner-local work are scheduler-owned + // payload: their concrete publication/resolution boundaries perform + // the full audit before mutation. Re-auditing them at every compiler + // checkpoint makes unrelated event state part of every compute loop. + if !validExecutorDriverHeaderForP(driver, p) { return false, false } case executorModeUnbound: @@ -208,7 +214,7 @@ func pollPreemptDepthZero(g *G, charge uint32) (bool, bool) { if !requested { if budget <= charge { p.servicePreemptBudget = servicePreemptSafepointBudget - requested = true + requested = servicePreemptBudgetExpired(p, mode) } else { p.servicePreemptBudget = budget - charge } diff --git a/runtime/internal/coro/critical_test.go b/runtime/internal/coro/critical_test.go index 2703a8bd8f..7ad29dff2b 100644 --- a/runtime/internal/coro/critical_test.go +++ b/runtime/internal/coro/critical_test.go @@ -106,6 +106,33 @@ func TestCriticalPollDoesNotConsumeBoundExecutorRequest(t *testing.T) { runtime.KeepAlive(task.frame.memory) } +func TestCriticalPollDefersSelectedExecutorPayloadAudit(t *testing.T) { + p := new(P) + driver, registry, executor := bindTestExecutorDriver(t, p) + task := newYieldingTestG(t, "critical-executor-selected-payload") + action, ok := BeginRunG(p, task.g) + if !ok { + t.Fatal("begin bound-executor selected-payload G") + } + _ = activatePreemptTestFrame(t, p, task, action) + + // This is an impossible owner-local resolution cursor. The complete + // lifecycle audit must reject it, but a compiler safepoint neither reads + // nor mutates that selected scheduler payload. Its hot gate depends only + // on the stable driver/P binding and the exact request handle. + driver.local.resolve = publishedEpochResolveCursor{phase: publishedEpochResolveDiscover} + if !validExecutorDriverHeaderForP(driver, p) || validExecutorDriverForP(driver, p) { + t.Fatal("selected payload damage did not remain outside the hot driver header") + } + if registry.Request(executor) != ExecutorRequestPublished { + t.Fatal("publish request beside selected executor payload") + } + if !PollPreemptCompiler(task.g) || !PollPreempt(task.g) || !registry.ObserveRequested(executor) { + t.Fatal("compiler or defensive poll rejected selected payload or consumed the executor request") + } + runtime.KeepAlive(task.frame.memory) +} + func TestCriticalExitObservesButDoesNotClaimTaskCancellation(t *testing.T) { p, task, _ := newActiveCriticalTestG(t, "critical-cancel") if !EnterCritical(task.g) || !RequestTaskCancellation(p, task.g, TaskCancelAbort) { diff --git a/runtime/internal/coro/direct_channel_completion.go b/runtime/internal/coro/direct_channel_completion.go new file mode 100644 index 0000000000..97a9012a27 --- /dev/null +++ b/runtime/internal/coro/direct_channel_completion.go @@ -0,0 +1,813 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coro + +import "unsafe" + +// DirectChannelCompletion is the frame-local rendezvous record used by an +// ordinary one-case unbuffered channel park. Unlike ChannelOperationSource it +// is not a general producer descriptor: the hchan queue itself pins the frame, +// serializes the typed effect, and admits exactly one match or cancellation. +// Cross-executor matchers publish this node into the exact owner's MPSC inbox; +// select, buffered channels, and independently retained callbacks continue to +// use the durable OperationID source protocol. +// +// The fields are private because the typed hchan adapter must use the closed +// begin/effect/publish API below. The node may cross physical threads only while +// its owning frame is parked and the target has retained that executor route. +type DirectChannelCompletion struct { + // next must remain the first word: the executor's permanent stub and every + // completion expose the same atomic intrusive-link address to the MPSC + // queue without converting pointers through uintptr. + next unsafe.Pointer + owner *ExecutorDriver + wait *WaitSetRecord + context unsafe.Pointer + route RouteID + state uint32 + small uint32 + preferred uint32 +} + +// DirectChannelParkStorageV1 is the core-owned prefix of the compiler's +// one-case channel spill. Keeping Wait, Completion, and Ticket in one typed +// object makes their address relation structural: runtime adapters pass one +// capability instead of decomposing it into several raw pointers which the +// core then has to correlate again. The fields are exported only because the +// adjacent runtime package embeds this prefix; their contained scheduler state +// remains opaque. +// +// Field order intentionally matches the former CoroChanParkV1 prefix, so this +// refactor does not increase native, WASM32, embedded, or bare-metal frames. +type DirectChannelParkStorageV1 struct { + Wait WaitSetRecord + Completion DirectChannelCompletion + Ticket ParkTicket +} + +type directChannelCompletionState uint32 + +const ( + directChannelCompletionUnused directChannelCompletionState = iota + directChannelCompletionBound + directChannelCompletionEffect + directChannelCompletionMatched + directChannelCompletionCanceled + directChannelCompletionPublished + directChannelCompletionTaken + directChannelCompletionMaterialized +) + +// DirectChannelCompletionBeginResult is the hchan-locked arbitration result. +// Canceled means the owner already selected task cancellation and the dequeued +// waiter is stale. Acquired grants the caller the sole typed-effect interval. +type DirectChannelCompletionBeginResult uint8 + +const ( + DirectChannelCompletionBeginInvalid DirectChannelCompletionBeginResult = iota + DirectChannelCompletionBeginCanceled + DirectChannelCompletionBeginAcquired +) + +// DirectChannelCompletionFinishResult tells the typed hchan adapter whether +// the owner-local scheduler transaction completed the peer immediately, the +// core published it into that owner's inbox, or a target route still has to +// publish and request the remote owner. Only the inline case may retire the +// hchan waiter before a later materialization reduction observes it. +type DirectChannelCompletionFinishResult uint8 + +const ( + DirectChannelCompletionFinishInvalid DirectChannelCompletionFinishResult = iota + DirectChannelCompletionFinishInline + DirectChannelCompletionFinishOwnerPublished + DirectChannelCompletionFinishNeedsTarget +) + +func directChannelCompletionLiveState(state directChannelCompletionState) bool { + return state >= directChannelCompletionBound && state <= directChannelCompletionTaken +} + +func validBoundDirectChannelCompletion(record *WaitSetRecord, completion *DirectChannelCompletion) bool { + if record == nil || completion == nil || !record.directChannel || + record.resumeKind != resumeBindingDirectChannel || record.resume != unsafe.Pointer(completion) || + completion.wait != record || completion.context == nil || + completion.owner == nil || !completion.route.Valid() || completion.owner.route != completion.route || + completion.owner.handle.Slot == 0 || completion.owner.handle.Generation == 0 || + !validParkTicket(record.ticket) { + return false + } + state := directChannelCompletionState(preemptLoad(&completion.state)) + if !directChannelCompletionLiveState(state) { + return false + } + small := preemptLoad(&completion.small) + preferred := RouteID(preemptLoad(&completion.preferred)) + if preferred != 0 && !preferred.Valid() { + return false + } + switch state { + case directChannelCompletionBound, directChannelCompletionCanceled: + return small == uint32(ResumeSmallInvalid) + case directChannelCompletionEffect, directChannelCompletionMatched, directChannelCompletionPublished, + directChannelCompletionTaken: + // Published/Taken can represent cancellation; the task token and zero + // result distinguish it at the owner materialization boundary. + return small <= uint32(^uint8(0)) + default: + return false + } +} + +// ValidDirectChannelCompletion is the typed runtime's frame-binding check. +// It exposes no scheduler/source internals and is valid only before owner +// materialization clears the node. +func ValidDirectChannelCompletion( + completion *DirectChannelCompletion, + wait *WaitSetRecord, +) bool { + return validBoundDirectChannelCompletion(wait, completion) +} + +func validCommittedCompactDirectChannelPark(g *G, frame *Frame, wait *WaitSetRecord) bool { + if wait == nil || wait.resumeKind != resumeBindingDirectChannel || !wait.directChannel || + wait.resume == nil { + return false + } + completion := (*DirectChannelCompletion)(wait.resume) + if completion.wait != wait || completion.context == nil || + frame.owner != g || frame.parkWait != wait || wait.g != g || + wait.state != waitSetRecordCommitted || wait.work != waitSetWorkIdle || + wait.activePrev != nil || wait.activeNext != nil || wait.workNext != nil { + return false + } + completionState := directChannelCompletionState(preemptLoad(&completion.state)) + small := preemptLoad(&completion.small) + if !directChannelCompletionLiveState(completionState) || + (completionState == directChannelCompletionBound || completionState == directChannelCompletionCanceled) && + small != uint32(ResumeSmallInvalid) { + return false + } + // PrepareCurrentDirectChannelPark is the sole producer of pendingParkSet's + // compact binding, and generated code suspends immediately after the runtime + // call. The identity, queue-link, and concurrently mutable completion word + // above are the independent commit boundary. The remaining + // ParkState payload is owner-only construction data already certified by + // that pending transition; no producer can change it before Resumed. + park := &g.park + return park.ticket == wait.ticket && park.phase == parkParked && !park.resolving && + park.taskCancelKind == TaskCancelNone && park.taskCancelPhase == taskCancelIdle && + park.cancelKind == ParkCancelNone && park.outcome == ParkOutcomePending +} + +// PrepareCurrentDirectChannelPark builds the source-free one-case park. Every +// fallible owner/frame observation precedes the no-fail write suffix. The +// caller publishes its typed hchan waiter only after this returns successfully. +func PrepareCurrentDirectChannelPark( + g *G, + handle unsafe.Pointer, + header *HeaderV1, + storage *DirectChannelParkStorageV1, +) (*ExecutorDriver, RouteID) { + if g == nil || handle == nil || header == nil || storage == nil { + return nil, 0 + } + wait := &storage.Wait + completion := &storage.Completion + context := unsafe.Pointer(storage) + p := g.runP + driver := (*ExecutorDriver)(nil) + if p != nil { + driver = p.executor + } + // The bounded runner retains run.issued across the complete physical + // llvm.coro.resume. Generated code can therefore present an exact one-shot + // compiler park capability: CheckedExecutorRun already authenticated the + // immutable P/G/frame binding, and the channel hook owns its spill storage + // from the preceding resume prologue through this no-suspend call. Keep the + // arbitrary-caller path below for compatibility adapters and tests. + if driver != nil && driver.run.issued == ActionCheckResume { + // run.issued is retained across the physical resume and can be written + // only by the validated bounded selector. Together with current/inResume + // it already freezes the driver, route, action kind, running G, and empty + // scheduler queues. The compiler's park-spill magic (checked by the typed + // caller) certifies fresh Wait/Completion storage. Recheck only mutable + // facts which this operation itself consumes. + if p.current != g || !p.inResume || !p.runDecisionTaken || + !gPreemptEnabledAtDepthZero(g) || g.pending.kind != pendingNone { + return nil, 0 + } + frame := g.active + if frame == nil || frame.handle != handle || frame.header != header || + frame.state != FrameActive || frame.parkWait != nil { + return nil, 0 + } + if p.inlineAwaitDepth == 0 { + if p.action.Handle != handle { + return nil, 0 + } + } else if !resumeActionOwnsActive(g, p.action, p.inlineAwaitDepth) { + return nil, 0 + } + + previous := g.park.ticket + switch g.park.phase { + case parkIdle: + if previous != (ParkTicket{}) { + return nil, 0 + } + case parkDelivered: + if !validParkTicket(previous) { + return nil, 0 + } + default: + return prepareCurrentDirectChannelParkCompatibility( + g, handle, header, storage, + ) + } + // This issued path is deliberately flat. nextParkTicket is shared by + // arbitrary source transactions and is not inlined into this already + // sizeable boundary by LLVM; spelling its three scalar rollover cases + // here avoids a hot aggregate-return call without changing the ticket + // sequence. + ticket := previous + if ticket == (ParkTicket{}) { + ticket.generation = 1 + } else if !validParkTicket(ticket) { + return nil, 0 + } else if ticket.generation != ^uint32(0) { + ticket.generation++ + } else if ticket.epoch != ^uint32(0) { + ticket.epoch++ + ticket.generation = 1 + } else { + return nil, 0 + } + + // The direct compiler spill is exact zero storage at this boundary + // except for ParkState's retained generation/Delivered phase. Initialize + // only live words; promotion and the resume prologue clear them before + // the lifecycle capability is made reusable. + g.park.ticket = ticket + g.park.phase = parkParked + completion.owner = driver + completion.wait = wait + completion.context = context + completion.route = driver.route + completion.state = uint32(directChannelCompletionBound) + wait.g = g + wait.resume = unsafe.Pointer(completion) + wait.ticket = ticket + wait.state = waitSetRecordCommitted + wait.resumeKind = resumeBindingDirectChannel + wait.directChannel = true + frame.parkWait = wait + g.pending.kind = pendingParkSet + g.pending.directChannel = true + g.pending.from = frame + storage.Ticket = ticket + return driver, driver.route + } + return prepareCurrentDirectChannelParkCompatibility( + g, handle, header, storage, + ) +} + +// prepareCurrentDirectChannelParkCompatibility retains the complete +// arbitrary-caller implementation. It is split only so the issued path can +// fail over for the legacy parkConsumed shape without recursively selecting +// itself again. +func prepareCurrentDirectChannelParkCompatibility( + g *G, + handle unsafe.Pointer, + header *HeaderV1, + storage *DirectChannelParkStorageV1, +) (*ExecutorDriver, RouteID) { + wait := &storage.Wait + completion := &storage.Completion + context := unsafe.Pointer(storage) + p := g.runP + driver := (*ExecutorDriver)(nil) + if p != nil { + driver = p.executor + } + frame, action := g.active, Action{} + if p != nil { + action = p.action + } + if p == nil || p.current != g || !p.inResume || driver == nil || driver.p != p || + !driver.route.Valid() || action.Kind != ActionResume || action.Flags != 0 || + action.Handle == nil || p.runDecision != (RunDecision{}) || !p.runDecisionTaken || + !gPreemptEnabledAtDepthZero(g) || g.pending.kind != pendingNone || + g.spawnChild != nil || g.waiting || g.park.resolving || + g.park.taskCancelKind != TaskCancelNone || g.park.taskCancelPhase != taskCancelIdle || + (wait.state != waitSetRecordUnused || wait.resume != nil) || + directChannelCompletionState(preemptLoad(&completion.state)) != directChannelCompletionUnused { + return nil, 0 + } + if frame == nil || frame.handle != handle || frame.header != header || frame.state != FrameActive || + frame.parkWait != nil || header.G != unsafe.Pointer(g) || + header.SuspendReason != uint16(SuspendPark) || header.Lifecycle != uint16(FrameSuspended) { + return nil, 0 + } + if p.inlineAwaitDepth == 0 { + if action.Handle != handle { + return nil, 0 + } + } else if !resumeActionOwnsActive(g, action, p.inlineAwaitDepth) { + return nil, 0 + } + switch g.park.phase { + case parkIdle: + if g.park.ticket != (ParkTicket{}) { + return nil, 0 + } + case parkDelivered: + if !validParkTicket(g.park.ticket) || g.park.cancelKind != ParkCancelNone || + g.park.outcome != ParkOutcomePending { + return nil, 0 + } + case parkConsumed: + if !validReusableSingleParkState(&g.park) { + return nil, 0 + } + default: + return nil, 0 + } + ticket, ok := nextParkTicket(g.park.ticket) + if !ok { + return nil, 0 + } + g.park = ParkState{ticket: ticket, phase: parkParked} + *completion = DirectChannelCompletion{ + owner: driver, wait: wait, context: context, + route: driver.route, state: uint32(directChannelCompletionBound), + } + *wait = WaitSetRecord{ + g: g, resume: unsafe.Pointer(completion), ticket: ticket, + state: waitSetRecordCommitted, resumeKind: resumeBindingDirectChannel, + directChannel: true, + } + frame.parkWait = wait + g.pending = pendingTransition{kind: pendingParkSet, directChannel: true, from: frame} + storage.Ticket = ticket + return driver, driver.route +} + +// BeginDirectChannelCompletion claims the typed hchan effect. The hchan lock +// protects queue linkage and payload storage; this atomic word arbitrates only +// against owner-side task cancellation. +func BeginDirectChannelCompletion(completion *DirectChannelCompletion) DirectChannelCompletionBeginResult { + if completion == nil { + return DirectChannelCompletionBeginInvalid + } + for { + switch directChannelCompletionState(preemptLoad(&completion.state)) { + case directChannelCompletionBound: + if preemptCompareAndSwap( + &completion.state, + uint32(directChannelCompletionBound), + uint32(directChannelCompletionEffect), + ) { + return DirectChannelCompletionBeginAcquired + } + case directChannelCompletionCanceled: + return DirectChannelCompletionBeginCanceled + case directChannelCompletionPublished: + if preemptLoad(&completion.small) == uint32(ResumeSmallInvalid) { + return DirectChannelCompletionBeginCanceled + } + return DirectChannelCompletionBeginInvalid + default: + return DirectChannelCompletionBeginInvalid + } + } +} + +// AbortDirectChannelCompletion releases an acquired pre-effect claim. It is +// used only when another endpoint cannot be committed under the same hchan +// lock; no typed payload or waiter status may have changed yet. +func AbortDirectChannelCompletion(completion *DirectChannelCompletion) bool { + return completion != nil && preemptLoad(&completion.small) == uint32(ResumeSmallInvalid) && + preemptCompareAndSwap( + &completion.state, + uint32(directChannelCompletionEffect), + uint32(directChannelCompletionBound), + ) +} + +// FinishDirectChannelCompletion closes the typed effect and returns the exact +// owner identity needed by the target publication shim. +func FinishDirectChannelCompletion( + completion *DirectChannelCompletion, + small uint8, + preferred RouteID, +) (*ExecutorDriver, RouteID, bool) { + if completion == nil || small == ResumeSmallInvalid || completion.owner == nil || + !completion.route.Valid() || preferred != 0 && !preferred.Valid() || + preemptLoad(&completion.state) != uint32(directChannelCompletionEffect) { + return nil, 0, false + } + preemptStore(&completion.small, uint32(small)) + preemptStore(&completion.preferred, uint32(preferred)) + if !preemptCompareAndSwap( + &completion.state, + uint32(directChannelCompletionEffect), + uint32(directChannelCompletionMatched), + ) { + return nil, 0, false + } + return completion.owner, completion.route, true +} + +// validActiveDirectChannelWaitHeader proves the owner-local queue and G/frame +// placement without re-entering the generic resume-binding validator. The +// caller immediately checks the compact completion, packet, and complete +// ParkState below, so repeating those same frame fields through +// validActiveWaitSetRecordFast would add no independent certificate. +func validActiveDirectChannelWaitHeader(p *P, record *WaitSetRecord) bool { + if p == nil || record == nil || record.state != waitSetRecordActive || + !validParkTicket(record.ticket) || record.g == nil || record.g.magic != gMagic || + record.g.state != GWaiting || !record.g.waiting || record.g.queued || + record.g.nextReady != nil || record.g.runP != nil || + record.g.transferState != runnableTransferGIdle || record.g.active == nil || + record.g.active.parkWait != record { + return false + } + if record.activePrev == nil { + if p.parkWaitHead != record { + return false + } + } else if record.activePrev.activeNext != record { + return false + } + if record.activeNext == nil { + return p.parkWaitTail == record + } + return record.activeNext.activePrev == record +} + +func completeDirectChannelWait( + driver *ExecutorDriver, + completion *DirectChannelCompletion, + expected directChannelCompletionState, + small uint8, +) bool { + if driver == nil || completion == nil || completion.owner != driver || driver.p == nil || + completion.wait == nil || completion.context == nil || + completion.route != driver.route || preemptLoad(&completion.state) != uint32(expected) || + preemptLoad(&completion.small) != uint32(small) { + return false + } + wait, p := completion.wait, driver.p + if wait.state != waitSetRecordActive || wait.work != waitSetWorkIdle || wait.workNext != nil || + wait.resumeKind != resumeBindingDirectChannel || wait.resume != unsafe.Pointer(completion) || + !validActiveDirectChannelWaitHeader(p, wait) || !validReadyQueueHeader(p) || p.readyCount == ^uint32(0) { + return false + } + g, frame := wait.g, wait.g.active + if frame == nil || frame.parkWait != wait || g.spawnChild != nil || g.spawnParent != nil || g.spawnP != nil { + return false + } + state := &g.park + // The compact active wait has no source/candidate list: after the committed + // transition is activated, only task cancellation may mutate ParkState + // before this exact completion winner. Recheck the correlation and mutable + // cancellation fields; construction-only zero payload was certified at the + // pending transition and has no independent writer. + if state.ticket != wait.ticket || state.phase != parkParked || state.resolving || + state.outcome != ParkOutcomePending { + return false + } + canceled := state.taskCancelPhase == taskCancelRequested + if canceled { + if !validTaskCancelKind(state.taskCancelKind) || state.cancelKind < ParkCancelTaskAbort { + return false + } + } else if state.taskCancelKind != TaskCancelNone || state.taskCancelPhase != taskCancelIdle || + state.cancelKind != ParkCancelNone || small == ResumeSmallInvalid { + return false + } + if expected == directChannelCompletionTaken && small == ResumeSmallInvalid && !canceled { + return false + } + if expected == directChannelCompletionEffect && canceled { + return false + } + + outcome, caseID, resultSmall := ParkOutcomeCompleted, uint32(1), small + if canceled { + outcome, caseID, resultSmall = ParkOutcomeCanceled, 0, ResumeSmallInvalid + } + materializeDirectChannelWaitUnchecked( + driver, completion, wait, state, resultSmall, outcome, caseID, + ) + return true +} + +// materializeDirectChannelWaitUnchecked is the common no-fail owner mutation +// after either the routed inbox or the current-owner hchan transaction has +// authenticated its exact completion generation. It contains no observation +// or branch so the two entry gates cannot drift in the state they publish. +func materializeDirectChannelWaitUnchecked( + driver *ExecutorDriver, + completion *DirectChannelCompletion, + wait *WaitSetRecord, + state *ParkState, + resultSmall uint8, + outcome ParkOutcome, + caseID uint32, +) { + preemptStore(&completion.small, uint32(resultSmall)) + // A compact Parked state owns no source/link payload. Its ticket and task + // cancellation receipt remain live; materialization changes only this small + // scalar overlay. Avoid clearing and reconstructing the complete ParkState. + state.phase = parkMaterialized + state.directChannel = true + state.seed = preemptLoad(&completion.preferred) + state.cancelKind = ParkCancelNone + state.outcome = outcome + state.winnerCase = caseID + preemptStore(&completion.state, uint32(directChannelCompletionMaterialized)) + promoteReadyWaitSetUnchecked(driver.p, wait) + driver.run.blocked = false + driver.run.actionsSinceSource = 0 + driver.run.readyDebt = true +} + +// FinishDirectChannelCompletionFromCompilerTask is the fused completion +// boundary for a compiler-owned one-case channel operation. The hidden task is +// already available to the typed hchan path; deriving its P/driver here avoids +// decomposing that private scheduler capability in runtime and then replaying +// the same relation at a second package boundary. +// +// A valid current task supplies the preferred producer route and admits the +// owner-local materialization fast path. Compatibility/foreign callers may +// pass nil and an advisory fallback route; they retain the routed publication +// path. NeedsTarget returns the exact owner and route which the target shim +// must retain and request. +func FinishDirectChannelCompletionFromCompilerTask( + current *G, + completion *DirectChannelCompletion, + small uint8, + fallback RouteID, +) (*ExecutorDriver, RouteID, DirectChannelCompletionFinishResult) { + if completion == nil || small == ResumeSmallInvalid { + return nil, 0, DirectChannelCompletionFinishInvalid + } + preferred := fallback + var currentDriver *ExecutorDriver + if current != nil { + p := current.runP + if p != nil && p.current == current && p.inResume { + driver := p.executor + if driver != nil && driver.p == p && driver.route.Valid() && + driver.sources.route == driver.route { + currentDriver = driver + preferred = driver.route + } + } + } + if preferred != 0 && !preferred.Valid() { + return nil, 0, DirectChannelCompletionFinishInvalid + } + if currentDriver != nil && completion.owner == currentDriver { + p := currentDriver.p + wait := completion.wait + // The legacy prepare-then-try ABI can match the current task before its + // committed waiter has crossed llvm.coro.resume and become Active. That + // record cannot be promoted while the task is still running; authenticate + // the exact adjacent pending capability and let the ordinary completion + // inbox materialize it after Resumed activates the wait. The fused V2 ABI + // probes before preparing and never takes this compatibility edge. + if wait != nil && wait.state == waitSetRecordCommitted && wait.g == current { + frame := current.active + if frame == nil || current.pending.kind != pendingParkSet || + !current.pending.directChannel || current.pending.from != frame || + !validCommittedCompactDirectChannelPark(current, frame, wait) { + return nil, 0, DirectChannelCompletionFinishInvalid + } + } else { + // The hchan lock and Effect state are the cross-thread arbitration + // boundary; current/inResume makes every scheduler field owner-only. + // Recheck the live generation and active-list correlations immediately + // consumed by the no-fail materialization suffix. + if completion.route != currentDriver.route || wait == nil || wait.g == nil || + preemptLoad(&completion.state) != uint32(directChannelCompletionEffect) || + wait.state != waitSetRecordActive || !wait.directChannel || + wait.resume != unsafe.Pointer(completion) { + return nil, 0, DirectChannelCompletionFinishInvalid + } + g := wait.g + frame := g.active + if frame == nil || frame.parkWait != wait { + return nil, 0, DirectChannelCompletionFinishInvalid + } + state := &g.park + if state.ticket != wait.ticket || state.phase != parkParked || + state.taskCancelKind != TaskCancelNone || state.taskCancelPhase != taskCancelIdle || + state.cancelKind != ParkCancelNone || p.readyCount == ^uint32(0) { + return nil, 0, DirectChannelCompletionFinishInvalid + } + preemptStore(&completion.preferred, uint32(preferred)) + materializeDirectChannelWaitUnchecked( + currentDriver, completion, wait, state, small, ParkOutcomeCompleted, 1, + ) + return nil, 0, DirectChannelCompletionFinishInline + } + } + owner, route, ok := FinishDirectChannelCompletion(completion, small, preferred) + if !ok { + return nil, 0, DirectChannelCompletionFinishInvalid + } + if currentDriver == owner { + if !PublishExecutorDirectChannelCompletion(owner, completion) { + return nil, 0, DirectChannelCompletionFinishInvalid + } + return nil, 0, DirectChannelCompletionFinishOwnerPublished + } + return owner, route, DirectChannelCompletionFinishNeedsTarget +} + +// PublishExecutorDirectChannelCompletion appends one terminal frame node to +// the exact owner's lock-free MPSC stack. Route/target glue must retain its +// executor ingress until this publication and the subsequent request finish. +func PublishExecutorDirectChannelCompletion( + driver *ExecutorDriver, + completion *DirectChannelCompletion, +) bool { + if driver == nil || completion == nil || completion.owner != driver || + completion.route != driver.route || driver.magic != executorDriverMagic || + driver.p == nil || driver.p.executor != driver || + preemptLoadPointer(&driver.directChannelHead) == nil { + return false + } + state := directChannelCompletionState(preemptLoad(&completion.state)) + if state != directChannelCompletionMatched && state != directChannelCompletionCanceled { + return false + } + if !preemptCompareAndSwap( + &completion.state, uint32(state), uint32(directChannelCompletionPublished), + ) { + return false + } + preemptStorePointer(&completion.next, nil) + previous := preemptSwapPointer(&driver.directChannelHead, unsafe.Pointer(completion)) + if previous == nil || previous == unsafe.Pointer(completion) { + return false + } + // Publication linearizes at the exchange, but the consumer cannot reach the + // node until this release link is visible. The target request happens only + // after the link, so a producer paused in this narrow interval cannot strand + // a sleeping executor. + preemptStorePointer((*unsafe.Pointer)(previous), unsafe.Pointer(completion)) + return true +} + +// DirectChannelCompletionSnapshot exposes only the typed runtime reduction +// payload after the owner has removed the node from its inbox. +func DirectChannelCompletionSnapshot( + completion *DirectChannelCompletion, +) (context unsafe.Pointer, small uint8, matched bool, ok bool) { + if completion == nil || preemptLoad(&completion.state) != uint32(directChannelCompletionTaken) || + completion.context == nil || preemptLoad(&completion.small) > uint32(^uint8(0)) { + return nil, ResumeSmallInvalid, false, false + } + small = uint8(preemptLoad(&completion.small)) + return completion.context, small, small != ResumeSmallInvalid, true +} + +// CommitDirectChannelCompletion consumes the typed runtime cleanup and +// materializes the frame-local completion before publishing the G runnable. +func CommitDirectChannelCompletion(completion *DirectChannelCompletion, small uint8) bool { + if completion == nil || completion.owner == nil { + return false + } + return completeDirectChannelWait( + completion.owner, completion, directChannelCompletionTaken, small, + ) +} + +func directChannelCompletionForWait(wait *WaitSetRecord) (*DirectChannelCompletion, bool) { + if wait == nil || wait.resumeKind != resumeBindingDirectChannel || wait.resume == nil { + return nil, false + } + completion := (*DirectChannelCompletion)(wait.resume) + return completion, validBoundDirectChannelCompletion(wait, completion) +} + +func requestDirectChannelCancellation(completion *DirectChannelCompletion) bool { + if completion == nil || completion.owner == nil { + return false + } + for { + switch directChannelCompletionState(preemptLoad(&completion.state)) { + case directChannelCompletionBound: + if !preemptCompareAndSwap( + &completion.state, + uint32(directChannelCompletionBound), + uint32(directChannelCompletionCanceled), + ) { + continue + } + return PublishExecutorDirectChannelCompletion(completion.owner, completion) + case directChannelCompletionEffect, directChannelCompletionMatched, + directChannelCompletionPublished, directChannelCompletionTaken: + // The physical winner owns completion and will observe the sticky task + // token while materializing its result. + return true + default: + return false + } + } +} + +func takeExecutorDirectChannelCompletion(driver *ExecutorDriver) (*DirectChannelCompletion, bool) { + if driver == nil || driver.directChannelTail == nil { + return nil, false + } + stub := unsafe.Pointer(&driver.directChannelStub) + tail := driver.directChannelTail + next := preemptLoadPointer((*unsafe.Pointer)(tail)) + if tail == stub { + if next == nil { + return nil, true + } + driver.directChannelTail = next + tail = next + next = preemptLoadPointer((*unsafe.Pointer)(tail)) + } + if next == nil { + head := preemptLoadPointer(&driver.directChannelHead) + if head == nil { + return nil, false + } + if tail != head { + // A producer has exchanged the head but has not linked its predecessor + // yet. It will request this executor after completing that link. + return nil, true + } + preemptStorePointer(&driver.directChannelStub, nil) + previous := preemptSwapPointer(&driver.directChannelHead, stub) + if previous == nil || previous == stub { + return nil, false + } + preemptStorePointer((*unsafe.Pointer)(previous), stub) + next = preemptLoadPointer((*unsafe.Pointer)(tail)) + if next == nil { + // Another producer may still be closing the predecessor link. The + // subsequent request is the retry obligation; do not spin here. + return nil, true + } + } + driver.directChannelTail = next + preemptStorePointer((*unsafe.Pointer)(tail), nil) + completion := (*DirectChannelCompletion)(tail) + if !preemptCompareAndSwap( + &completion.state, + uint32(directChannelCompletionPublished), + uint32(directChannelCompletionTaken), + ) { + return nil, false + } + return completion, true +} + +func executorDirectChannelCompletionPending(driver *ExecutorDriver) bool { + if driver == nil || driver.directChannelTail == nil { + return false + } + stub := unsafe.Pointer(&driver.directChannelStub) + tail := driver.directChannelTail + next := preemptLoadPointer((*unsafe.Pointer)(tail)) + if tail == stub { + return next != nil + } + if next != nil { + return true + } + return preemptLoadPointer(&driver.directChannelHead) == tail +} + +func executorDirectChannelInboxIdle(driver *ExecutorDriver) bool { + if driver == nil || driver.directChannelTail == nil { + return false + } + stub := unsafe.Pointer(&driver.directChannelStub) + return driver.directChannelTail == stub && + preemptLoadPointer(&driver.directChannelHead) == stub && + preemptLoadPointer(&driver.directChannelStub) == nil +} diff --git a/runtime/internal/coro/direct_channel_completion_test.go b/runtime/internal/coro/direct_channel_completion_test.go new file mode 100644 index 0000000000..6d63cf6639 --- /dev/null +++ b/runtime/internal/coro/direct_channel_completion_test.go @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coro + +import ( + "runtime" + "sync" + "testing" + "unsafe" +) + +func TestDirectChannelCompletionInboxConcurrentReuse(t *testing.T) { + const ( + producers = 8 + iterations = 500 + ) + p := new(P) + driver := &ExecutorDriver{ + magic: executorDriverMagic, + state: executorDriverActive, + p: p, + route: RouteID(1), + } + p.executor = driver + stub := unsafe.Pointer(&driver.directChannelStub) + preemptStorePointer(&driver.directChannelHead, stub) + driver.directChannelTail = stub + + nodes := make([]DirectChannelCompletion, producers) + ack := make([]chan struct{}, producers) + index := make(map[*DirectChannelCompletion]int, producers) + for producer := range nodes { + node := &nodes[producer] + node.owner = driver + node.route = driver.route + preemptStore(&node.state, uint32(directChannelCompletionMatched)) + ack[producer] = make(chan struct{}, 1) + index[node] = producer + } + + errors := make(chan int, producers) + var group sync.WaitGroup + group.Add(producers) + for producer := range nodes { + go func(producer int) { + defer group.Done() + for iteration := 0; iteration < iterations; iteration++ { + if !PublishExecutorDirectChannelCompletion(driver, &nodes[producer]) { + errors <- producer + return + } + <-ack[producer] + } + }(producer) + } + + seen := make([]int, producers) + for completed := 0; completed < producers*iterations; { + node, ok := takeExecutorDirectChannelCompletion(driver) + if !ok { + t.Fatal("take concurrent direct channel completion") + } + if node == nil { + select { + case producer := <-errors: + t.Fatalf("producer %d could not publish direct channel completion", producer) + default: + } + runtime.Gosched() + continue + } + producer, known := index[node] + if !known { + t.Fatalf("take unknown direct channel completion %p", node) + } + seen[producer]++ + completed++ + preemptStore(&node.state, uint32(directChannelCompletionMatched)) + ack[producer] <- struct{}{} + } + group.Wait() + select { + case producer := <-errors: + t.Fatalf("producer %d could not publish direct channel completion", producer) + default: + } + for producer, count := range seen { + if count != iterations { + t.Fatalf("producer %d completions = %d, want %d", producer, count, iterations) + } + } + if !executorDirectChannelInboxIdle(driver) || executorDirectChannelCompletionPending(driver) { + t.Fatal("direct channel completion inbox retained a node") + } +} + +func TestDirectChannelCompletionPublishesToSleepingOwner(t *testing.T) { + p := new(P) + driver := &ExecutorDriver{ + magic: executorDriverMagic, + state: executorDriverSleeping, + p: p, + route: RouteID(1), + } + p.executor = driver + stub := unsafe.Pointer(&driver.directChannelStub) + preemptStorePointer(&driver.directChannelHead, stub) + driver.directChannelTail = stub + + completion := &DirectChannelCompletion{ + owner: driver, + route: driver.route, + state: uint32(directChannelCompletionMatched), + } + if !PublishExecutorDirectChannelCompletion(driver, completion) { + t.Fatal("publish direct channel completion to sleeping owner") + } + got, ok := takeExecutorDirectChannelCompletion(driver) + if !ok || got != completion { + t.Fatalf("take sleeping-owner completion = (%p, %t), want (%p, true)", got, ok, completion) + } +} diff --git a/runtime/internal/coro/execution_quota.go b/runtime/internal/coro/execution_quota.go index da48ca7765..1f22ebf75e 100644 --- a/runtime/internal/coro/execution_quota.go +++ b/runtime/internal/coro/execution_quota.go @@ -41,6 +41,24 @@ type ExecutionQuota struct { holders uint32 } +// BindExecutorServicePressure gives one already-bound executor a read-only +// view of this quota's waiter publication word. The quota and driver must both +// remain at stable addresses through driver retirement. This is startup-only: +// a runtime binds every fleet route after Start and before any managed resume. +// +// The word is advisory, not a second admission gate. A compiler safepoint only +// uses a nonzero value to return to the scheduler; TryAcquire and Release remain +// the sole owners of managed-execution admission. +func BindExecutorServicePressure(driver *ExecutorDriver, quota *ExecutionQuota) bool { + if !validExecutorDriver(driver) || driver.state != executorDriverActive || + driver.servicePressure != nil || quota == nil || + preemptLoad("a.lifecycle) != uint32(executionQuotaActive) { + return false + } + driver.servicePressure = "a.waiters + return true +} + type executionQuotaLifecycle uint32 const ( @@ -276,6 +294,13 @@ func (quota *ExecutionQuota) TryAcquire(route RouteID) (acquired, ok bool) { if !claimed { return false, false } + // A route whose bit was already published is an awakened contender. A + // freshly released holder has no bit; if it immediately races back into + // TryAcquire while another route is waiting, it must join the waiter set + // instead of repeatedly stealing the permit before that owner's doorbell + // wake can run. This is advisory FIFO at the physical-route boundary, not a + // ticket lock: uncontended acquisition retains the one-CAS fast path. + wasWaiting := preemptLoad("a.waiters)&mask != 0 for { if preemptLoad("a.lifecycle) != uint32(executionQuotaActive) { _ = executionQuotaClearWaiter(quota, mask) @@ -290,6 +315,26 @@ func (quota *ExecutionQuota) TryAcquire(route RouteID) (acquired, ok bool) { limit := preemptLoad("a.limit) active := preemptLoad("a.active) if limit != 0 && active < limit { + if !wasWaiting && preemptLoad("a.waiters)&^mask != 0 { + if !executionQuotaSetWaiter(quota, mask) { + _, _ = executionQuotaTransitionHolder( + quota, + route, + executionQuotaHolderClaiming, + executionQuotaHolderIdle, + ) + return false, false + } + if _, deferred := executionQuotaTransitionHolder( + quota, + route, + executionQuotaHolderClaiming, + executionQuotaHolderIdle, + ); !deferred { + return false, false + } + return false, true + } if !preemptCompareAndSwap("a.active, active, active+1) { continue } diff --git a/runtime/internal/coro/execution_quota_test.go b/runtime/internal/coro/execution_quota_test.go index 9134771431..c08394f854 100644 --- a/runtime/internal/coro/execution_quota_test.go +++ b/runtime/internal/coro/execution_quota_test.go @@ -74,15 +74,24 @@ func TestExecutionQuotaLifecycleAndStickyWake(t *testing.T) { if wake, ok := quota.Release(1); !ok || !wake { t.Fatalf("route 1 release = (%t, %t), want sticky wake", wake, ok) } + if acquired, ok := quota.TryAcquire(1); acquired || !ok { + t.Fatalf("freshly released route bypassed published waiter: (%t, %t)", acquired, ok) + } if held, ok := quota.Held(1); !ok || held { t.Fatalf("released route 1 held lease = (%t, %t)", held, ok) } if acquired, ok := quota.TryAcquire(3); !acquired || !ok { t.Fatal("route 3 did not acquire released permit") } - if _, ok := quota.Release(3); !ok { + if wake, ok := quota.Release(3); !ok || !wake { t.Fatal("route 3 release failed") } + if acquired, ok := quota.TryAcquire(1); !acquired || !ok { + t.Fatal("deferred route 1 did not acquire after waiter release") + } + if _, ok := quota.Release(1); !ok { + t.Fatal("route 1 final release failed") + } if wake, ok := quota.Seal(); !ok || wake { t.Fatalf("execution quota seal = (%t, %t)", wake, ok) } diff --git a/runtime/internal/coro/executor_driver.go b/runtime/internal/coro/executor_driver.go index 3c047fd66e..cd1f3fbea1 100644 --- a/runtime/internal/coro/executor_driver.go +++ b/runtime/internal/coro/executor_driver.go @@ -25,6 +25,11 @@ import "unsafe" // // Every method is scheduler-owner-only. The driver, P, registry, and table must // remain at stable addresses from BindExecutor through ConfirmExecutorClose. +// requestGate caches the exact registry slot gate during that lifetime; foreign +// producers may update the gate atomically, but they never retain the driver. +// servicePressure is an optional process-shared scheduler-pressure word. A +// fleet quota binds its waiter word once at startup so a locally isolated G can +// still yield to a remote executor waiting for managed-execution capacity. // A real target surrounds a successful PrepareExecutorSleep with its retained // source poll and calls WakeExecutor after a real or spurious wake. // @@ -35,19 +40,28 @@ import "unsafe" // last-G terminal close handoff, while the target-specific join dispatcher and // multi-P executor migration remain later layers. type ExecutorDriver struct { - magic uint32 - state executorDriverState - p *P - registry *ExecutorRegistry - handle ExecutorHandle - route RouteID - sources ExecutorSourceSet - poll executorPollTransaction - local ownerLocalCompletionCursor - run executorRunCursor - prepareNow int64 - hasPrepareNow bool - terminalKind ActionKind + magic uint32 + state executorDriverState + p *P + registry *ExecutorRegistry + handle ExecutorHandle + requestGate *uint32 + servicePressure *uint32 + route RouteID + sources ExecutorSourceSet + poll executorPollTransaction + local ownerLocalCompletionCursor + // directChannelHead is the producer exchange cursor; directChannelTail is + // owner-only, and directChannelStub is the permanent one-word sentinel. + // Together they form an intrusive MPSC queue for frame-local one-case hchan + // completions. A node's owning frame remains pinned by its WaitSetRecord. + directChannelHead unsafe.Pointer + directChannelTail unsafe.Pointer + directChannelStub unsafe.Pointer + run executorRunCursor + prepareNow int64 + hasPrepareNow bool + terminalKind ActionKind } type executorDriverState uint8 @@ -72,6 +86,25 @@ func validExecutorDriverHeader(driver *ExecutorDriver) bool { if driver == nil || driver.magic != executorDriverMagic || driver.state == executorDriverUnbound { return false } + if driver.state == executorDriverActive { + return driver.terminalKind == ActionInvalid && !driver.hasPrepareNow && driver.prepareNow == 0 && + driver.p != nil && driver.registry != nil && + driver.handle.Slot != 0 && driver.handle.Generation != 0 && driver.requestGate != nil && + driver.route.Valid() && driver.sources.route == driver.route && + driver.p.executor == driver && preemptLoad(&driver.p.executorMode) == executorModeBound && + validExecutorSourceSetHeader(&driver.sources, driver.p) && + validExecutorRunCursor(&driver.run, driver.p) + } + return validExecutorDriverColdHeader(driver) +} + +// validExecutorDriverColdHeader retains the complete lifecycle-state audit +// outside the active runner. Keeping it out of the common reduction selector +// avoids calculating terminal/prepare truth tables for every dispatch, source, +// and action while preserving the exact sleep/close diagnostics. +// +//go:noinline +func validExecutorDriverColdHeader(driver *ExecutorDriver) bool { terminalKind := driver.terminalKind validTerminalState := driver.state == executorDriverTerminalClosing && (terminalKind == ActionDestroy || terminalKind == ActionPanicDestroy) @@ -85,15 +118,24 @@ func validExecutorDriverHeader(driver *ExecutorDriver) bool { } return (driver.state == executorDriverTerminalClosing) == validTerminalState && driver.p != nil && driver.registry != nil && driver.handle.Slot != 0 && driver.handle.Generation != 0 && + driver.requestGate != nil && driver.route.Valid() && driver.sources.route == driver.route && driver.p.executor == driver && preemptLoad(&driver.p.executorMode) == executorModeBound && validExecutorSourceSetHeader(&driver.sources, driver.p) && - validOwnerLocalCompletionHeader(&driver.local, driver.p) && validExecutorRunCursor(&driver.run, driver.p) } func validExecutorDriver(driver *ExecutorDriver) bool { - return validExecutorDriverHeader(driver) && + // The owner-local cursor is selected scheduler work, not an immutable + // driver binding. Keep its complete audit at full lifecycle boundaries and + // validate it again immediately before local publication or resolution; + // unrelated source and managed-resume probes must not make this optional + // queue part of the common driver-header gate. + if !validExecutorDriverHeader(driver) { + return false + } + slot, ok := executorSlot(driver.registry, driver.handle) + return ok && driver.requestGate == &slot.gate && validExecutorSourceSet(&driver.sources, driver.p) && validExecutorPollTransaction(&driver.poll, &driver.sources) && validOwnerLocalCompletion(&driver.local, driver.p) @@ -153,35 +195,97 @@ func CurrentExecutorDriver(g *G) (*ExecutorDriver, ExecutorHandle, RouteID, bool return driver, driver.handle, driver.route, true } +// CurrentExecutorDriverForActiveResume is the narrow runtime-context +// capability used after the physical runtime has already authenticated g as +// its currently installed logical task. It accepts only the bounded runner's +// issued CheckResume interval and rechecks the exact P/G/action, driver, source +// and route identities. Unlike CurrentExecutorDriver it does not repeat the +// registry-slot and complete active-frame audit: neither registry lifetime nor +// frame identity can change inside this no-suspend physical resume. +// +// The returned driver is scheduler-owner-only and must not survive the current +// resume or be passed to a producer. Callers without an independently proven +// current runtime task must use CurrentExecutorDriver instead. +func CurrentExecutorDriverForActiveResume(g *G) (*ExecutorDriver, RouteID, bool) { + if !ValidG(g) || g.transferState != runnableTransferGIdle || !resumeGateTaken(g) { + return nil, 0, false + } + p := g.runP + driver := p.executor + if driver == nil || driver.magic != executorDriverMagic || + driver.state != executorDriverActive || driver.p != p || + driver.run.issued != ActionCheckResume || + preemptLoad(&p.executorMode) != executorModeBound || + driver.handle.Slot == 0 || driver.handle.Generation == 0 || + !driver.route.Valid() || !validExecutorSourceSetHeader(&driver.sources, p) || + driver.sources.route != driver.route { + return nil, 0, false + } + return driver, driver.route, true +} + +// CurrentExecutorDriverForCompilerTask is the narrow capability boundary for +// a hidden G parameter carried by generated physical coroutine code. The +// scheduler has already authenticated the resume before that parameter can be +// observed; this check therefore freezes only the mutable G/P/driver relation +// needed until the next no-suspend runtime call returns. Source-specific +// operations must still validate their own source and endpoint identities. +// +// Callers which recovered a G through TLS, a callback, or public metadata must +// use CurrentExecutorDriver or CurrentExecutorDriverForActiveResume instead. +func CurrentExecutorDriverForCompilerTask(g *G) (*ExecutorDriver, RouteID, bool) { + if g == nil { + return nil, 0, false + } + p := g.runP + if p == nil || p.current != g || !p.inResume { + return nil, 0, false + } + driver := p.executor + // The hidden task and private inResume episode are the scheduler's + // certificate: no target can close, rebind, transfer, or replace this + // P/driver while generated code is active below llvm.coro.resume. Retain the + // exact pointer and route correlation needed by the no-suspend caller; the + // next park/source operation validates its own mutable endpoint state. + if driver == nil || driver.p != p || !driver.route.Valid() || + driver.sources.route != driver.route { + return nil, 0, false + } + return driver, driver.route, true +} + // currentExecutorParkDriver resolves the exact executor during the narrow // compiler park/resume-hook window. The active frame has already published // SuspendPark/FrameSuspended while the scheduler still owns the same // ActionResume episode. Typed adapters add only their closed source-owner // validation after this common proof. func currentExecutorParkDriver(g *G) (*ExecutorDriver, ExecutorHandle, RouteID, bool) { - if !ValidG(g) || g.transferState != runnableTransferGIdle || !resumeGateTaken(g) || - g.runP == nil || g.active == nil || g.active.handle == nil || g.active.header == nil { + driver, route, current := CurrentExecutorDriverForCompilerTask(g) + if !current || g.active == nil || g.active.handle == nil || g.active.header == nil { return nil, ExecutorHandle{}, 0, false } - p := g.runP - driver := p.executor + p := driver.p handle := g.active.handle header := g.active.header - if !validExecutorDriverHeaderForP(driver, p) || p.current != g || !p.inResume || - !expectedAction(p, g, p.action, ActionResume) || !activeResumeOwnedByAction(g) || - g.state != GRunning || g.active.state != FrameActive || + action := p.action + if action.Kind != ActionResume || action.Flags != 0 || action.Handle == nil || + p.runDecision != (RunDecision{}) || !p.runDecisionTaken || + !gPreemptEnabledAtDepthZero(g) || + g.active.state != FrameActive || g.active.handle != handle || g.active.header != header || header.G != unsafe.Pointer(g) || header.SuspendReason != uint16(SuspendPark) || header.Lifecycle != uint16(FrameSuspended) || !driver.route.Valid() || driver.handle.Slot == 0 || driver.handle.Generation == 0 { return nil, ExecutorHandle{}, 0, false } - slot, ok := executorSlot(driver.registry, driver.handle) - if !ok || preemptLoad(&slot.generation) != driver.handle.Generation || - preemptLoad(&slot.state) != uint32(executorActive) { + if p.inlineAwaitDepth == 0 { + if action.Handle != handle { + return nil, ExecutorHandle{}, 0, false + } + } else if !resumeActionOwnsActive(g, action, p.inlineAwaitDepth) { return nil, ExecutorHandle{}, 0, false } - return driver, driver.handle, driver.route, true + return driver, driver.handle, route, true } // CurrentExecutorSourceCatalog returns the exact owner P and direct-call source @@ -344,10 +448,15 @@ func idleExecutorScheduler(p *P) bool { // its executorMode load; executorMode is a capability guard, not a refcounted // admission barrier for migration from the legacy ABI. func bindExecutorAtRoute(driver *ExecutorDriver, p *P, registry *ExecutorRegistry, handle ExecutorHandle, route RouteID, catalog ExecutorSourceCatalog) bool { + requestSlot, requestOK := executorSlot(registry, handle) if driver == nil || driver.magic != 0 || driver.state != executorDriverUnbound || driver.p != nil || - driver.registry != nil || driver.handle != (ExecutorHandle{}) || driver.route != 0 || driver.sources != (ExecutorSourceSet{}) || + driver.registry != nil || driver.handle != (ExecutorHandle{}) || driver.requestGate != nil || + driver.servicePressure != nil || + driver.route != 0 || driver.sources != (ExecutorSourceSet{}) || driver.poll != (executorPollTransaction{}) || driver.local != (ownerLocalCompletionCursor{}) || + driver.directChannelHead != nil || driver.directChannelTail != nil || + driver.directChannelStub != nil || driver.run != (executorRunCursor{}) || driver.prepareNow != 0 || driver.hasPrepareNow || driver.terminalKind != ActionInvalid || @@ -355,7 +464,8 @@ func bindExecutorAtRoute(driver *ExecutorDriver, p *P, registry *ExecutorRegistr p.osThreadLockOwner != nil || p.foreignReentry != nil || preemptLoad(&p.schedule) != scheduleIdle || !idleExecutorScheduler(p) || p.readyHead != nil || p.readyTail != nil || !emptySchedulerWaitQueues(p) || - !route.Valid() || !activeExecutorHandle(registry, handle) || !bindExecutorSourceSetAtRoute(&driver.sources, p, route, catalog) { + !route.Valid() || !requestOK || !activeExecutorHandle(registry, handle) || + !bindExecutorSourceSetAtRoute(&driver.sources, p, route, catalog) { return false } driver.magic = executorDriverMagic @@ -363,7 +473,11 @@ func bindExecutorAtRoute(driver *ExecutorDriver, p *P, registry *ExecutorRegistr driver.p = p driver.registry = registry driver.handle = handle + driver.requestGate = &requestSlot.gate driver.route = route + directChannelStub := unsafe.Pointer(&driver.directChannelStub) + preemptStorePointer(&driver.directChannelHead, directChannelStub) + driver.directChannelTail = directChannelStub p.executor = driver preemptStore(&p.executorMode, executorModeBound) return true @@ -391,9 +505,17 @@ func (driver *ExecutorDriver) Route() (RouteID, bool) { func publishExecutorSourcesInState(driver *ExecutorDriver, now int64, withDeadline bool, state executorDriverState) (scan executorSourceScan, ok bool) { if !validExecutorDriver(driver) || driver.state != state || driver.poll.phase != executorPollIdle || - !emptyExecutorRunCursor(driver) || !idleExecutorScheduler(driver.p) { + driver.run != (executorRunCursor{}) || !emptyOwnerLocalCompletion(&driver.local) || + !idleExecutorScheduler(driver.p) { return executorSourceScan{}, false } + // The producer-owned direct-channel inbox is an independent durable work + // source. In particular, a producer may append to it after the owner arms + // the executor gate and while this bounded source-catalog scan is running. + // Source publication neither consumes nor interprets that inbox, so making + // its emptiness a scan invariant would turn legal ingress into corruption. + // Sleep admission observes it again after the scan and leaves IdleArmed; + // the unified runner then owns materialization. return driver.sources.publishPass(driver.p, now, withDeadline) } @@ -508,6 +630,63 @@ func leaveExecutorIdleForRun(driver *ExecutorDriver) bool { return validExecutorDriver(driver) } +// wakeableExecutorRunCursor accepts the producer-visible work which may arrive +// after CommitSleep. The owner run cursor and owner-local reducer must still be +// exactly idle, but a direct-channel producer is specifically allowed to have +// appended an MPSC inbox node before it requests and wakes this executor. +// Requiring emptyExecutorRunCursor here would turn that legal wake into a +// fail-closed result because that stronger helper deliberately requires the +// producer inbox to be empty for sleep admission and lifecycle transitions. +func wakeableExecutorRunCursor(driver *ExecutorDriver) bool { + return driver != nil && driver.run == (executorRunCursor{}) && + emptyOwnerLocalCompletion(&driver.local) && + driver.directChannelTail != nil && + preemptLoadPointer(&driver.directChannelHead) != nil +} + +// ExecutorWorkerCompletionProbe is one short-lived owner-side observation of +// the worker source's durable pending word. It exposes neither operation +// identities nor source mutation. A target may retain it only across a bounded +// non-suspending active-spin policy immediately before ArmIdle. +type ExecutorWorkerCompletionProbe struct { + source *WorkerOperationSource +} + +func (probe ExecutorWorkerCompletionProbe) Valid() bool { + return probe.source != nil +} + +// Ready is an acquire observation of a completed worker publication. The +// producer stores this word only after the exact payload and mailbox are +// durable, so the owner may safely re-enter its unified source reducer. +func (probe ExecutorWorkerCompletionProbe) Ready() bool { + return probe.source != nil && probe.source.Pending() +} + +// PrepareExecutorWorkerCompletionProbe observes the one condition under which +// a native target may profitably defer ArmIdle: an exact submitted worker +// operation is still incomplete. It never turns that advisory observation into +// a correctness obligation. A target which does not observe Ready within its +// bounded policy must use the ordinary retained wait transaction unchanged. +func PrepareExecutorWorkerCompletionProbe( + driver *ExecutorDriver, +) (probe ExecutorWorkerCompletionProbe, awaiting, ready, ok bool) { + if !validExecutorDriver(driver) || driver.state != executorDriverActive || + driver.run.issued != ActionInvalid || driver.poll.phase != executorPollIdle || + !emptyOwnerLocalCompletion(&driver.local) || + !executorDirectChannelInboxIdle(driver) || !idleExecutorScheduler(driver.p) { + return ExecutorWorkerCompletionProbe{}, false, false, false + } + if driver.sources.worker == nil { + return ExecutorWorkerCompletionProbe{}, false, false, true + } + awaiting, ready, ok = driver.sources.worker.submittedCompletionState(driver.p) + if !ok || !awaiting && !ready { + return ExecutorWorkerCompletionProbe{}, awaiting, ready, ok + } + return ExecutorWorkerCompletionProbe{source: driver.sources.worker}, awaiting, ready, true +} + // PrepareExecutorSleep executes ArmIdle, an unconditional source fact scan, // and exact CommitSleep only when no runnable exists and parked Gs remain. // Source resolution stays in the unified runner. A true sleep result authorizes @@ -515,9 +694,13 @@ func leaveExecutorIdleForRun(driver *ExecutorDriver) bool { // request won and the scheduler should continue without blocking. func PrepareExecutorSleep(driver *ExecutorDriver) (sleep bool, ok bool) { if !validExecutorDriver(driver) || driver.sources.usesMonotonicTime() || driver.state != executorDriverActive || - !emptyExecutorRunCursor(driver) || !idleExecutorScheduler(driver.p) { + driver.run != (executorRunCursor{}) || !emptyOwnerLocalCompletion(&driver.local) || + !idleExecutorScheduler(driver.p) { return false, false } + if !executorDirectChannelInboxIdle(driver) { + return false, wakeableExecutorRunCursor(driver) + } if runnableForOSThreadOwner(driver.p) || !HasWaiting(driver.p) { return false, true } @@ -542,7 +725,8 @@ func PrepareExecutorSleep(driver *ExecutorDriver) (sleep bool, ok bool) { _, _ = driver.registry.LeaveIdle(driver.handle) return false, false } - hasWork := drained != 0 || runnableForOSThreadOwner(driver.p) || driver.sources.pending(driver.p) || + hasWork := drained != 0 || !executorDirectChannelInboxIdle(driver) || + runnableForOSThreadOwner(driver.p) || driver.sources.pending(driver.p) || driver.registry.ObserveRequested(driver.handle) || preemptLoad(&driver.p.schedule) != scheduleIdle if hasWork { if !leaveExecutorIdleForRun(driver) { @@ -573,9 +757,16 @@ func prepareExecutorSleepAt( allowEmpty bool, ) (prepared bool, ok bool) { if !validExecutorDriver(driver) || !driver.sources.usesMonotonicTime() || driver.state != executorDriverActive || - !emptyExecutorRunCursor(driver) || !idleExecutorScheduler(driver.p) || now < 0 { + driver.run != (executorRunCursor{}) || !emptyOwnerLocalCompletion(&driver.local) || + !idleExecutorScheduler(driver.p) || now < 0 { return false, false } + if !executorDirectChannelInboxIdle(driver) { + if !wakeableExecutorRunCursor(driver) { + return false, false + } + return false, true + } if runnableForOSThreadOwner(driver.p) || !allowEmpty && !HasWaiting(driver.p) { return false, true } @@ -596,7 +787,8 @@ func prepareExecutorSleepAt( _ = leaveExecutorIdle(driver) return false, false } - hasWork := scan.completed != 0 || runnableForOSThreadOwner(driver.p) || + hasWork := scan.completed != 0 || !executorDirectChannelInboxIdle(driver) || + runnableForOSThreadOwner(driver.p) || driver.sources.pending(driver.p) || driver.registry.ObserveRequested(driver.handle) || preemptLoad(&driver.p.schedule) != scheduleIdle if hasWork { @@ -633,9 +825,16 @@ func PrepareExecutorStandbyAt(driver *ExecutorDriver, now int64) (prepared bool, // preparation and restores the active driver. func CommitExecutorSleepAt(driver *ExecutorDriver, now int64) (sleep bool, deadline int64, hasDeadline, ok bool) { if !validExecutorDriver(driver) || !driver.sources.usesMonotonicTime() || - driver.state != executorDriverIdlePreparing || !emptyExecutorRunCursor(driver) || !idleExecutorScheduler(driver.p) { + driver.state != executorDriverIdlePreparing || driver.run != (executorRunCursor{}) || + !emptyOwnerLocalCompletion(&driver.local) || !idleExecutorScheduler(driver.p) { return false, 0, false, false } + if !executorDirectChannelInboxIdle(driver) { + if !wakeableExecutorRunCursor(driver) || !leaveExecutorIdleForRun(driver) { + return false, 0, false, false + } + return false, 0, false, true + } if now < driver.prepareNow { _ = leaveExecutorIdle(driver) return false, 0, false, false @@ -646,7 +845,8 @@ func CommitExecutorSleepAt(driver *ExecutorDriver, now int64) (sleep bool, deadl _ = leaveExecutorIdle(driver) return false, 0, false, false } - hasWork := scan.completed != 0 || runnableForOSThreadOwner(driver.p) || + hasWork := scan.completed != 0 || !executorDirectChannelInboxIdle(driver) || + runnableForOSThreadOwner(driver.p) || driver.sources.pending(driver.p) || driver.registry.ObserveRequested(driver.handle) || preemptLoad(&driver.p.schedule) != scheduleIdle if hasWork { @@ -673,7 +873,7 @@ func CommitExecutorSleepAt(driver *ExecutorDriver, now int64) (sleep bool, deadl func wakeExecutorRun(driver *ExecutorDriver, now int64, withDeadline bool) bool { if !validExecutorDriver(driver) || driver.sources.usesMonotonicTime() != withDeadline || - driver.state != executorDriverSleeping || !emptyExecutorRunCursor(driver) || + driver.state != executorDriverSleeping || !wakeableExecutorRunCursor(driver) || !idleExecutorScheduler(driver.p) || withDeadline && now < 0 { return false } diff --git a/runtime/internal/coro/executor_driver_test.go b/runtime/internal/coro/executor_driver_test.go index 3fd167060c..623430f292 100644 --- a/runtime/internal/coro/executor_driver_test.go +++ b/runtime/internal/coro/executor_driver_test.go @@ -96,6 +96,10 @@ func finishReadyDriverTasks(t *testing.T, p *P, tasks map[*G]*yieldingTestG) { func TestExecutorDriverBindCloseLifecycle(t *testing.T) { p := new(P) driver, registry, handle := bindTestExecutorDriver(t, p) + slot, ok := executorSlot(registry, handle) + if !ok || driver.requestGate != &slot.gate { + t.Fatal("bound driver did not retain its exact stable request gate") + } if RequestSchedule(p) || preemptLoad(&p.schedule) != scheduleIdle { t.Fatal("legacy P request entered a bound executor") } @@ -142,6 +146,22 @@ func TestExecutorDriverHotHeaderDefersDeepCatalogAndPollAudits(t *testing.T) { t.Fatal("complete driver audit accepted an invalid logical poll cursor") } driver.poll = executorPollTransaction{} + + // Owner-local completion is another selected-work cursor. Its exact + // publication/resolution gates and complete diagnostics validate payload; + // an unrelated managed-resume observation keeps the immutable driver hot + // header independent of dormant local queue state. + driver.local.resolve = publishedEpochResolveCursor{phase: publishedEpochResolveDiscover} + if !validExecutorDriverHeader(driver) { + t.Fatal("hot header inspected the owner-local completion cursor") + } + if validExecutorDriver(driver) { + t.Fatal("complete driver audit accepted an invalid owner-local completion cursor") + } + if pending, ok := ExecutorRunManagedResumePending(driver); !ok || pending { + t.Fatalf("observational hot gate over local cursor damage = (%t, %t)", pending, ok) + } + driver.local = ownerLocalCompletionCursor{} if !validExecutorDriver(driver) { t.Fatal("restored driver failed complete audit") } @@ -217,6 +237,114 @@ func TestExecutorRunWakeDefersSourceService(t *testing.T) { closeTestExecutorDriver(t, driver) } +func TestExecutorRunWakeAcceptsDirectChannelArrivalAfterSleep(t *testing.T) { + p := new(P) + driver, registry, _, handle := bindTestExecutorDriverWithTimers(t, p) + prepared, ok := PrepareExecutorStandbyAt(driver, 10) + if !ok || !prepared { + t.Fatalf("prepare direct-channel standby = (%t, %t)", prepared, ok) + } + sleep, deadline, hasDeadline, ok := CommitExecutorSleepAt(driver, 11) + if !ok || !sleep || hasDeadline || deadline != 0 { + t.Fatalf("commit direct-channel standby = (%t, %d, %t, %t)", sleep, deadline, hasDeadline, ok) + } + + completion := &DirectChannelCompletion{ + owner: driver, + route: driver.route, + state: uint32(directChannelCompletionMatched), + } + if !PublishExecutorDirectChannelCompletion(driver, completion) { + t.Fatal("publish direct-channel completion after executor sleep") + } + if request := registry.Request(handle); request != ExecutorRequestIdleWake { + t.Fatalf("request sleeping executor after direct completion = %d", request) + } + if !WakeExecutorAt(driver, 12) || driver.state != executorDriverActive || + !driver.run.sourceMore { + t.Fatalf("wake over direct-channel arrival: state=%d run=%+v", driver.state, driver.run) + } + if got, takeOK := takeExecutorDirectChannelCompletion(driver); !takeOK || got != completion { + t.Fatalf("take post-sleep direct completion = (%p, %t), want (%p, true)", got, takeOK, completion) + } + drainTimerAwareExecutorRunSources(t, driver, 13) + closeTestExecutorDriver(t, driver) +} + +func TestPrepareExecutorStandbyDefersDirectChannelIngress(t *testing.T) { + p := new(P) + driver, _, _, _ := bindTestExecutorDriverWithTimers(t, p) + completion := &DirectChannelCompletion{ + owner: driver, + route: driver.route, + state: uint32(directChannelCompletionMatched), + } + if !PublishExecutorDirectChannelCompletion(driver, completion) { + t.Fatal("publish direct-channel completion before standby") + } + if prepared, ok := PrepareExecutorStandbyAt(driver, 10); !ok || prepared { + t.Fatalf("standby over direct-channel ingress = (%t, %t), want (false, true)", prepared, ok) + } + if got, ok := takeExecutorDirectChannelCompletion(driver); !ok || got != completion { + t.Fatalf("take deferred standby completion = (%p, %t), want (%p, true)", got, ok, completion) + } + closeTestExecutorDriver(t, driver) +} + +func TestCommitExecutorStandbyDefersDirectChannelIngress(t *testing.T) { + p := new(P) + driver, _, _, _ := bindTestExecutorDriverWithTimers(t, p) + if prepared, ok := PrepareExecutorStandbyAt(driver, 10); !ok || !prepared { + t.Fatalf("prepare standby before direct ingress = (%t, %t)", prepared, ok) + } + completion := &DirectChannelCompletion{ + owner: driver, + route: driver.route, + state: uint32(directChannelCompletionMatched), + } + if !PublishExecutorDirectChannelCompletion(driver, completion) { + t.Fatal("publish direct-channel completion during standby preparation") + } + sleep, deadline, hasDeadline, ok := CommitExecutorSleepAt(driver, 11) + if !ok || sleep || deadline != 0 || hasDeadline || driver.state != executorDriverActive || + !driver.run.sourceMore { + t.Fatalf("commit standby over direct ingress = (%t, %d, %t, %t), state=%d run=%+v", + sleep, deadline, hasDeadline, ok, driver.state, driver.run) + } + if got, ok := takeExecutorDirectChannelCompletion(driver); !ok || got != completion { + t.Fatalf("take commit-deferred completion = (%p, %t), want (%p, true)", got, ok, completion) + } + drainTimerAwareExecutorRunSources(t, driver, 12) + closeTestExecutorDriver(t, driver) +} + +func TestExecutorStandbySourceScanAllowsDirectChannelIngress(t *testing.T) { + p := new(P) + driver, _, _, _ := bindTestExecutorDriverWithTimers(t, p) + if !driver.registry.ArmIdle(driver.handle) { + t.Fatal("arm executor idle before source scan") + } + completion := &DirectChannelCompletion{ + owner: driver, + route: driver.route, + state: uint32(directChannelCompletionMatched), + } + if !PublishExecutorDirectChannelCompletion(driver, completion) { + t.Fatal("publish direct-channel completion after idle arm") + } + if scan, ok := publishExecutorSourcesAt(driver, 10, true); !ok || scan != (executorSourceScan{}) { + t.Fatalf("source scan over direct-channel ingress = (%+v, %t)", scan, ok) + } + if !leaveExecutorIdleForRun(driver) { + t.Fatal("leave idle after direct-channel ingress") + } + if got, ok := takeExecutorDirectChannelCompletion(driver); !ok || got != completion { + t.Fatalf("take scan-racing completion = (%p, %t), want (%p, true)", got, ok, completion) + } + drainTimerAwareExecutorRunSources(t, driver, 11) + closeTestExecutorDriver(t, driver) +} + func TestPrepareExecutorStandbyDefersRacingRequest(t *testing.T) { p := new(P) driver, registry, _, handle := bindTestExecutorDriverWithTimers(t, p) diff --git a/runtime/internal/coro/executor_fleet.go b/runtime/internal/coro/executor_fleet.go index 69192ece0f..0436c9547a 100644 --- a/runtime/internal/coro/executor_fleet.go +++ b/runtime/internal/coro/executor_fleet.go @@ -556,16 +556,14 @@ func restoreRunnableDemandAfterFailedClaim(slot *executorFleetSlot) bool { // DistributePNeutralRunnable services at most one global demand from an exact // source owner after a stable physical action. A source normally exports about // half of its local runnable queue, bounded by one destination mailbox, so one -// continuation remains local and a lone yielding G cannot bounce between idle -// Ps. Source links remain owner-only: an idle route publishes only a scalar -// demand and never concurrently reads or mutates the victim queue. The only -// single-runnable exception is a never-run initial frame, derived directly -// from frozen G/frame state rather than a target-owned spawn pointer. It may -// use an active route even before that route publishes demand: otherwise a -// child spawned immediately before a non-safepointed compute loop could never -// reach an idle P. Target selection scans the fixed route catalog beginning -// after the source route; a demanded target is protected by its route producer -// lease until mailbox publication and executor request have both completed. +// continuation remains local and a lone G cannot bounce between idle Ps. A +// Source links remain owner-only: an idle route publishes only a scalar demand +// and never concurrently reads or mutates the victim queue. Requiring that +// demand for every transfer is also a physical-service capability: a logical +// route may be bound before its target starts an M, host turn, or interrupt +// executor. Target selection scans the fixed route catalog beginning after the +// source route; a demanded target is protected by its route producer lease +// until mailbox publication and executor request have both completed. // // An empty valid result is ordinary: there was no demand, no surplus, or every // demanded mailbox was transiently contended/full. ok=false denotes a broken @@ -586,10 +584,10 @@ func (fleet *ExecutorFleet) DistributePNeutralRunnable( if !stableRunnableTransferP(source) { return RunnableDistribution{}, true } - batchLimit := source.readyCount / 2 if source.readyCount == 1 { - batchLimit = 1 + return RunnableDistribution{}, true } + batchLimit := source.readyCount / 2 if batchLimit > RunnableTransferMailboxCapacity { batchLimit = RunnableTransferMailboxCapacity } @@ -598,11 +596,6 @@ func (fleet *ExecutorFleet) DistributePNeutralRunnable( if prepared == 0 { return RunnableDistribution{}, true } - candidate := candidates[0] - initialCandidate := initialPNeutralRunnableState(candidate) - if source.readyCount == 1 && !initialCandidate { - return RunnableDistribution{}, true - } for offset := uint32(1); offset < ExecutorFleetCapacity; offset++ { index := (sourceHandle.Route - 1 + offset) % ExecutorFleetCapacity target := &fleet.slots[index] @@ -642,34 +635,6 @@ func (fleet *ExecutorFleet) DistributePNeutralRunnable( } return distribution, distribution.Valid() } - if initialCandidate { - var initial [RunnableTransferMailboxCapacity]*G - initial[0] = candidate - for offset := uint32(1); offset < ExecutorFleetCapacity; offset++ { - index := (sourceHandle.Route - 1 + offset) % ExecutorFleetCapacity - target := &fleet.slots[index] - if preemptLoad(&target.state) != uint32(executorFleetSlotActive) || - target.handle == sourceHandle { - continue - } - id, count, request, published := fleet.publishPreparedPNeutralRunnableBatchAndRequest( - target.handle, - source, - &initial, - 1, - ) - if !published { - continue - } - distribution = RunnableDistribution{ - Target: target.handle, - Transfer: id, - Count: count, - Request: request, - } - return distribution, distribution.Valid() - } - } return RunnableDistribution{}, true } @@ -832,6 +797,19 @@ func (fleet *ExecutorFleet) RequestTimerExecutor(route RouteID) ExecutorRequestR return fleet.routes.RequestTimerExecutor(route) } +// RequestExecutor wakes the exact fleet route for a fact already published in +// executor-owned storage rather than a source catalog. +func (fleet *ExecutorFleet) RequestExecutor(handle ExecutorFleetHandle) ExecutorRequestResult { + if fleet == nil || !handle.Valid() { + return ExecutorRequestInvalid + } + route, ok := handle.RouteID() + if !ok { + return ExecutorRequestInvalid + } + return fleet.routes.RequestExecutor(route, handle.Executor) +} + // RequestChannelExecutor routes the wake half of an already committed typed // channel rendezvous. The Channel source fact is published by the hchan // transaction before this call; this method only resolves and requests the diff --git a/runtime/internal/coro/executor_fleet_test.go b/runtime/internal/coro/executor_fleet_test.go index 4e77853224..498aa30288 100644 --- a/runtime/internal/coro/executor_fleet_test.go +++ b/runtime/internal/coro/executor_fleet_test.go @@ -651,7 +651,7 @@ func TestExecutorFleetDemandDistributesSurplusWithoutBouncingLastRunnable(t *tes } } -func TestExecutorFleetDemandSharesSingleInitialButNotSingleYielded(t *testing.T) { +func TestExecutorFleetDemandKeepsSingleRunnableLocal(t *testing.T) { t.Run("initial", func(t *testing.T) { fleet := new(ExecutorFleet) source := bindExecutorFleetManualFixture(t, fleet) @@ -661,12 +661,30 @@ func TestExecutorFleetDemandSharesSingleInitialButNotSingleYielded(t *testing.T) t.Fatal("prepare single initial demand") } distribution, ok := fleet.DistributePNeutralRunnable(source.handle, source.p) - if !ok || !distribution.Valid() || distribution.Target != target.handle || - distribution.Count != 1 || - source.p.readyHead != nil || source.p.readyTail != nil { - t.Fatalf("single initial distribution = %+v/%t source=(%p,%p)", + if !ok || distribution != (RunnableDistribution{}) || + source.p.readyHead != task.g || source.p.readyTail != task.g || !task.g.queued { + t.Fatalf("single initial runnable migrated = %+v/%t source=(%p,%p)", distribution, ok, source.p.readyHead, source.p.readyTail) } + _ = target + }) + t.Run("initial-surplus-without-demand", func(t *testing.T) { + fleet := new(ExecutorFleet) + source := bindExecutorFleetManualFixture(t, fleet) + target := bindExecutorFleetManualFixture(t, fleet) + first := newYieldingTestG(t, "fleet-initial-surplus-first") + second := newYieldingTestG(t, "fleet-initial-surplus-second") + if !Enqueue(source.p, first.g) || !Enqueue(source.p, second.g) { + t.Fatal("prepare initial surplus without target demand") + } + distribution, ok := fleet.DistributePNeutralRunnable(source.handle, source.p) + if !ok || distribution != (RunnableDistribution{}) || + source.p.readyHead != first.g || source.p.readyTail != second.g || + source.p.readyCount != 2 || !first.g.queued || !second.g.queued { + t.Fatalf("initial surplus moved without physical-service demand = %+v/%t source=(%p,%p,%d)", + distribution, ok, source.p.readyHead, source.p.readyTail, source.p.readyCount) + } + _ = target }) t.Run("yielded", func(t *testing.T) { fleet := new(ExecutorFleet) diff --git a/runtime/internal/coro/executor_progress.go b/runtime/internal/coro/executor_progress.go index 6d957859fa..4d6e092c21 100644 --- a/runtime/internal/coro/executor_progress.go +++ b/runtime/internal/coro/executor_progress.go @@ -73,12 +73,16 @@ const ( executorCatalogDone ) -// Timer expiry and reactor readiness commonly arrive in bursts. Keeping this -// quantum small amortizes runner/validation dispatch without turning one host -// reduction into an unbounded catalog walk on embedded or single-threaded -// targets. Other source types retain one-entry reductions because they may run -// stronger admission or control protocols per entry. -const executorCatalogBatchQuantum uint32 = 8 +// Timer expiry and reactor readiness commonly arrive in bursts. A timer +// reduction consumes at most one fixed catalog page: this amortizes the +// runner/validation boundary while retaining the same allocation-free bound +// on embedded and single-threaded targets. Poll keeps the smaller quantum +// because each reactor entry may carry a stronger OS-facing protocol. Other +// source types retain one-entry reductions. +const ( + executorTimerCatalogBatchQuantum uint32 = TimerRegistrationPageCapacity + executorPollCatalogBatchQuantum uint32 = 8 +) // executorPollTransaction is scheduler-owner-only continuation state. It has // no callback-visible pointer and is embedded at a stable address in the @@ -314,8 +318,11 @@ func publishExecutorCatalogEntry(driver *ExecutorDriver) bool { transaction.deadline, transaction.hasDeadline = deadline, true } transaction.cursor++ - if uint32(transaction.cursor) == limit && !transaction.advanceCatalogSource(sources) { - return false + if uint32(transaction.cursor) == limit { + if !sources.timers.commitTimerRegistrationMinimum(transaction.deadline, transaction.hasDeadline) || + !transaction.advanceCatalogSource(sources) { + return false + } } case executorCatalogPoll: if index == 0 && !sources.poll.beginDrainPass(p) { @@ -418,9 +425,28 @@ func publishExecutorCatalogReduction(driver *ExecutorDriver) bool { return false } source := driver.poll.source + if source == executorCatalogTimers && driver.poll.cursor == 0 { + deadline, hasDeadline, skip, ok := timerRegistrationFastDeadline( + driver.sources.timers, + driver.p, + driver.poll.now, + ) + if !ok { + return false + } + if skip { + if hasDeadline && (!driver.poll.hasDeadline || deadline < driver.poll.deadline) { + driver.poll.deadline, driver.poll.hasDeadline = deadline, true + } + return driver.poll.advanceCatalogSource(&driver.sources) + } + } limit := uint32(1) - if source == executorCatalogTimers || source == executorCatalogPoll { - limit = executorCatalogBatchQuantum + switch source { + case executorCatalogTimers: + limit = executorTimerCatalogBatchQuantum + case executorCatalogPoll: + limit = executorPollCatalogBatchQuantum } for visited := uint32(0); visited < limit; visited++ { if !publishExecutorCatalogEntry(driver) { @@ -487,6 +513,20 @@ func pollBoundExecutorSliceAt(driver *ExecutorDriver, now int64, withDeadline bo used := uint32(0) for used < budget { transaction := &driver.poll + // A typed cleanup is a direct-runtime boundary, not another core + // reduction. A batched runner entry may have reached it after consuming + // earlier source work; return that bounded progress so the next unified + // step can expose ExecutorRunStepMaterialize. A caller which enters with + // the cleanup already pending violated the runner selector contract. + if _, pending := pendingResumeCleanupStepForCursor(&transaction.resolve); pending { + if used == 0 { + return transaction.total, ExecutorPollProgress{}, false + } + progress, progressOK := executorProgressFromScan( + transaction.total, used, budget, false, true, false, + ) + return transaction.total, progress, progressOK + } switch transaction.phase { case executorPollEpochAPublish, executorPollEpochBPublish: if transaction.resampleNow { diff --git a/runtime/internal/coro/executor_progress_test.go b/runtime/internal/coro/executor_progress_test.go index b9f0975b13..6f5e73cee0 100644 --- a/runtime/internal/coro/executor_progress_test.go +++ b/runtime/internal/coro/executor_progress_test.go @@ -171,7 +171,7 @@ func TestExecutorPollReductionBatchesDueTimers(t *testing.T) { t.Fatal("bind batched timer executor") } - const count = executorCatalogBatchQuantum + const count = executorTimerCatalogBatchQuantum parks := make([]*timerV2TestPark, int(count)) handles := make([]TimerRegistrationHandle, int(count)) for index := 0; index < int(count); index++ { @@ -184,8 +184,10 @@ func TestExecutorPollReductionBatchesDueTimers(t *testing.T) { uint32(index+1), 1, ) - if !attached { - t.Fatalf("reserve batched timer %d", index) + wantCursor := uint32(index+1) % TimerRegistrationConfiguredCapacity(timers) + if !attached || timer.Slot != uint32(index+1) || timers.reserveCursor != wantCursor { + t.Fatalf("reserve batched timer %d = (%+v,%t), cursor=%d want=%d", + index, timer, attached, timers.reserveCursor, wantCursor) } commitTimerV2TestPark(t, p, park) parks[index], handles[index] = park, timer diff --git a/runtime/internal/coro/executor_resume_handoff_test.go b/runtime/internal/coro/executor_resume_handoff_test.go index 24bbde1fe3..7afe5b16e5 100644 --- a/runtime/internal/coro/executor_resume_handoff_test.go +++ b/runtime/internal/coro/executor_resume_handoff_test.go @@ -156,10 +156,11 @@ func TestExecutorResumeHandoffRunsReplacementAndRestoresExactResume(t *testing.T t.Fatalf("request replacement source transaction = %d", result) } sourceSteps := 0 + sourceUsed := uint32(0) for { sourceStep, advanced := NextExecutorRunStep(driver) if !advanced || sourceStep.Kind != ExecutorRunStepSource || - sourceStep.Poll.Used != 1 { + sourceStep.Poll.Used == 0 || sourceStep.Poll.Used > executorRunSourceBatchQuantum { t.Fatalf( "replacement source step %d = (%+v, %t)", sourceSteps, @@ -168,6 +169,7 @@ func TestExecutorResumeHandoffRunsReplacementAndRestoresExactResume(t *testing.T ) } sourceSteps++ + sourceUsed += sourceStep.Poll.Used if sourceStep.Poll.Complete { break } @@ -177,10 +179,14 @@ func TestExecutorResumeHandoffRunsReplacementAndRestoresExactResume(t *testing.T t.Fatal("returned across replacement source A/ack/B transaction") } } - if sourceSteps < 2 || !ExecutorResumeHandoffReturnable(driver) { + if want, budgetOK := MinExecutorPollBudget(driver); !budgetOK || sourceUsed != want || + !ExecutorResumeHandoffReturnable(driver) { t.Fatalf( - "replacement source transaction = %d steps, returnable=%t", + "replacement source transaction = %d reductions in %d steps, want=(%d,%t), returnable=%t", + sourceUsed, sourceSteps, + want, + budgetOK, ExecutorResumeHandoffReturnable(driver), ) } diff --git a/runtime/internal/coro/explicit_status.go b/runtime/internal/coro/explicit_status.go index 9a2d3bc47d..2eeb6e15d3 100644 --- a/runtime/internal/coro/explicit_status.go +++ b/runtime/internal/coro/explicit_status.go @@ -147,7 +147,7 @@ func PrepareExplicitStatus( } if status != ExplicitStatusPanic || typeWord == nil || handle == nil || header == nil || header.Flags != 0 || g.state != GRunning || g.active == nil || g.root == nil || g.runP == nil || - g.pending.kind != pendingNone || g.pending.from != nil || g.pending.target != nil || + g.pending.kind != pendingNone || g.pending.directChannel || g.pending.from != nil || g.pending.target != nil || g.destroyTarget != nil || g.destroyRoot || g.queued || g.nextReady != nil || g.waiting || g.spawnChild != nil || g.spawnParent != nil || g.spawnP != nil || !releasableParkState(&g.park) || g.park.taskCancelPhase == taskCancelRequested || g.panicUnwind { diff --git a/runtime/internal/coro/frame.go b/runtime/internal/coro/frame.go index 5e96aa792b..3601faf979 100644 --- a/runtime/internal/coro/frame.go +++ b/runtime/internal/coro/frame.go @@ -52,7 +52,11 @@ type FrameDescriptorV1 struct { File string } -const FrameDescriptorTraceHiddenV1 uint32 = 1 << 0 +const ( + FrameDescriptorTraceHiddenV1 uint32 = 1 << 0 + FrameDescriptorNoRuntimeContextV1 uint32 = 1 << 1 + frameDescriptorAllowedFlagsV1 = FrameDescriptorTraceHiddenV1 | FrameDescriptorNoRuntimeContextV1 +) // SuspendReason describes why a coroutine returned control to its scheduler. type SuspendReason uint16 @@ -89,6 +93,37 @@ const ( FrameDestroyed ) +// frameRuntimeContextMode is a publication-time cache of the immutable +// compiler descriptor capability. Generated frames are always published via +// PublishFrameV2/V3, so their hot resume path never has to decode Go strings +// and descriptor flags again. Legacy/test PublishFrame callers retain the +// unknown value and are validated from the live descriptor when queried. +type frameRuntimeContextMode uint8 + +const ( + frameRuntimeContextUnknown frameRuntimeContextMode = iota + frameRuntimeContextRequired + frameRuntimeContextNotRequired +) + +func descriptorRuntimeContextMode(descriptor unsafe.Pointer) (frameRuntimeContextMode, bool) { + if descriptor == nil { + return frameRuntimeContextUnknown, false + } + value := (*FrameDescriptorV1)(descriptor) + // Runtime-context installation is an execution capability, independent of + // optional trace names. Legacy PublishFrame callers may legally publish an + // anonymous descriptor; panic-trace readers retain their separate, stricter + // Function validation at the point where that diagnostic string is needed. + if value.Version != 1 || value.Flags&^frameDescriptorAllowedFlagsV1 != 0 { + return frameRuntimeContextUnknown, false + } + if value.Flags&FrameDescriptorNoRuntimeContextV1 != 0 { + return frameRuntimeContextNotRequired, true + } + return frameRuntimeContextRequired, true +} + const gMagic uint32 = 0x434f524f // "CORO" type pendingKind uint8 @@ -108,9 +143,16 @@ const ( ) type pendingTransition struct { - kind pendingKind - from *Frame - target *Frame + kind pendingKind + // directChannel is a one-resume capability produced only by the compact + // compiler/runtime channel transaction. It certifies that from, its wait + // record, and the adjacent ParkState were constructed together before the + // hchan waiter became visible. The bit is owner-only and is consumed on the + // immediately following llvm.coro.resume return; generic parks leave it + // clear and retain their complete graph audit. + directChannel bool + from *Frame + target *Frame } // Frame is scheduler-owned metadata. It lives at the beginning of the same @@ -141,14 +183,32 @@ type Frame struct { allocationSize uintptr panicLine uint32 state FrameState + runtimeContext frameRuntimeContextMode // retainPanicTrace is set only after a managed child publishes // CompletionPanic. It occupies the padding before parent on pointer-aligned // targets and transfers the destroyed allocation to the task trace chain. retainPanicTrace bool - parent *Frame - next *Frame + // borrowedStorage identifies scheduler metadata injected into an LLVM + // coroutine frame by the compiler. Its LLVM storage is owned by the exact + // static parent frame, so llvm.coro.free deliberately skips the ordinary + // ReleaseFrame callback after CoroAnnotationElide. The runtime still links, + // validates, and destroys this metadata through the same Frame protocol. + borrowedStorage bool + parent *Frame + next *Frame } +// BorrowedFrameStorageV2 is the compiler/runtime capacity contract for one +// Frame stored inside an elidable coroutine. It is intentionally opaque to +// generated code: the compiler only reserves and forwards this pointer, while +// the runtime initializes and remains the sole owner of Frame's private +// layout. Twenty pointer +// words cover both wasm32 and native layouts with expansion room; the compile- +// time assertion fails if runtime metadata ever outgrows the ABI capacity. +type BorrowedFrameStorageV2 [20]uintptr + +var _ [int(unsafe.Sizeof(BorrowedFrameStorageV2{})) - int(unsafe.Sizeof(Frame{}))]byte + // ValidG reports whether g has been initialized as a coroutine task. func ValidG(g *G) bool { return g != nil && g.magic == gMagic @@ -205,9 +265,10 @@ func Zero(ptr unsafe.Pointer, size uintptr) { } } -// RegisterFrame initializes a combined allocation and links it into g. raw -// must be the base returned by the target runtime allocator, and total must be -// exactly FrameAllocationSize(size, align). +// RegisterFrame initializes a zero-filled combined allocation and links it +// into g. raw must be the base returned by the target runtime allocator, whose +// contract guarantees the complete range is cleared, and total must be exactly +// FrameAllocationSize(size, align). func RegisterFrame(g *G, raw unsafe.Pointer, total, size, align uintptr, descriptor unsafe.Pointer) (unsafe.Pointer, bool) { want, ok := FrameAllocationSize(size, align) if !ValidG(g) || raw == nil || descriptor == nil || !ok || total != want || @@ -218,7 +279,6 @@ func RegisterFrame(g *G, raw unsafe.Pointer, total, size, align uintptr, descrip if !ok { return nil, false } - Zero(raw, total) frame := (*Frame)(raw) frame.owner = g frame.storage = storage @@ -280,6 +340,76 @@ func PublishFrame(g *G, handle unsafe.Pointer, header *HeaderV1, storage unsafe. return true } +// PublishFrameV2 binds either an ordinary dynamically allocated LLVM frame or +// compiler-injected metadata for an allocation-elided static child. metadata +// is ignored on the dynamic V1-compatible path; keeping that path unchanged +// makes the first elision gate reversible and isolates its lifetime rules. +func PublishFrameV2( + g *G, handle unsafe.Pointer, header *HeaderV1, storage, metadata unsafe.Pointer, +) bool { + if header == nil { + return false + } + mode, modeOK := descriptorRuntimeContextMode(header.Descriptor) + if !modeOK { + return false + } + if storage != nil { + if metadata == nil || !PublishFrame(g, handle, header, storage) { + return false + } + frame := FrameFromStorage(storage) + if frame == nil { + return false + } + frame.runtimeContext = mode + return true + } + if !ValidG(g) || handle == nil || header == nil || metadata == nil || + metadata == unsafe.Pointer(g) || metadata == unsafe.Pointer(header) || + uintptr(metadata)%unsafe.Alignof(Frame{}) != 0 || + header.G != unsafe.Pointer(g) || header.Descriptor == nil || + header.Lifecycle != uint16(FrameInitialSuspended) || + header.SuspendReason != uint16(SuspendNone) || findFrame(g, handle) != nil { + return false + } + Zero(metadata, unsafe.Sizeof(Frame{})) + frame := (*Frame)(metadata) + frame.owner = g + frame.handle = handle + frame.header = header + frame.descriptor = header.Descriptor + frame.state = FrameInitialSuspended + frame.runtimeContext = mode + frame.borrowedStorage = true + frame.next = g.frames + g.frames = frame + header.AllocationBase = metadata + return true +} + +// PublishFrameV3 initializes the complete compiler/runtime header and then +// publishes either dynamic storage or compiler-borrowed metadata. Keeping the +// initialization in this shared helper makes coroutine ramps small: generated +// code supplies only immutable descriptor/result operands and never duplicates +// the scheduler header's ten-field initialization sequence. +func PublishFrameV3( + g *G, handle unsafe.Pointer, header *HeaderV1, storage, metadata, + descriptor, resultSlot unsafe.Pointer, +) bool { + if !ValidG(g) || handle == nil || header == nil || metadata == nil || descriptor == nil { + return false + } + *header = HeaderV1{ + G: unsafe.Pointer(g), + Descriptor: descriptor, + ResultSlot: resultSlot, + SuspendReason: uint16(SuspendNone), + Lifecycle: uint16(FrameInitialSuspended), + } + return PublishFrameV2(g, handle, header, storage, metadata) +} + // PrepareAwait records a parent-to-child handoff. It never resumes either // coroutine; only the runtime driver may perform handle operations requested // by the scheduler action protocol. @@ -306,6 +436,53 @@ func prepareAwait( return true } +// compilerReleasableParkState consumes the owner-only certificate established +// when a park reaches a source-free phase. Compiler hooks run inside one +// already authenticated resume episode, so replaying the complete union audit +// at every ordinary Go call does not establish a new ownership boundary. +// Compatibility and lifecycle APIs continue to use releasableParkState. +func compilerReleasableParkState(state *ParkState) bool { + if state == nil || state.resolving || state.directChannel || state.attached != 0 || state.head != nil || + !validTaskCancelState(state.taskCancelKind, state.taskCancelPhase) { + return false + } + switch state.phase { + case parkIdle, parkConsumed, parkDelivered: + return true + default: + return false + } +} + +// PrepareAwaitCompletionCompiler is the private compiler-hook lane for an +// ordinary managed call. The child ramp has just published the newest frame +// and the current frame is the exact active parent, so both frame identities +// are available in O(1). If that certificate is absent, retain the complete +// compatibility validator and fail-closed behavior. +func PrepareAwaitCompletionCompiler(g *G, parentHandle, childHandle unsafe.Pointer) bool { + if ValidG(g) && parentHandle != nil && childHandle != nil && resumeGateTaken(g) && + g.pending.kind == pendingNone && g.spawnChild == nil && + compilerReleasableParkState(&g.park) { + parent, child := g.active, g.frames + if parent != nil && child != nil && parent != child && + parent.handle == parentHandle && child.handle == childHandle && + parent.owner == g && child.owner == g && + parent.header != nil && child.header != nil && + parent.state == FrameActive && child.state == FrameInitialSuspended && + parent.header.SuspendReason == uint16(SuspendCall) && + parent.header.Lifecycle == uint16(FrameSuspended) && + child.header.Parent == parentHandle && child.parent == nil && + emptyCompletionRecord(&parent.completion) { + parent.completion.child = child.handle + parent.completion.status = completionArmed + child.parent = parent + g.pending = pendingTransition{kind: pendingAwait, from: parent, target: child} + return true + } + } + return PrepareAwaitCompletion(g, parentHandle, childHandle) +} + // PrepareAwait preserves the original V1 scheduler transaction. It is kept for // adapters and tests that intentionally have no child-outcome transport. func PrepareAwait(g *G, parentHandle, childHandle unsafe.Pointer) bool { @@ -374,6 +551,31 @@ func PrepareCompleteStatus(g *G, handle unsafe.Pointer, header *HeaderV1, status return true } +// PrepareCompleteStatusCompiler handles the dominant ordinary-return suffix +// from compiler-generated code using the active-frame and parent-completion +// certificates already established by PrepareAwaitCompletionCompiler. Panic, +// Goexit, cancellation, roots, and any uncertain shape retain the complete +// status validator below. +func PrepareCompleteStatusCompiler(g *G, handle unsafe.Pointer, header *HeaderV1, status CompletionStatus) bool { + if status == CompletionReturn && ValidG(g) && resumeGateTaken(g) && + handle != nil && header != nil && g.pending.kind == pendingNone && + g.spawnChild == nil && compilerReleasableParkState(&g.park) && + g.park.taskCancelPhase != taskCancelRequested { + frame := g.active + if frame != nil && frame.parent != nil && frame.handle == handle && frame.header == header && + frame.owner == g && frame.state == FrameActive && + header.SuspendReason == uint16(SuspendFrameComplete) && + header.Lifecycle == uint16(FrameFinalSuspended) { + if awaitCompletionArmedForChild(frame) && + publishAwaitCompletion(frame.parent, CompletionReturn, nil, nil) { + g.pending = pendingTransition{kind: pendingComplete, from: frame} + return true + } + } + } + return PrepareCompleteStatus(g, handle, header, status) +} + // PrepareComplete preserves the normal-return V1 adapter contract. func PrepareComplete(g *G, handle unsafe.Pointer, header *HeaderV1) bool { return PrepareCompleteStatus(g, handle, header, CompletionReturn) @@ -475,6 +677,51 @@ func ReleaseFrame(g *G, storage unsafe.Pointer, size, align uintptr, descriptor return raw, total, true } +// CommitFrameDestroyV2 completes the physical destroy of an allocation-elided +// static frame. A dynamic frame has already been unlinked by ReleaseFrame and +// is accepted without a second mutation. The adapter invokes this immediately +// after every llvm.coro.destroy, before any logical Destroyed transition. +func CommitFrameDestroyV2(g *G, handle unsafe.Pointer) bool { + if !ValidG(g) || handle == nil || !gPreemptEnabledAtDepthZero(g) { + return false + } + frame := g.destroyTarget + if frame == nil { + // The dynamic free callback already consumed the exact target. A stale + // schedulable frame with the same handle would make that receipt invalid. + return findFrame(g, handle) == nil + } + if frame.handle != handle || !frame.borrowedStorage || frame.storage != nil || + frame.rawBase != nil || frame.allocationSize != 0 || frame.owner != g || + frame.header == nil || frame.parkWait != nil || frame.state != FrameDestroyPending || + frame.header.AllocationBase != unsafe.Pointer(frame) || + frame.header.Lifecycle != uint16(FrameDestroyPending) || !unlinkFrame(g, frame) { + return false + } + frame.state = FrameDestroyed + frame.panicLine = frame.header.Line + frame.header.Lifecycle = uint16(FrameDestroyed) + g.destroyTarget = nil + if retainDestroyedBorrowedPanicTraceFrame(g, frame) { + return true + } + Zero(unsafe.Pointer(frame), unsafe.Sizeof(Frame{})) + return true +} + +// CommitFrameDestroyCompiler consumes the receipt produced synchronously by +// the immediately preceding compiler-owned llvm.coro.destroy. A dynamic frame +// can clear destroyTarget only through ReleaseFrame after validating the exact +// storage, descriptor, size, lifecycle, and target. Allocation-elided frames +// retain destroyTarget and therefore use the complete unlink transaction. +func CommitFrameDestroyCompiler(g *G, handle unsafe.Pointer) bool { + if ValidG(g) && handle != nil && gPreemptEnabledAtDepthZero(g) && + g.destroyTarget == nil { + return true + } + return CommitFrameDestroyV2(g, handle) +} + // PanicTraceFrameSnapshot is the allocation-free diagnostic prefix retained // from one destroyed physical frame. Function and File point into immutable // descriptor storage emitted by the compiler. @@ -519,7 +766,7 @@ func ActiveTraceFrame(g *G) (PanicTraceFrameSnapshot, bool) { } descriptor := (*FrameDescriptorV1)(frame.descriptor) if descriptor.Version != 1 || - descriptor.Flags & ^FrameDescriptorTraceHiddenV1 != 0 || + descriptor.Flags & ^frameDescriptorAllowedFlagsV1 != 0 || len(descriptor.Function) == 0 { return PanicTraceFrameSnapshot{}, false } @@ -613,26 +860,31 @@ func ReplacePanicTrace(g *G, handle unsafe.Pointer) bool { return stagePanicTraceDiscard(g) } -func retainPanicTraceFrame( - g *G, - raw unsafe.Pointer, - total uintptr, - typeWord, dataWord unsafe.Pointer, +func retainPanicTraceFrameMetadata( + g *G, frame *Frame, raw unsafe.Pointer, total uintptr, typeWord, dataWord unsafe.Pointer, ) bool { - if !ValidG(g) || raw == nil || total == 0 || typeWord == nil || + if !ValidG(g) || frame == nil || typeWord == nil || g.destroyTarget != nil || g.panicTraceCount == ^uint32(0) || !emptyPanicTrace(g) && !activePanicTrace(g) { return false } - frame := (*Frame)(raw) - if frame.owner != g || frame.rawBase != raw || frame.allocationSize != total || + if frame.borrowedStorage { + if raw != nil || total != 0 || frame.rawBase != nil || frame.allocationSize != 0 || + frame.storage != nil { + return false + } + } else if raw == nil || total == 0 || frame.rawBase != raw || + frame.allocationSize != total || raw != unsafe.Pointer(frame) { + return false + } + if frame.owner != g || frame.state != FrameDestroyed || frame.next != nil || frame.descriptor == nil || !emptyCompletionRecord(&frame.completion) { return false } descriptor := (*FrameDescriptorV1)(frame.descriptor) if descriptor.Version != 1 || - descriptor.Flags & ^FrameDescriptorTraceHiddenV1 != 0 || + descriptor.Flags & ^frameDescriptorAllowedFlagsV1 != 0 || len(descriptor.Function) == 0 { return false } @@ -665,6 +917,39 @@ func retainPanicTraceFrame( return true } +func retainPanicTraceFrame( + g *G, + raw unsafe.Pointer, + total uintptr, + typeWord, dataWord unsafe.Pointer, +) bool { + if raw == nil { + return false + } + return retainPanicTraceFrameMetadata(g, (*Frame)(raw), raw, total, typeWord, dataWord) +} + +func retainDestroyedBorrowedPanicTraceFrame(g *G, frame *Frame) bool { + if frame == nil || !frame.borrowedStorage { + return false + } + if frame.retainPanicTrace && frame.parent != nil && + frame.parent.completion.status == CompletionPanic && + frame.parent.completion.child == frame.handle && + frame.parent.completion.typeWord != nil { + record := frame.parent.completion + return retainPanicTraceFrameMetadata( + g, frame, nil, 0, record.typeWord, record.dataWord, + ) + } + if g.panicUnwind && publishedPanicRecord(&g.panicRecord) { + return retainPanicTraceFrameMetadata( + g, frame, nil, 0, g.panicRecord.typeWord, g.panicRecord.dataWord, + ) + } + return false +} + // RetainPendingPanicTraceFrame transfers one managed child frame whose panic // remains owned by its resumed parent. The parent CompletionRecord supplies // the stable panic identity; later propagation appends ancestors, while an @@ -711,23 +996,35 @@ func TakeDiscardedPanicTraceFrame(g *G) (raw unsafe.Pointer, total uintptr, ok b if !ValidG(g) { return nil, 0, false } - if emptyPanicTrace(g) { - return nil, 0, true - } - if !stagedPanicTraceDiscard(g) { - return nil, 0, false - } - frame := g.panicTraceHead - if frame.owner != g || frame.rawBase != unsafe.Pointer(frame) || - frame.allocationSize == 0 || frame.state != FrameDestroyed || - frame.header != nil || frame.descriptor == nil { - return nil, 0, false + for { + if emptyPanicTrace(g) { + return nil, 0, true + } + if !stagedPanicTraceDiscard(g) { + return nil, 0, false + } + frame := g.panicTraceHead + if frame.owner != g || frame.state != FrameDestroyed || + frame.header != nil || frame.descriptor == nil { + return nil, 0, false + } + if frame.borrowedStorage { + if frame.rawBase != nil || frame.allocationSize != 0 || frame.storage != nil { + return nil, 0, false + } + } else if frame.rawBase != unsafe.Pointer(frame) || frame.allocationSize == 0 { + return nil, 0, false + } + g.panicTraceHead = frame.next + frame.next = nil + frame.parent = nil + frame.completion = CompletionRecord{} + if frame.borrowedStorage { + Zero(unsafe.Pointer(frame), unsafe.Sizeof(Frame{})) + continue + } + return frame.rawBase, frame.allocationSize, true } - g.panicTraceHead = frame.next - frame.next = nil - frame.parent = nil - frame.completion = CompletionRecord{} - return frame.rawBase, frame.allocationSize, true } // FirstPanicTraceFrame returns the opaque cursor for the deepest retained @@ -748,13 +1045,14 @@ func LoadPanicTraceFrame(g *G, cursor unsafe.Pointer) (snapshot PanicTraceFrameS return PanicTraceFrameSnapshot{}, nil, false } frame := (*Frame)(cursor) - if frame.owner != g || frame.rawBase != cursor || frame.allocationSize == 0 || - frame.state != FrameDestroyed || frame.header != nil || frame.descriptor == nil { + if frame.owner != g || frame.state != FrameDestroyed || frame.header != nil || frame.descriptor == nil || + frame.borrowedStorage && (frame.rawBase != nil || frame.allocationSize != 0 || frame.storage != nil) || + !frame.borrowedStorage && (frame.rawBase != cursor || frame.allocationSize == 0) { return PanicTraceFrameSnapshot{}, nil, false } descriptor := (*FrameDescriptorV1)(frame.descriptor) if descriptor.Version != 1 || - descriptor.Flags & ^FrameDescriptorTraceHiddenV1 != 0 || + descriptor.Flags & ^frameDescriptorAllowedFlagsV1 != 0 || len(descriptor.Function) == 0 { return PanicTraceFrameSnapshot{}, nil, false } diff --git a/runtime/internal/coro/frame_test.go b/runtime/internal/coro/frame_test.go index ae6f316918..3e4aa4ad00 100644 --- a/runtime/internal/coro/frame_test.go +++ b/runtime/internal/coro/frame_test.go @@ -55,6 +55,9 @@ func TestHeaderV1TargetNeutralLayout(t *testing.T) { } func TestFrameAllocationLayout(t *testing.T) { + if got, capacity := unsafe.Sizeof(Frame{}), unsafe.Sizeof(BorrowedFrameStorageV2{}); got > capacity { + t.Fatalf("Frame size = %d, borrowed ABI capacity = %d", got, capacity) + } for _, align := range []uintptr{1, 2, 4, 8, 16, 64} { total, ok := FrameAllocationSize(37, align) if !ok { @@ -89,6 +92,98 @@ func TestFrameAllocationLayout(t *testing.T) { } } +func TestBorrowedFrameV2PublishAndDestroy(t *testing.T) { + g := new(G) + if !InitG(g) { + t.Fatal("InitG failed") + } + handle := unsafe.Pointer(new(byte)) + descriptor := &FrameDescriptorV1{Version: 1, ResultAlign: 1, Function: "test.borrowed"} + header := &HeaderV1{ + G: unsafe.Pointer(g), + Descriptor: unsafe.Pointer(descriptor), + SuspendReason: uint16(SuspendNone), + Lifecycle: uint16(FrameInitialSuspended), + } + metadata := new(BorrowedFrameStorageV2) + for index := range metadata { + metadata[index] = ^uintptr(0) + } + if !PublishFrameV2(g, handle, header, nil, unsafe.Pointer(metadata)) { + t.Fatal("publish borrowed frame") + } + frame := (*Frame)(unsafe.Pointer(metadata)) + if g.frames != frame || frame.owner != g || frame.handle != handle || + frame.header != header || frame.storage != nil || frame.rawBase != nil || + !frame.borrowedStorage || frame.state != FrameInitialSuspended || + header.AllocationBase != unsafe.Pointer(frame) { + t.Fatalf("borrowed publication = %+v, header base=%p", frame, header.AllocationBase) + } + frame.state = FrameDestroyPending + header.Lifecycle = uint16(FrameDestroyPending) + g.destroyTarget = frame + if !CommitFrameDestroyV2(g, handle) { + t.Fatal("commit borrowed frame destroy") + } + if g.frames != nil || g.destroyTarget != nil || findFrame(g, handle) != nil { + t.Fatal("borrowed destroy retained scheduler ownership") + } + if *frame != (Frame{}) { + t.Fatalf("ordinary borrowed destroy did not clear metadata: %+v", frame) + } + if header.Lifecycle != uint16(FrameDestroyed) { + t.Fatalf("borrowed header lifecycle = %d, want destroyed", header.Lifecycle) + } +} + +func TestBorrowedFrameV3InitializesHeader(t *testing.T) { + g := new(G) + if !InitG(g) { + t.Fatal("InitG failed") + } + handle := unsafe.Pointer(new(byte)) + descriptor := &FrameDescriptorV1{Version: 1, ResultAlign: 1, Function: "test.borrowed.v3"} + resultSlot := unsafe.Pointer(new(uintptr)) + header := &HeaderV1{ + G: unsafe.Pointer(new(byte)), + Parent: unsafe.Pointer(new(byte)), + Descriptor: unsafe.Pointer(new(byte)), + AllocationBase: unsafe.Pointer(new(byte)), + ResultSlot: unsafe.Pointer(new(byte)), + SuspendReason: ^uint16(0), + Lifecycle: ^uint16(0), + StateID: ^uint32(0), + Line: ^uint32(0), + Flags: ^uint32(0), + } + metadata := new(BorrowedFrameStorageV2) + for index := range metadata { + metadata[index] = ^uintptr(0) + } + if !PublishFrameV3( + g, handle, header, nil, unsafe.Pointer(metadata), + unsafe.Pointer(descriptor), resultSlot, + ) { + t.Fatal("publish initialized borrowed frame") + } + frame := (*Frame)(unsafe.Pointer(metadata)) + if g.frames != frame || frame.owner != g || frame.handle != handle || frame.header != header || + frame.descriptor != unsafe.Pointer(descriptor) || !frame.borrowedStorage || + header.G != unsafe.Pointer(g) || header.Parent != nil || + header.Descriptor != unsafe.Pointer(descriptor) || header.AllocationBase != unsafe.Pointer(frame) || + header.ResultSlot != resultSlot || header.SuspendReason != uint16(SuspendNone) || + header.Lifecycle != uint16(FrameInitialSuspended) || header.StateID != 0 || + header.Line != 0 || header.Flags != 0 { + t.Fatalf("V3 borrowed publication frame=%+v header=%+v", frame, header) + } + frame.state = FrameDestroyPending + header.Lifecycle = uint16(FrameDestroyPending) + g.destroyTarget = frame + if !CommitFrameDestroyV2(g, handle) { + t.Fatal("commit initialized borrowed frame destroy") + } +} + type testFrame struct { handle unsafe.Pointer header *HeaderV1 diff --git a/runtime/internal/coro/inline_await.go b/runtime/internal/coro/inline_await.go index f37195c963..94a4a75b08 100644 --- a/runtime/internal/coro/inline_await.go +++ b/runtime/internal/coro/inline_await.go @@ -113,6 +113,23 @@ func validInlineAwaitParentDepth(g *G, frame *Frame, depth uint8) bool { return frame.parent != nil && validInlineAwaitEdge(frame.parent, frame) } +// compilerInlineAwaitParentDepth consumes the inductive edge owned by the +// immediately enclosing compiler hook. The complete ancestry walk remains at +// real suspension, event, and foreign-handoff boundaries. +func compilerInlineAwaitParentDepth(g *G, frame *Frame, depth uint8) bool { + if g == nil || g.runP == nil || frame == nil || depth > maxInlineAwaitDepth { + return false + } + p := g.runP + if depth == 0 { + return p.action.Kind == ActionResume && p.action.Flags == 0 && + p.action.Handle == frame.handle + } + parent := frame.parent + return parent != nil && parent.completion.child == frame.handle && + parent.state == FrameSuspended +} + // BeginInlineAwait consumes one already validated pendingAwait and activates // its initial-suspended child inside the current physical resume episode. A // depth refusal does not mutate the ordinary scheduler transaction. @@ -148,6 +165,40 @@ func BeginInlineAwait(g *G, parentHandle, childHandle unsafe.Pointer) InlineAwai return InlineAwaitStarted } +// BeginInlineAwaitCompiler consumes the exact pendingAwait written by the +// adjacent compiler prepare hook. It keeps the bounded-depth refusal and all +// externally observable mutations identical while avoiding a second full +// graph audit on the ordinary synchronous call path. +func BeginInlineAwaitCompiler(g *G, parentHandle, childHandle unsafe.Pointer) InlineAwaitDisposition { + if ValidG(g) && parentHandle != nil && childHandle != nil && resumeGateTaken(g) && + g.runP != nil && g.destroyTarget == nil && !g.destroyRoot && + g.spawnChild == nil && g.spawnParent == nil && g.spawnP == nil && + !g.waiting && compilerReleasableParkState(&g.park) { + p := g.runP + pending := g.pending + parent, child := pending.from, pending.target + if pending.kind == pendingAwait && parent != nil && child != nil && + parent.handle == parentHandle && child.handle == childHandle && + g.active == parent && parent.owner == g && child.owner == g && + parent.state == FrameActive && child.state == FrameInitialSuspended && + child.parent == parent && child.header != nil && + child.header.Lifecycle == uint16(FrameInitialSuspended) && + awaitCompletionArmedForChild(child) && + compilerInlineAwaitParentDepth(g, parent, p.inlineAwaitDepth) { + if p.inlineAwaitDepth >= maxInlineAwaitDepth { + return InlineAwaitDeclined + } + parent.state = FrameSuspended + child.state = FrameActive + g.active = child + g.pending = pendingTransition{kind: pendingInlineStart, from: parent, target: child} + p.inlineAwaitDepth++ + return InlineAwaitStarted + } + } + return BeginInlineAwait(g, parentHandle, childHandle) +} + // validInlineAwaitEdgeForBegin is the pre-dispatch form: prepareAwait has // already published/armed the headers and record, while parent.state remains // FrameActive until either inline selection or dispatchPending commits it. @@ -187,6 +238,39 @@ func takeInlineAwaitInitialDecision(g *G, expected ParkTicket) bool { return true } +// takeInlineAwaitInitialDecisionCompiler consumes the pendingInlineStart +// certificate produced immediately before the compiler enters the child's +// initial llvm.coro.resume. BeginInlineAwaitCompiler already authenticated the +// complete edge and bounded ancestry; no external producer can mutate this +// owner-private transaction before the child prologue takes it. Any uncertain +// shape falls back to takeInlineAwaitInitialDecision through TakeRunDecision. +func takeInlineAwaitInitialDecisionCompiler(g *G) bool { + if !ValidG(g) || g.runP == nil { + return false + } + p := g.runP + pending := g.pending + parent, child := pending.from, pending.target + if pending.kind != pendingInlineStart || parent == nil || child == nil || + p.inlineAwaitDepth == 0 || p.current != g || !p.inResume || + g.state != GRunning || !p.runDecisionTaken || + p.runDecision != (RunDecision{}) || + p.action.Kind != ActionResume || p.action.Flags != 0 || + g.active != child || child.owner != g || child.parent != parent || + child.state != FrameActive || child.header == nil || + child.header.G != unsafe.Pointer(g) || + child.header.Parent != parent.handle || + child.header.SuspendReason != uint16(SuspendNone) || + child.header.Lifecycle != uint16(FrameInitialSuspended) || + parent.completion.child != child.handle || + parent.completion.status != completionArmed || + !compilerInlineAwaitParentDepth(g, parent, p.inlineAwaitDepth-1) { + return false + } + g.pending = pendingTransition{} + return true +} + // FinishInlineAwait commits the return from the adapter's nested // llvm.coro.resume. done is the adapter's exact llvm.coro.done result. func FinishInlineAwait( @@ -213,8 +297,8 @@ func FinishInlineAwait( !terminalInlineCompletion(&parent.completion, child.handle) { return InlineAwaitInvalid } - destroy, yielded, ok := dispatchPending(g, child) - if !ok || yielded || destroy != child || g.active != parent || + destroy, yielded, directPark, ok := dispatchPending(g, child) + if !ok || yielded || directPark || destroy != child || g.active != parent || g.destroyTarget != child || child.state != FrameDestroyPending { return InlineAwaitInvalid } @@ -235,6 +319,50 @@ func FinishInlineAwait( return InlineAwaitSuspend } +// FinishInlineAwaitCompiler commits the dominant normal-return transaction +// directly from its private pendingComplete certificate. Slow suspension, +// panic, cancellation, and any uncertain shape retain FinishInlineAwait's +// complete graph validation and common dispatch path. +func FinishInlineAwaitCompiler( + g *G, parentHandle, childHandle unsafe.Pointer, done bool, +) InlineAwaitDisposition { + if done && ValidG(g) && parentHandle != nil && childHandle != nil && g.runP != nil { + p := g.runP + child := g.active + if child != nil { + parent := child.parent + pending := g.pending + record := (*CompletionRecord)(nil) + if parent != nil { + record = &parent.completion + } + if parent != nil && record != nil && + parent.handle == parentHandle && child.handle == childHandle && + pending.kind == pendingComplete && pending.from == child && pending.target == nil && + p.inlineAwaitDepth != 0 && p.current == g && p.inResume && + g.state == GRunning && p.runDecisionTaken && + p.runDecision == (RunDecision{}) && + p.action.Kind == ActionResume && p.action.Flags == 0 && + g.destroyTarget == nil && !g.destroyRoot && + child.owner == g && child.state == FrameActive && child.header != nil && + child.header.SuspendReason == uint16(SuspendFrameComplete) && + child.header.Lifecycle == uint16(FrameFinalSuspended) && + record.child == childHandle && record.status == CompletionReturn && + record.typeWord == nil && record.dataWord == nil && + compilerInlineAwaitParentDepth(g, parent, p.inlineAwaitDepth-1) { + g.pending = pendingTransition{} + g.active = parent + child.state = FrameDestroyPending + child.header.Lifecycle = uint16(FrameDestroyPending) + g.destroyTarget = child + p.inlineAwaitDepth-- + return InlineAwaitDestroy + } + } + } + return FinishInlineAwait(g, parentHandle, childHandle, done) +} + func terminalInlineCompletion(record *CompletionRecord, childHandle unsafe.Pointer) bool { if record == nil || childHandle == nil || record.child != childHandle { return false @@ -279,6 +407,35 @@ func CommitInlineAwaitDestroy(g *G, parentHandle, childHandle unsafe.Pointer) bo return true } +// CommitInlineAwaitDestroyCompiler consumes the exact physical-destroy +// receipt immediately after CommitFrameDestroyV2. The parent completion record +// is the durable child identity, so a second scan of the remaining frame list +// is unnecessary on this private suffix. +func CommitInlineAwaitDestroyCompiler(g *G, parentHandle, childHandle unsafe.Pointer) bool { + if ValidG(g) && parentHandle != nil && childHandle != nil && g.runP != nil { + p := g.runP + parent := g.active + if parent != nil && parent.handle == parentHandle && parent.owner == g && + parent.header != nil && parent.state == FrameSuspended && + parent.header.SuspendReason == uint16(SuspendCall) && + parent.header.Lifecycle == uint16(FrameSuspended) && + p.current == g && p.inResume && g.state == GRunning && + p.runDecisionTaken && p.runDecision == (RunDecision{}) && + p.action.Kind == ActionResume && p.action.Flags == 0 && + g.pending == (pendingTransition{}) && g.destroyTarget == nil && + !g.destroyRoot && parent.completion.child == childHandle && + parent.completion.status == CompletionReturn && + parent.completion.typeWord == nil && parent.completion.dataWord == nil && + compilerInlineAwaitParentDepth(g, parent, p.inlineAwaitDepth) { + parent.state = FrameActive + parent.header.SuspendReason = uint16(SuspendNone) + parent.header.Lifecycle = uint16(FrameActive) + return true + } + } + return CommitInlineAwaitDestroy(g, parentHandle, childHandle) +} + // resumedFrameForAction accepts the ordinary one-frame return or a completely // unwound inline chain. In the latter case dispatchPending must consume the // deepest active frame, not the outer ActionResume handle whose native call diff --git a/runtime/internal/coro/inline_await_test.go b/runtime/internal/coro/inline_await_test.go index c0e04f12e2..acd42898ef 100644 --- a/runtime/internal/coro/inline_await_test.go +++ b/runtime/internal/coro/inline_await_test.go @@ -31,6 +31,10 @@ type inlineAwaitFixture struct { } func newInlineAwaitFixture(t *testing.T) *inlineAwaitFixture { + return newInlineAwaitFixtureForCompiler(t, false) +} + +func newInlineAwaitFixtureForCompiler(t *testing.T, compiler bool) *inlineAwaitFixture { t.Helper() g := new(G) if !InitG(g) { @@ -58,13 +62,30 @@ func newInlineAwaitFixture(t *testing.T) *inlineAwaitFixture { } parent.header.SuspendReason = uint16(SuspendCall) parent.header.Lifecycle = uint16(FrameSuspended) - if !PrepareAwaitCompletion(g, parent.handle, child.handle) { + var prepared bool + if compiler { + prepared = PrepareAwaitCompletionCompiler(g, parent.handle, child.handle) + } else { + prepared = PrepareAwaitCompletion(g, parent.handle, child.handle) + } + if !prepared { t.Fatal("prepare inline-await child") } - if got := BeginInlineAwait(g, parent.handle, child.handle); got != InlineAwaitStarted { - t.Fatalf("begin inline-await = %d, want started", got) + var disposition InlineAwaitDisposition + if compiler { + disposition = BeginInlineAwaitCompiler(g, parent.handle, child.handle) + } else { + disposition = BeginInlineAwait(g, parent.handle, child.handle) } - if outcome, caseID, lease, task, taken := TakeRunDecision(g, ParkTicket{}); !taken || outcome != ParkOutcomePending || caseID != 0 || + if disposition != InlineAwaitStarted { + t.Fatalf("begin inline-await = %d, want started", disposition) + } + if compiler { + if outcome, caseID, task, source, generation, taken := TakeRunDecisionWordsCompiler(g, 0, 0); !taken || outcome != 0 || caseID != 0 || task != 0 || source != 0 || generation != 0 { + t.Fatalf("take compiler inline child initial gate = (%d, %d, %d, %d, %d, %t)", + outcome, caseID, task, source, generation, taken) + } + } else if outcome, caseID, lease, task, taken := TakeRunDecision(g, ParkTicket{}); !taken || outcome != ParkOutcomePending || caseID != 0 || lease != (OperationResultLease{}) || task != TaskCancelNone { t.Fatalf("take inline child initial gate = (%d, %d, %+v, %d, %t)", outcome, caseID, lease, task, taken) } @@ -73,6 +94,65 @@ func newInlineAwaitFixture(t *testing.T) *inlineAwaitFixture { return &inlineAwaitFixture{p: p, g: g, parent: parent, child: child, action: action} } +func TestCompilerInlineAwaitCompletesThroughTrustedSuffix(t *testing.T) { + fixture := newInlineAwaitFixtureForCompiler(t, true) + fixture.child.header.SuspendReason = uint16(SuspendFrameComplete) + fixture.child.header.Lifecycle = uint16(FrameFinalSuspended) + if !PrepareCompleteStatusCompiler( + fixture.g, fixture.child.handle, fixture.child.header, CompletionReturn, + ) { + t.Fatal("publish compiler inline child return") + } + if got := FinishInlineAwaitCompiler( + fixture.g, fixture.parent.handle, fixture.child.handle, true, + ); got != InlineAwaitDestroy { + t.Fatalf("finish compiler inline-await = %d, want destroy", got) + } + if fixture.p.inlineAwaitDepth != 0 || fixture.g.active != FrameFromStorage(fixture.parent.storage) { + t.Fatalf("compiler inline destroy did not restore parent ownership: depth=%d active=%p", + fixture.p.inlineAwaitDepth, fixture.g.active) + } + releaseTestFrame(t, fixture.g, fixture.child) + if !CommitFrameDestroyCompiler(fixture.g, fixture.child.handle) { + t.Fatal("commit compiler physical child destroy") + } + if !CommitInlineAwaitDestroyCompiler(fixture.g, fixture.parent.handle, fixture.child.handle) { + t.Fatal("commit compiler inline child destroy") + } + snapshot, ok := ConsumeAwaitCompletionCompiler(fixture.g, fixture.parent.handle) + if !ok || snapshot != (CompletionSnapshot{Status: CompletionReturn}) { + t.Fatalf("consume compiler inline child return = (%+v, %t)", snapshot, ok) + } + if fixture.parent.header.SuspendReason != uint16(SuspendNone) || + fixture.parent.header.Lifecycle != uint16(FrameActive) { + t.Fatalf("compiler inline parent header = (%d, %d), want active", + fixture.parent.header.SuspendReason, fixture.parent.header.Lifecycle) + } + runtime.KeepAlive(fixture.parent.memory) +} + +func TestCompilerInlineAwaitSlowYieldUsesCheckedDispatch(t *testing.T) { + fixture := newInlineAwaitFixtureForCompiler(t, true) + fixture.child.header.SuspendReason = uint16(SuspendYield) + fixture.child.header.Lifecycle = uint16(FrameSuspended) + if !PrepareYield(fixture.g, fixture.child.handle, fixture.child.header) { + t.Fatal("publish compiler inline child yield") + } + if got := FinishInlineAwaitCompiler( + fixture.g, fixture.parent.handle, fixture.child.handle, false, + ); got != InlineAwaitSuspend { + t.Fatalf("finish yielding compiler inline child = %d, want suspend", got) + } + action, ok := Resumed(fixture.p, fixture.g, fixture.action) + if !ok || action.Kind != ActionYield || action.Handle != nil || + fixture.g.state != GRunnable || !fixture.g.queued { + t.Fatalf("dispatch compiler inline yield = (%+v, %t), state=%d queued=%t", + action, ok, fixture.g.state, fixture.g.queued) + } + runtime.KeepAlive(fixture.parent.memory) + runtime.KeepAlive(fixture.child.memory) +} + func (fixture *inlineAwaitFixture) finishFastChild(t *testing.T) { t.Helper() if got := FinishInlineAwait( diff --git a/runtime/internal/coro/manual_operation_source.go b/runtime/internal/coro/manual_operation_source.go index cb2d2c7ff3..fdd9d87db1 100644 --- a/runtime/internal/coro/manual_operation_source.go +++ b/runtime/internal/coro/manual_operation_source.go @@ -105,6 +105,18 @@ type ManualOperationSource struct { affectedTail uint32 } +// ManualOperationReservation is an owner-stack capability for one reusable +// manual-event slot. It exists only between the current-executor preflight and +// the no-suspend park transaction; producers and coroutine frames retain only +// the resulting OperationID. +type ManualOperationReservation struct { + source *ManualOperationSource + owner *P + slot *manualOperationSlot + index uint32 + capacity uint32 +} + func ManualOperationConfiguredCapacity(source *ManualOperationSource) uint32 { if source == nil { return 0 @@ -210,22 +222,52 @@ func ConfigureManualOperationPages(source *ManualOperationSource, pages []Manual return true } -// CanReserveManualOperation is the allocation-free owner preflight used before -// BeginParkSet mutates a logical wait transaction. -func CanReserveManualOperation(p *P, source *ManualOperationSource) bool { - if !validManualOperationOwner(source, p) { - return false - } - for index := uint32(0); index < ManualOperationConfiguredCapacity(source); index++ { +func (source *ManualOperationSource) preflightManualReservationOwned( + p *P, +) (ManualOperationReservation, bool) { + capacity := ManualOperationConfiguredCapacity(source) + for index := uint32(0); index < capacity; index++ { slot, ok := manualOperationSlotAt(source, index) if !ok { - return false + return ManualOperationReservation{}, false } if preemptLoad(&slot.generation) != ^uint32(0) && manualOperationReusableSlot(source, slot, index) { - return true + return ManualOperationReservation{ + source: source, owner: p, slot: slot, index: index, capacity: capacity, + }, true } } - return false + return ManualOperationReservation{}, false +} + +func validManualOperationReservationHeader( + source *ManualOperationSource, + p *P, + reservation ManualOperationReservation, +) bool { + return source != nil && p != nil && reservation.source == source && + reservation.owner == p && reservation.slot != nil && + reservation.capacity != 0 && reservation.index < reservation.capacity +} + +// PreflightManualOperationReservation authenticates the source owner and +// selects the exact reusable slot once. The capability is consumed before any +// suspension; the final generation CAS remains in the reservation commit. +func PreflightManualOperationReservation( + p *P, + source *ManualOperationSource, +) (ManualOperationReservation, bool) { + if !validManualOperationOwner(source, p) { + return ManualOperationReservation{}, false + } + return source.preflightManualReservationOwned(p) +} + +// CanReserveManualOperation is the allocation-free owner preflight used before +// BeginParkSet mutates a logical wait transaction. +func CanReserveManualOperation(p *P, source *ManualOperationSource) bool { + _, ok := PreflightManualOperationReservation(p, source) + return ok } // AttachManualOperationPage publishes one pristine target-owned page while @@ -277,52 +319,72 @@ func validManualOperationLiveSlot(source *ManualOperationSource, p *P, index uin return ok && slot.record.Matches(id) } +func (source *ManualOperationSource) reserveAndAttachManualSlot( + state *ParkState, + ticket ParkTicket, + wait *WaitSetRecord, + caseID uint32, + reservation ManualOperationReservation, +) (OperationID, bool) { + slot, index := reservation.slot, reservation.index + if slot == nil || index >= reservation.capacity || + reservation.capacity != ManualOperationConfiguredCapacity(source) { + return OperationID{}, false + } + generation, begun := beginProducerSourceSlot(&slot.producerSourceSlot) + if !begun { + return OperationID{}, false + } + id, ok := MakeOperationIDAtRoute(OperationSourceManual, source.route, index+1, generation) + if !ok || !PrepareOperationAtGeneration(&slot.record, id) { + return OperationID{}, false + } + attached := false + if wait == nil { + attached = AttachParkOperation(state, ticket, &slot.record, caseID) + } else { + attached = AttachParkWaitOperation(state, ticket, wait, &slot.record, caseID) + } + if !attached { + if !AbortReservedOperation(&slot.record, id) || + !resetProducerSourceSlot(&slot.producerSourceSlot, generation) { + return OperationID{}, false + } + return OperationID{}, false + } + if !activateProducerSourceSlot(&slot.producerSourceSlot, generation) { + return OperationID{}, false + } + if index+1 > source.scanLimit { + source.scanLimit = index + 1 + } + return id, true +} + // ReserveAndAttachManualOperation reserves one physical slot generation and // attaches its stable OperationRecord to a preparing logical wait-set. No // producer is admitted until all owner pointers are initialized and Active is // release-published. func (source *ManualOperationSource) reserveAndAttach(p *P, state *ParkState, ticket ParkTicket, wait *WaitSetRecord, caseID uint32) (OperationID, bool) { - if !validManualOperationOwner(source, p) { + reservation, ok := PreflightManualOperationReservation(p, source) + if !ok { return OperationID{}, false } - for index := uint32(0); index < ManualOperationConfiguredCapacity(source); index++ { - slot, slotOK := manualOperationSlotAt(source, index) - if !slotOK { - return OperationID{}, false - } - if !manualOperationReusableSlot(source, slot, index) || preemptLoad(&slot.generation) == ^uint32(0) { - continue - } - generation, begun := beginProducerSourceSlot(&slot.producerSourceSlot) - if !begun { - return OperationID{}, false - } - id, ok := MakeOperationIDAtRoute(OperationSourceManual, source.route, index+1, generation) - if !ok || !PrepareOperationAtGeneration(&slot.record, id) { - return OperationID{}, false - } - attached := false - if wait == nil { - attached = AttachParkOperation(state, ticket, &slot.record, caseID) - } else { - attached = AttachParkWaitOperation(state, ticket, wait, &slot.record, caseID) - } - if !attached { - if !AbortReservedOperation(&slot.record, id) || - !resetProducerSourceSlot(&slot.producerSourceSlot, generation) { - return OperationID{}, false - } - return OperationID{}, false - } - if !activateProducerSourceSlot(&slot.producerSourceSlot, generation) { - return OperationID{}, false - } - if index+1 > source.scanLimit { - source.scanLimit = index + 1 - } - return id, true + return source.reserveAndAttachManualSlot(state, ticket, wait, caseID, reservation) +} + +func (source *ManualOperationSource) reserveAndAttachManualReservation( + p *P, + state *ParkState, + ticket ParkTicket, + wait *WaitSetRecord, + caseID uint32, + reservation ManualOperationReservation, +) (OperationID, bool) { + if !validManualOperationReservationHeader(source, p, reservation) { + return OperationID{}, false } - return OperationID{}, false + return source.reserveAndAttachManualSlot(state, ticket, wait, caseID, reservation) } func (source *ManualOperationSource) ReserveAndAttach(p *P, state *ParkState, ticket ParkTicket, caseID uint32) (OperationID, bool) { diff --git a/runtime/internal/coro/manual_park_owner.go b/runtime/internal/coro/manual_park_owner.go index ebd2d96bd4..d06cdbeb9a 100644 --- a/runtime/internal/coro/manual_park_owner.go +++ b/runtime/internal/coro/manual_park_owner.go @@ -30,6 +30,28 @@ func CurrentExecutorManualDriver(g *G) (*ExecutorDriver, ExecutorHandle, RouteID return driver, handle, route, true } +// CurrentExecutorManualReservation combines current-G authentication with the +// common one-event source preflight. current distinguishes an invalid owner +// from ordinary catalog exhaustion so the runtime may grow stable pages only +// on the latter path. +func CurrentExecutorManualReservation( + g *G, +) ( + driver *ExecutorDriver, + handle ExecutorHandle, + route RouteID, + reservation ManualOperationReservation, + current bool, + reserved bool, +) { + driver, handle, route, current = CurrentExecutorManualDriver(g) + if !current { + return nil, ExecutorHandle{}, 0, ManualOperationReservation{}, false, false + } + reservation, reserved = driver.sources.manual.preflightManualReservationOwned(driver.p) + return +} + func currentExecutorManualSource(driver *ExecutorDriver, g *G) (*ManualOperationSource, bool) { if driver == nil || !ValidG(g) || g.active == nil { return nil, false @@ -53,22 +75,77 @@ func PrepareSingleManualPark( caseID uint32, seed uint32, ) (ParkTicket, OperationID, bool) { + if g == nil || g.runP == nil { + return ParkTicket{}, OperationID{}, false + } + reservation, ok := PreflightManualOperationReservation(g.runP, source) + if !ok { + return ParkTicket{}, OperationID{}, false + } + return prepareSingleManualParkReserved( + g, handle, header, source, wait, reservation, caseID, seed, + ) +} + +func prepareSingleManualParkReserved( + g *G, + handle unsafe.Pointer, + header *HeaderV1, + source *ManualOperationSource, + wait *WaitSetRecord, + reservation ManualOperationReservation, + caseID uint32, + seed uint32, +) (ParkTicket, OperationID, bool) { + if g != nil && g.park.taskCancelKind == TaskCancelNone && + g.park.taskCancelPhase == taskCancelIdle { + return prepareSingleManualParkOrdinary( + g, handle, header, source, wait, reservation, caseID, seed, + ) + } if !ValidG(g) || handle == nil || header == nil || source == nil || wait == nil || *wait != (WaitSetRecord{}) || caseID == 0 || !resumeGateTaken(g) || g.runP == nil || - !CanReserveManualOperation(g.runP, source) { + !validManualOperationReservationHeader(source, g.runP, reservation) { return ParkTicket{}, OperationID{}, false } ticket, ok := BeginParkSet(&g.park, 1, seed) if !ok || !PrepareWaitSetRecord(wait, g, ticket) { return ParkTicket{}, OperationID{}, false } - id, ok := source.ReserveAndAttachWait(g.runP, &g.park, ticket, wait, caseID) + id, ok := source.reserveAndAttachManualReservation( + g.runP, &g.park, ticket, wait, caseID, reservation, + ) if !ok || !SealParkSet(&g.park, ticket) || !PrepareParkSet(g, handle, header, ticket, wait) { return ParkTicket{}, OperationID{}, false } return ticket, id, true } +func prepareSingleManualParkOrdinary( + g *G, + handle unsafe.Pointer, + header *HeaderV1, + source *ManualOperationSource, + wait *WaitSetRecord, + reservation ManualOperationReservation, + caseID uint32, + seed uint32, +) (ParkTicket, OperationID, bool) { + prepared, ok := preflightSingleParkPreparation(g, handle, header, wait, seed) + if !ok || source == nil || + !validManualOperationReservationHeader(source, prepared.p, reservation) { + return ParkTicket{}, OperationID{}, false + } + prepared.begin() + id, ok := source.reserveAndAttachManualSlot( + &g.park, prepared.ticket, wait, caseID, reservation, + ) + if !ok || !prepared.commit(id, caseID) { + return ParkTicket{}, OperationID{}, false + } + return prepared.ticket, id, true +} + func PrepareCurrentExecutorManualPark( driver *ExecutorDriver, g *G, @@ -86,6 +163,88 @@ func PrepareCurrentExecutorManualPark( return ticket, operation, driver.handle, ok } +// PrepareCurrentExecutorManualParkReserved consumes the slot capability +// returned by CurrentExecutorManualReservation. The call must remain in the +// same compiler-owned no-suspend park interval; exact driver/P/source identity +// prevents a capability from crossing an owner transition. +func PrepareCurrentExecutorManualParkReserved( + driver *ExecutorDriver, + g *G, + handle unsafe.Pointer, + header *HeaderV1, + wait *WaitSetRecord, + reservation ManualOperationReservation, + caseID uint32, + seed uint32, +) (ParkTicket, OperationID, ExecutorHandle, bool) { + if driver == nil || g == nil || driver.magic != executorDriverMagic || + driver.state != executorDriverActive || driver.p == nil || driver.p != g.runP || + driver.sources.manual == nil || + !validManualOperationReservationHeader(driver.sources.manual, driver.p, reservation) { + return ParkTicket{}, OperationID{}, ExecutorHandle{}, false + } + ticket, operation, ok := prepareSingleManualParkReserved( + g, handle, header, driver.sources.manual, wait, reservation, caseID, seed, + ) + return ticket, operation, driver.handle, ok +} + +// PrepareCurrentExecutorManualCleanupParkReserved extends the one-event +// reservation transaction through installation of its fixed keyed/manual +// cleanup descriptor. The compiler/runtime caller must consume the reservation +// and all frame-local storage in one no-suspend interval. The park builder has +// already performed the complete one-link audit, so the final scalar relation +// is sufficient and avoids immediately walking the same G/P/source graph again. +func PrepareCurrentExecutorManualCleanupParkReserved( + driver *ExecutorDriver, + g *G, + handle unsafe.Pointer, + header *HeaderV1, + wait *WaitSetRecord, + reservation ManualOperationReservation, + packet *ResumePacket, + plan *ResumeCleanupPlan, + context unsafe.Pointer, + entry *OperationID, + caseID uint32, + seed uint32, +) (ParkTicket, OperationID, ExecutorHandle, bool) { + if packet == nil || plan == nil || context == nil || entry == nil || + *packet != (ResumePacket{}) || *plan != (ResumeCleanupPlan{}) || + *entry != (OperationID{}) { + return ParkTicket{}, OperationID{}, ExecutorHandle{}, false + } + ticket, operation, executor, ok := PrepareCurrentExecutorManualParkReserved( + driver, g, handle, header, wait, reservation, caseID, seed, + ) + if !ok { + return ParkTicket{}, OperationID{}, ExecutorHandle{}, false + } + *entry = operation + state := &g.park + if wait.state != waitSetRecordCommitted || wait.work != waitSetWorkIdle || + wait.resume != nil || wait.resumeKind != resumeBindingNone || wait.g != g || + wait.ticket != ticket || g.active == nil || g.active.parkWait != wait || + g.pending.kind != pendingParkSet || g.pending.from != g.active || + state.phase != parkParked || state.ticket != ticket || state.expected != 1 || + state.attached != 1 || state.head == nil || state.head.previous != nil || + state.head.next != nil || state.head.wait != wait || state.head.park != state || + state.head.ticket != ticket || state.head.caseID != caseID || + state.head.operation == nil || state.head.operation.id != operation || + operation.Source() != OperationSourceManual { + return ParkTicket{}, OperationID{}, ExecutorHandle{}, false + } + installWaitSetResumeCleanup(wait, packet, plan, ResumeCleanupBinding{ + Kind: ResumeCleanupKeyedPark, + Context: context, + Entries: unsafe.Pointer(entry), + Count: 1, + RuntimeCount: 1, + Stride: unsafe.Sizeof(OperationID{}), + }) + return ticket, operation, executor, true +} + // FinishCurrentExecutorManualPark releases the exact source generation after // the compiler resume gate consumed its ParkTicket. Logical detach has already // sealed producer admission; this function proves quiescence, releases any @@ -124,3 +283,189 @@ func FinishCurrentExecutorManualPark( } return source.Recycle(p, operation) } + +// OwnerLocalManualCompletion is a transient, stack-only capability for the +// exact interval between owner-side logical completion and runtime-private +// keyed cleanup. It never crosses a suspension and exposes no scheduler or +// source pointer to the runtime package; that package receives only the closed +// ResumeCleanupStep it already knows how to materialize. +type OwnerLocalManualCompletion struct { + current *G + driver *ExecutorDriver + wait *WaitSetRecord + plan *ResumeCleanupPlan + id OperationID +} + +// BeginOwnerLocalManualCompletionCurrent collapses the common same-P, +// one-event keyed park through logical resolution and source detach. The +// runtime must materialize the returned ResumeCleanupStep and then call +// FinishOwnerLocalManualCompletionCurrent before it returns to user code. +// +// handled=false, ok=true is a clean fallback and changes no source state. Once +// handled is true, ok=false is a fail-closed invariant failure after the exact +// mailbox was claimed; callers must not retry through the external source +// path. Multi-event, cancellation, cross-P, occupied-source, and non-keyed +// cleanup shapes retain the durable mailbox + request + A/ack/B protocol. +func BeginOwnerLocalManualCompletionCurrent( + current *G, + driver *ExecutorDriver, + id OperationID, +) ( + completion OwnerLocalManualCompletion, + cleanup ResumeCleanupStep, + handled bool, + ok bool, +) { + if current == nil || driver == nil || driver.magic != executorDriverMagic || + driver.state != executorDriverActive || driver.p == nil || driver.p.executor != driver || + driver.p.current != current || current.runP != driver.p || + driver.sources.manual == nil || driver.sources.manual.owner != driver.p || + driver.sources.manual.route != driver.route || !id.Valid() || + id.Source() != OperationSourceManual || id.Route() != driver.route { + return OwnerLocalManualCompletion{}, ResumeCleanupStep{}, false, true + } + p, source := driver.p, driver.sources.manual + slot, slotOK := manualOperationSlotFor(source, id) + if !slotOK || preemptLoad(&slot.generation) != id.Generation { + return OwnerLocalManualCompletion{}, ResumeCleanupStep{}, false, false + } + record := &slot.record + wait := record.link.wait + admission := ownerLocalCompletionAdmissionForCurrent(driver, wait) + if source.scanLimit != 1 || id.LocalSlot() != 1 || preemptLoad(&source.pending) != 0 || + source.affectedHead != 0 || source.affectedTail != 0 || slot.nextAffected != 0 || + preemptLoad(&slot.mailbox) != uint32(manualOperationMailboxEmpty) || + producerSourceLifecycle(preemptLoad(&slot.state)) != producerSourceActive || + admission == ownerLocalCompletionRejected || + !emptyOwnerLocalCompletion(&driver.local) || driver.poll != (executorPollTransaction{}) { + return OwnerLocalManualCompletion{}, ResumeCleanupStep{}, false, true + } + plan := (*ResumeCleanupPlan)(wait.resume) + entry := resumeCleanupIDAt(plan, 0) + state := &wait.g.park + if wait.resumeKind != resumeBindingCleanup || plan == nil || + !validTrustedResumeCleanupPlanState(wait, plan) || + plan.phase != resumeCleanupBound || plan.kind != ResumeCleanupKeyedPark || + plan.count != 1 || plan.runtime != 1 || plan.index != 0 || plan.claim != nil || + entry == nil || *entry != id || plan.context == nil || + record.id != id || record.phase != operationActive || + record.disposition != OperationDispositionPending || record.resolutionApplied || + record.link.operation != record || record.link.park != state || + record.link.ticket != wait.ticket || record.resultState != operationResultEmpty || + state.phase != parkParked || state.resolving || state.expected != 1 || state.attached != 1 || + state.cancelKind != ParkCancelNone || state.taskCancelKind != TaskCancelNone || + state.taskCancelPhase != taskCancelIdle || + !validOperationCandidate(record) || + operationCandidateMode(record) != OperationCommitIrreversibleCompletion || + operationCandidateState(record) != OperationCommitIdle || + operationCandidateIsPublished(record) || + !operationCandidatePendingResultStorageValid(record) { + return OwnerLocalManualCompletion{}, ResumeCleanupStep{}, false, true + } + + // Request the bounded safepoint before the irreversible mailbox claim. If + // another producer wins the CAS, the request is harmless and the ordinary + // source path remains complete. + if !RequestPreempt(current) { + return OwnerLocalManualCompletion{}, ResumeCleanupStep{}, false, false + } + if !preemptCompareAndSwap( + &slot.mailbox, + uint32(manualOperationMailboxEmpty), + uint32(manualOperationMailboxDraining), + ) { + return OwnerLocalManualCompletion{}, ResumeCleanupStep{}, false, true + } + handled = true + if PublishOperationCompletion(record, id) != OperationCompletionPublished { + return OwnerLocalManualCompletion{}, ResumeCleanupStep{}, true, false + } + preemptStore(&slot.mailbox, uint32(manualOperationMailboxDelivered)) + + // Consume the exact owner-work admission without entering either the local + // FIFO or the external source epoch. The admission was validated after the + // currently running G and owner P were authenticated and no suspension can + // occur in this transaction. + switch admission { + case ownerLocalCompletionIdle: + if wait.work != waitSetWorkIdle || wait.workNext != nil { + return OwnerLocalManualCompletion{}, ResumeCleanupStep{}, true, false + } + case ownerLocalCompletionAffectedHead: + if wait.work != waitSetWorkQueued || p.affectedWaitHead != wait { + return OwnerLocalManualCompletion{}, ResumeCleanupStep{}, true, false + } + next := wait.workNext + if next == nil && p.affectedWaitTail != wait { + return OwnerLocalManualCompletion{}, ResumeCleanupStep{}, true, false + } + p.affectedWaitHead = next + if next == nil { + p.affectedWaitTail = nil + } + wait.workNext = nil + default: + return OwnerLocalManualCompletion{}, ResumeCleanupStep{}, true, false + } + wait.work = waitSetWorkResolving + resolution, result := resolveAffectedOperationPublishedEpoch(record, id) + if result != affectedOperationResolved || + resolution != (CompletionResolution{WaitSets: 1, Completed: 1, Winners: 1}) || + source.ApplyOne(p, id, record) != OperationApplyDetached { + return OwnerLocalManualCompletion{}, ResumeCleanupStep{}, true, false + } + if retry, await, progressOK := finishWaitSetApplyProgress(wait, false, false); !progressOK || retry || await || !beginResumeCleanup(wait, plan) { + return OwnerLocalManualCompletion{}, ResumeCleanupStep{}, true, false + } + cleanup = ResumeCleanupStep{ + Kind: plan.kind, + Context: plan.context, + Index: plan.index, + WinnerCase: plan.caseID, + Outcome: plan.outcome, + plan: plan, + } + completion = OwnerLocalManualCompletion{ + current: current, + driver: driver, + wait: wait, + plan: plan, + id: id, + } + return completion, cleanup, true, true +} + +// FinishOwnerLocalManualCompletionCurrent consumes a successfully prepared +// completion after the runtime has committed its private cleanup step. The +// source-neutral single-operation cleanup closes admission, releases the +// result lease, recycles the slot, materializes the ResumePacket, and promotes +// the peer without another scheduler source epoch. +func FinishOwnerLocalManualCompletionCurrent( + completion *OwnerLocalManualCompletion, +) bool { + if completion == nil || completion.current == nil || completion.driver == nil || + completion.wait == nil || completion.plan == nil || !completion.id.Valid() { + return false + } + current, driver, wait, plan := completion.current, completion.driver, completion.wait, completion.plan + p := driver.p + if driver.magic != executorDriverMagic || driver.state != executorDriverActive || + p == nil || p.executor != driver || p.current != current || current.runP != p || + driver.sources.manual == nil || driver.sources.manual.owner != p || + driver.sources.manual.route != driver.route || completion.id.Route() != driver.route || + wait.resumeKind != resumeBindingCleanup || wait.resume != unsafe.Pointer(plan) || + wait.work != waitSetWorkResolving || plan.phase != resumeCleanupConfirm || + plan.kind != ResumeCleanupKeyedPark || plan.count != 1 || plan.runtime != 1 || + plan.index != 0 || plan.claim != nil || !validTrustedResumeCleanupPlanState(wait, plan) { + return false + } + finalized, cleanupOK := advanceResumeCleanupCore(&driver.sources, p, wait, plan) + if !cleanupOK || !finalized || wait.resumeKind != resumeBindingMaterialized || + wait.work != waitSetWorkResolving || wait.g.park.phase != parkMaterialized || + !promoteReadyWaitSet(&driver.sources, p, wait) { + return false + } + *completion = OwnerLocalManualCompletion{} + return true +} diff --git a/runtime/internal/coro/operation_route.go b/runtime/internal/coro/operation_route.go index c2f278485d..532da5562c 100644 --- a/runtime/internal/coro/operation_route.go +++ b/runtime/internal/coro/operation_route.go @@ -563,6 +563,37 @@ func (registry *OperationRouteRegistry) RequestTimerExecutor(route RouteID) Exec return result } +// RequestExecutor requests one exact bound route without attributing the wake +// to a catalog source. It is used by frame-local direct channel completions +// whose terminal fact already lives in the executor's MPSC inbox. +func (registry *OperationRouteRegistry) RequestExecutor( + route RouteID, + executor ExecutorHandle, +) ExecutorRequestResult { + if !route.Valid() || executor == (ExecutorHandle{}) { + return ExecutorRequestInvalid + } + slot, ok := operationRouteSlotFor(registry, route) + if !ok { + return ExecutorRequestStale + } + if !operationRouteAcquireProducer(slot) { + state := operationRouteLifecycle(preemptLoad(&slot.state)) + if state == operationRouteClosing || state == operationRouteQuiesced || state == operationRouteRetired { + return ExecutorRequestClosed + } + return ExecutorRequestStale + } + result := ExecutorRequestInvalid + if preemptLoad(&slot.state) == uint32(operationRouteActive) && + preemptLoad(&slot.route) == uint32(route) && slot.executor == executor && + slot.executorRegistry != nil { + result = slot.executorRegistry.Request(slot.executor) + } + operationRouteReleaseProducer(slot) + return result +} + // RequestChannelExecutor requests the exact executor after the typed hchan // adapter has durably committed one Channel endpoint. Unlike PostAndRequest, // this method does not publish a source fact: the adapter's external commit diff --git a/runtime/internal/coro/os_thread_affinity.go b/runtime/internal/coro/os_thread_affinity.go index 63cf220cac..6e4cf4b3a9 100644 --- a/runtime/internal/coro/os_thread_affinity.go +++ b/runtime/internal/coro/os_thread_affinity.go @@ -205,20 +205,34 @@ func CurrentOSThreadLocked(g *G) bool { p.osThreadSuspend == osThreadSuspendAttached } +// OSThreadSuspendHandoffCandidate is the target adapter's O(1) negative +// filter after an ordinary Yield/Park action has already committed. A true +// result is only a hint: PrepareOSThreadSuspendHandoff still proves the exact +// driver, task, action, and physical-owner relation before mutation. A false +// result is sufficient because an unlocked task has no M-affinity state to +// transfer. +func OSThreadSuspendHandoffCandidate(g *G) bool { + return g != nil && g.osThreadLockDepth != 0 +} + // releaseOSThreadLockForExit closes the logical lease before terminal G // publication. retireOwner distinguishes a balanced/unlocked exit from a G // which still held LockOSThread: the scheduler must clear both logical pointers // in either case, while the target must terminate the latter physical owner. func releaseOSThreadLockForExit(p *P, g *G) (retireOwner, ok bool) { if p == nil || g == nil || p.current != g || g.runP != p || - p.osThreadSuspend != osThreadSuspendAttached || osThreadForeignReentryAffined(g) { return false, false } if g.osThreadLockDepth == 0 { - return false, p.osThreadLockOwner != g - } - if p.osThreadLockOwner != g { + // A replacement M may finish an unlocked peer while the locked owner is + // detached in Park or YieldNeedsPeer. The peer owns no physical lease to + // retire, and its exit must preserve the suspended owner's P-local lock + // header. validOSThreadRunOwner proves both that replacement relation and + // the ordinary attached/unlocked case without weakening locked-G exit. + return false, validOSThreadRunOwner(p, g) + } + if p.osThreadSuspend != osThreadSuspendAttached || p.osThreadLockOwner != g { return false, false } g.osThreadLockDepth = 0 @@ -255,14 +269,22 @@ func osThreadSuspendPeerReady(p *P, owner *G) bool { return false } -func validCompletedOSThreadSuspendAction(p *P, task *G, kind ActionKind) bool { - if !completedExecutorRunAction(p, task, Action{Kind: kind}) { - return false - } +func validCompletedOSThreadSuspendPlacement(p *P, task *G, kind ActionKind) bool { switch kind { case ActionYield: // Resumed appends the just-yielded continuation at the owner tail. return p.readyTail == task && task.nextReady == nil + case ActionPark: + return true + default: + return false + } +} + +func validCompletedOSThreadSuspendPayload(p *P, task *G, kind ActionKind) bool { + switch kind { + case ActionYield: + return true case ActionPark: // Validate the exact newly parked record and its two active-list // neighbors without traversing unrelated parked tasks or candidates. @@ -294,7 +316,8 @@ func PrepareOSThreadSuspendHandoff( p := driver.p if p == nil || task == nil || !idleExecutorScheduler(p) || - !validCompletedOSThreadSuspendAction(p, task, kind) { + !completedExecutorRunAction(p, task, Action{Kind: kind}) || + !validCompletedOSThreadSuspendPlacement(p, task, kind) { return false, false } // The target observes every committed Yield/Park. An unlocked task needs no @@ -304,6 +327,9 @@ func PrepareOSThreadSuspendHandoff( if task.osThreadLockDepth == 0 { return false, true } + if !validCompletedOSThreadSuspendPayload(p, task, kind) { + return false, false + } // The overwhelmingly common unlocked path above needs only the scheduler's // owner-maintained O(1) headers and the exact completed task. Retain the // complete queue audits for the exceptional physical-owner transition, @@ -334,6 +360,25 @@ func PrepareOSThreadSuspendHandoff( // transaction; returnable becomes true only at a stable scheduler boundary // after a parked owner is runnable or a yielding owner's one peer Action debt // has been satisfied. +// +// OSThreadSuspendHandoffPossible is the constant-time negative gate for the +// overwhelmingly common executor which has never acquired LockOSThread. A +// positive result is only a hint and must be followed by the complete status +// audit below; false is exact because an attached P with no lock owner has no +// handoff state which a stable reduction could return to. +func OSThreadSuspendHandoffPossible(driver *ExecutorDriver) (possible, ok bool) { + if driver == nil || driver.magic != executorDriverMagic || driver.state != executorDriverActive || + driver.p == nil || driver.p.executor != driver || + preemptLoad(&driver.p.executorMode) != executorModeBound { + return false, false + } + p := driver.p + if p.osThreadLockOwner == nil && p.osThreadSuspend == osThreadSuspendAttached { + return false, true + } + return true, true +} + func OSThreadSuspendHandoffStatus( driver *ExecutorDriver, ) (detached, returnable, ok bool) { @@ -437,3 +482,17 @@ func commitOSThreadPeerAction(p *P) { p.osThreadSuspend = osThreadSuspendYieldPeerServiced } } + +// osThreadSuspendCurrentPeerNeedsService reports the one case where an +// otherwise sole-running bound G must still cross a scheduler boundary. The +// current replacement peer itself is the outstanding physical Action; it is +// not in the ready queue, so runnableForOSThreadOwner cannot represent this +// debt until the peer first yields. +func osThreadSuspendCurrentPeerNeedsService(p *P) bool { + if p == nil || p.osThreadSuspend != osThreadSuspendYieldNeedsPeer { + return false + } + owner, current := p.osThreadLockOwner, p.current + return owner != nil && current != nil && current != owner && + current.osThreadLockDepth == 0 +} diff --git a/runtime/internal/coro/os_thread_suspend_handoff_test.go b/runtime/internal/coro/os_thread_suspend_handoff_test.go index d8c68bb74e..1c76f41ae3 100644 --- a/runtime/internal/coro/os_thread_suspend_handoff_test.go +++ b/runtime/internal/coro/os_thread_suspend_handoff_test.go @@ -61,10 +61,16 @@ func TestOSThreadYieldHandoffRunsOnePeerAndRestoresOwnerFIFO(t *testing.T) { t.Fatal("enqueue locked-yield fixture") } kind := commitLockedRunnerYield(t, driver, owner) + if !OSThreadSuspendHandoffCandidate(owner.g) { + t.Fatal("locked yield did not remain a physical-owner handoff candidate") + } required, ok := PrepareOSThreadSuspendHandoff(driver, owner.g, kind.Kind) if !ok || !required { t.Fatalf("prepare locked-yield handoff = (%t, %t)", required, ok) } + if possible, valid := OSThreadSuspendHandoffPossible(driver); !valid || !possible { + t.Fatalf("locked-yield fast status gate = (%t, %t)", possible, valid) + } if p.readyHead != first.g || first.g.nextReady != second.g || second.g.nextReady != owner.g || p.readyTail != owner.g { t.Fatal("locked-yield detach reordered ready FIFO") @@ -105,6 +111,48 @@ func TestOSThreadYieldHandoffRunsOnePeerAndRestoresOwnerFIFO(t *testing.T) { runtime.KeepAlive(second.frame.memory) } +func TestOSThreadYieldHandoffAllowsReplacementPeerRootExit(t *testing.T) { + p := new(P) + driver, _, _ := bindTestExecutorDriver(t, p) + owner := newYieldingTestG(t, "locked-yield-exit-owner") + peer := newYieldingTestG(t, "locked-yield-exit-peer") + if !Enqueue(p, owner.g) || !Enqueue(p, peer.g) { + t.Fatal("enqueue locked-yield peer-exit fixture") + } + kind := commitLockedRunnerYield(t, driver, owner) + if required, ok := PrepareOSThreadSuspendHandoff(driver, owner.g, kind.Kind); !ok || !required { + t.Fatal("prepare locked-yield peer-exit handoff") + } + + target := queueRunnerCheckDestroy(t, driver, peer) + step := runnerNextPhysicalAction(t, driver, peer, ActionCheckDestroy) + destroy, ok := Checked(p, peer.g, step.Action, true) + if !ok || destroy.Kind != ActionDestroy || peer.g.destroyTarget != target { + t.Fatalf("check replacement peer destroy = (%+v, %t)", destroy, ok) + } + releaseTestFrame(t, peer.g, peer.frame) + completed, ok := DestroyedBounded(p, peer.g, destroy) + if !ok || completed.Kind != ActionComplete || ActionRetiresPhysicalOwner(completed) || + !CommitExecutorRunAction(driver, peer.g, completed) { + t.Fatalf("complete replacement peer = (%+v, %t)", completed, ok) + } + if p.osThreadLockOwner != owner.g || + owner.g.osThreadLockDepth == 0 || + peer.g.state != GDead { + t.Fatal("replacement peer exit disturbed locked owner") + } + if detached, returnable, valid := OSThreadSuspendHandoffStatus(driver); !valid || + !detached || !returnable { + t.Fatalf("peer-exit locked-yield status = (%t, %t, %t)", detached, returnable, valid) + } + if !RestoreOSThreadSuspendHandoff(driver, owner.g) { + t.Fatal("restore locked-yield owner after peer exit") + } + _ = runnerNextPhysicalAction(t, driver, owner, ActionCheckResume) + runtime.KeepAlive(owner.frame.memory) + runtime.KeepAlive(peer.frame.memory) +} + func TestOSThreadSuspendHandoffUnlockedActionIsNoop(t *testing.T) { p := new(P) driver, _, _ := bindTestExecutorDriver(t, p) @@ -114,11 +162,17 @@ func TestOSThreadSuspendHandoffUnlockedActionIsNoop(t *testing.T) { } step := runnerNextPhysicalAction(t, driver, task, ActionCheckResume) runnerYieldAction(t, driver, step, task) + if OSThreadSuspendHandoffCandidate(task.g) { + t.Fatal("unlocked yield became a physical-owner handoff candidate") + } if required, prepared := PrepareOSThreadSuspendHandoff( driver, task.g, ActionYield, ); !prepared || required { t.Fatalf("unlocked-yield handoff = (%t, %t)", required, prepared) } + if possible, valid := OSThreadSuspendHandoffPossible(driver); !valid || possible { + t.Fatalf("unlocked-yield fast status gate = (%t, %t)", possible, valid) + } if detached, returnable, valid := OSThreadSuspendHandoffStatus(driver); !valid || detached || returnable { t.Fatalf("unlocked-yield status = (%t, %t, %t)", diff --git a/runtime/internal/coro/owner_local_completion.go b/runtime/internal/coro/owner_local_completion.go index 37aaf5f9d1..0c8500430c 100644 --- a/runtime/internal/coro/owner_local_completion.go +++ b/runtime/internal/coro/owner_local_completion.go @@ -16,6 +16,8 @@ package coro +import "unsafe" + // ownerLocalCompletionCursor is the scheduler-owned fast lane for a physical // completion produced by the G which currently owns this exact P. The producer // publishes the typed source result while it already owns the source's runtime @@ -31,6 +33,124 @@ type ownerLocalCompletionCursor struct { head *WaitSetRecord tail *WaitSetRecord resolve publishedEpochResolveCursor + // scratch is scheduler-owner-only transient storage for one source + // reduction. Keeping it with the stable driver avoids forcing LLGo's + // conservative escape analysis to heap-allocate a publishedEpochResolveStep + // every time serviceExecutorRunLocal passes its address across a function + // boundary. It is always cleared before that service returns and is not part + // of the durable completion state. + scratch publishedEpochResolveStep +} + +type ownerLocalCompletionAdmission uint8 + +const ( + ownerLocalCompletionRejected ownerLocalCompletionAdmission = iota + ownerLocalCompletionIdle + ownerLocalCompletionAffectedHead +) + +// ownerLocalDirectChannelCleanupHeader is the hot selector for an already +// certified direct cleanup continuation. Every pointer and immutable descriptor +// field below was fully audited immediately before directChannel was installed; +// only CommitResumeCleanupStep may mutate this interval, and it can change only +// phase/index/small through its exact plan token. The final source reduction +// calls ownerLocalDirectChannelCleanupState once before any retirement effect. +func ownerLocalDirectChannelCleanupHeader( + cursor *publishedEpochResolveCursor, +) (*WaitSetRecord, *ResumeCleanupPlan, *OperationID, bool) { + if cursor == nil || !cursor.directChannel || cursor.phase != publishedEpochResolvePromote || + cursor.park != (parkResolutionCursor{}) || cursor.link != nil || cursor.claim != nil || + cursor.forced != nil || cursor.waitRetry || cursor.waitAwait || cursor.hasChannel || + cursor.claimOwned || cursor.wait == nil || cursor.batchTail == nil || + cursor.batchTail.workNext != nil || cursor.nextWait != cursor.wait.workNext { + return nil, nil, nil, false + } + wait := cursor.wait + if wait.state != waitSetRecordActive || wait.resumeKind != resumeBindingCleanup || + wait.resume == nil || wait.g == nil || wait.work != waitSetWorkResolving || + wait.g.park.phase != parkConsumed { + return nil, nil, nil, false + } + plan := (*ResumeCleanupPlan)(wait.resume) + if plan == nil || plan.packet == nil || plan.claim == nil || plan.context == nil || + plan.entries == nil || plan.kind != ResumeCleanupChannelDirect || plan.index != 0 { + return nil, nil, nil, false + } + if plan.phase != resumeCleanupRuntime && plan.phase != resumeCleanupConfirm { + return nil, nil, nil, false + } + id := (*OperationID)(unsafe.Add(plan.entries, plan.idOffset)) + return wait, plan, id, true +} + +// ownerLocalDirectChannelCleanupState performs the one complete scalar audit +// at the effectful retirement boundary. The two intervening runner selections +// use only ownerLocalDirectChannelCleanupHeader, so arbitrary graph validation +// is never multiplied by the number of scheduler host entries. +func ownerLocalDirectChannelCleanupState( + cursor *publishedEpochResolveCursor, +) (*WaitSetRecord, *ResumeCleanupPlan, *OperationID, bool) { + wait, plan, id, headerOK := ownerLocalDirectChannelCleanupHeader(cursor) + if !headerOK || !ValidG(wait.g) || wait.ticket != wait.g.park.ticket || + !validParkTicket(wait.ticket) || wait.g.state != GWaiting || !wait.g.waiting || + wait.g.queued || wait.g.nextReady != nil || wait.g.runP != nil || + wait.g.transferState != runnableTransferGIdle || wait.g.active == nil || + wait.g.active.parkWait != wait { + return nil, nil, nil, false + } + idSize := unsafe.Sizeof(OperationID{}) + if plan.count != 1 || plan.runtime != 1 || plan.ticket != wait.ticket || + plan.stride < idSize || plan.idOffset > plan.stride-idSize || plan.caseID != 1 || + plan.outcome != ParkOutcomeCompleted || !plan.lease.Valid() || + plan.result != ResumeResultNone || selectClaimLoad(plan.claim) != selectClaimClaimed || + plan.phase == resumeCleanupRuntime && plan.small != ResumeSmallInvalid || + plan.phase == resumeCleanupConfirm && plan.small == ResumeSmallInvalid { + return nil, nil, nil, false + } + packet := plan.packet + if packet.state != resumePacketBound || packet.ticket != wait.ticket || + packet.source != (OperationID{}) || packet.scalar != (ScalarResultPayloadV1{}) || + packet.caseID != 0 || packet.outcome != ParkOutcomePending || + packet.result != ResumeResultNone || packet.small != ResumeSmallInvalid { + return nil, nil, nil, false + } + leaseID, leaseOK := plan.lease.ID() + if !id.Valid() || id.Source() != OperationSourceChannel || !leaseOK || leaseID != *id || + plan.lease.ticket != wait.ticket { + return nil, nil, nil, false + } + state := &wait.g.park + if state.phase != parkConsumed || state.resolving || state.expected != 1 || + state.attached != 0 || state.seed != 0 || state.hasDefault || + state.taskCancelKind != TaskCancelNone || state.taskCancelPhase != taskCancelIdle || + state.cancelKind != ParkCancelNone || state.outcome != ParkOutcomeCompleted || + state.winnerCase != 1 || state.winnerID != *id || state.winnerRecord != nil || + state.head != nil { + return nil, nil, nil, false + } + return wait, plan, id, true +} + +func validOwnerLocalDirectChannelCursor( + cursor *publishedEpochResolveCursor, + p *P, +) bool { + wait, _, _, ok := ownerLocalDirectChannelCleanupHeader(cursor) + if !ok || p == nil { + return false + } + if wait.activePrev == nil { + if p.parkWaitHead != wait { + return false + } + } else if wait.activePrev.activeNext != wait { + return false + } + if wait.activeNext == nil { + return p.parkWaitTail == wait + } + return wait.activeNext.activePrev == wait } func validOwnerLocalCompletionHeader(local *ownerLocalCompletionCursor, p *P) bool { @@ -42,7 +162,9 @@ func validOwnerLocalCompletionHeader(local *ownerLocalCompletionCursor, p *P) bo if local.resolve != (publishedEpochResolveCursor{}) { return false } - } else if local.head != nil || !validPublishedEpochResolveCursor(&local.resolve, p) { + } else if local.head != nil || local.resolve.directChannel && + !validOwnerLocalDirectChannelCursor(&local.resolve, p) || + !local.resolve.directChannel && !validPublishedEpochResolveCursor(&local.resolve, p) { return false } return local.head == nil || local.head.work == waitSetWorkQueued && @@ -71,18 +193,103 @@ func validOwnerLocalCompletion(local *ownerLocalCompletionCursor, p *P) bool { } func emptyOwnerLocalCompletion(local *ownerLocalCompletionCursor) bool { - return local != nil && *local == (ownerLocalCompletionCursor{}) + return local != nil && local.head == nil && local.tail == nil && + local.resolve == (publishedEpochResolveCursor{}) && + local.scratch == (publishedEpochResolveStep{}) } -func canAppendOwnerLocalCompletion(driver *ExecutorDriver, record *WaitSetRecord) bool { - return driver != nil && validRunningExecutorOwner(driver) && - validOwnerLocalCompletionHeader(&driver.local, driver.p) && - driver.local.resolve == (publishedEpochResolveCursor{}) && - record != nil && record.work == waitSetWorkIdle && record.workNext == nil && - validActiveWaitSetRecordFast(driver.p, record) +func ownerLocalCompletionAdmissionFor( + driver *ExecutorDriver, + record *WaitSetRecord, +) ownerLocalCompletionAdmission { + if driver == nil || !validRunningExecutorOwner(driver) { + return ownerLocalCompletionRejected + } + return ownerLocalCompletionAdmissionForCurrent(driver, record) } -func appendOwnerLocalCompletionUnchecked(driver *ExecutorDriver, record *WaitSetRecord) { +// ownerLocalCompletionAdmissionForCurrent is the transient-capability half of +// ownerLocalCompletionAdmissionFor. Callers must have obtained driver together +// with the currently running G from CurrentExecutorDriver and must not cross a +// suspension before this call. The reciprocal G/P/source identities are +// checked at that public boundary; repeating the complete running-frame proof +// here would turn every same-P channel handoff into another full scheduler +// validation. +func ownerLocalCompletionAdmissionForCurrent( + driver *ExecutorDriver, + record *WaitSetRecord, +) ownerLocalCompletionAdmission { + if driver == nil || driver.p == nil || + !validOwnerLocalCompletionHeader(&driver.local, driver.p) || + driver.local.resolve != (publishedEpochResolveCursor{}) || record == nil || + !validActiveWaitSetRecordFast(driver.p, record) { + return ownerLocalCompletionRejected + } + if record.work == waitSetWorkIdle && record.workNext == nil { + return ownerLocalCompletionIdle + } + // A newly parked wait carries one mandatory initial visit. If it is the + // exact affected FIFO head, the stronger same-owner completion can consume + // that visit without scanning or unlinking an interior node. Validate the + // new head as well so the O(1) splice cannot conceal a malformed prefix. + p := driver.p + if record.work != waitSetWorkQueued || p.affectedWaitHead != record || + !validAffectedWaitQueueHeader(p) { + return ownerLocalCompletionRejected + } + next := record.workNext + if next == nil { + if p.affectedWaitTail != record { + return ownerLocalCompletionRejected + } + } else if next == record || p.affectedWaitTail == record || + next.work != waitSetWorkQueued || !validActiveWaitSetRecordFast(p, next) { + return ownerLocalCompletionRejected + } + return ownerLocalCompletionAffectedHead +} + +// ownerLocalDirectCompletionAdmissionForCurrent consumes the stronger frozen +// direct-channel capability after prepareOwnerLocalDirectHeld has matched its +// source generation, claim, cleanup plan, and exact owner P. Resumed performed +// the complete ParkState/G/active-queue audit before setting record Active; +// another G running on that same P excludes every owner mutation here. +// +// The compact lane accepts only an empty local resolver and an isolated +// mandatory affected visit. More complex FIFO shapes fall back to +// ownerLocalCompletionAdmissionForCurrent and retain its complete checks. +func ownerLocalDirectCompletionAdmissionForCurrent( + driver *ExecutorDriver, + record *WaitSetRecord, +) ownerLocalCompletionAdmission { + if driver == nil || driver.p == nil || record == nil || + !emptyOwnerLocalCompletion(&driver.local) || record.state != waitSetRecordActive { + return ownerLocalCompletionRejected + } + if record.work == waitSetWorkIdle && record.workNext == nil { + return ownerLocalCompletionIdle + } + p := driver.p + if record.work != waitSetWorkQueued || record.workNext != nil || + p.affectedWaitHead != record || p.affectedWaitTail != record { + return ownerLocalCompletionRejected + } + return ownerLocalCompletionAffectedHead +} + +func appendOwnerLocalCompletionUnchecked( + driver *ExecutorDriver, + record *WaitSetRecord, + admission ownerLocalCompletionAdmission, +) { + if admission == ownerLocalCompletionAffectedHead { + next := record.workNext + driver.p.affectedWaitHead = next + if next == nil { + driver.p.affectedWaitTail = nil + } + record.workNext = nil + } record.work = waitSetWorkQueued if driver.local.tail == nil { driver.local.head = record @@ -92,18 +299,195 @@ func appendOwnerLocalCompletionUnchecked(driver *ExecutorDriver, record *WaitSet driver.local.tail = record } +// completeOwnerLocalDirectChannelInline consumes the same-P capability which +// prepareOwnerLocalDirectHeld established before the hchan effect. The source +// admission and private select claim have already made that effect terminal; +// for an ordinary, uncanceled one-case park there is no typed runtime work to +// defer to an ExecutorRunStep. Collapse the fixed owner-only resolve/apply/ +// materialize sequence and append the peer to this P's ready queue. Select, +// cancellation, an occupied local resolver, and every cross-owner completion +// retain the ordinary owner-local or durable source path. +// +// handled=false is a clean pre-effect-shape fallback and makes no mutation. +// Once handled is true, ok=false is an invariant failure after the physical +// channel effect and must remain fail-closed. +func completeOwnerLocalDirectChannelInline( + driver *ExecutorDriver, + wait *WaitSetRecord, + admission ownerLocalCompletionAdmission, + source *ChannelOperationSource, + slot *channelOperationSlot, + id OperationID, + small uint8, +) (handled, ok bool) { + if driver == nil || wait == nil || wait.g == nil || source == nil || slot == nil || !id.Valid() || + small == ResumeSmallInvalid || driver.p == nil || source != driver.sources.channel || + source.owner != driver.p || source.route != driver.route || id.Route() != driver.route || + driver.local.head != nil || driver.local.tail != nil || + driver.local.resolve != (publishedEpochResolveCursor{}) || + driver.local.scratch != (publishedEpochResolveStep{}) { + return false, true + } + p, g := driver.p, wait.g + state, frame := &g.park, g.active + if !wait.directChannel || wait.state != waitSetRecordActive || + g.state != GWaiting || !g.waiting || g.queued || g.nextReady != nil || g.runP != nil || + g.transferState != runnableTransferGIdle || frame == nil || frame.parkWait != wait || + g.spawnChild != nil || g.spawnParent != nil || g.spawnP != nil || + state.phase != parkParked || state.resolving || state.ticket != wait.ticket || + !validParkTicket(wait.ticket) || state.expected != 1 || state.attached != 1 || + state.seed > 1 || state.hasDefault || state.cancelKind != ParkCancelNone || + state.taskCancelKind != TaskCancelNone || state.taskCancelPhase != taskCancelIdle || + state.outcome != ParkOutcomePending || state.winnerCase != 0 || + state.winnerID != (OperationID{}) || state.winnerRecord != nil { + return false, true + } + plan := (*ResumeCleanupPlan)(wait.resume) + if !validDirectChannelBoundResumeState(wait, plan) || + *(*OperationID)(plan.entries) != id || selectClaimLoad(plan.claim) != selectClaimClaimed { + return true, false + } + // CommitOwnerLocalDirectWithResult sealed producer ingress immediately + // after publishing the physical result and terminal claim. Do not begin the + // irreversible inline detach until every producer admitted before that seal + // has left. The already prepared owner-local FIFO retries ApplyOne after + // quiescence without losing the committed typed result. + if preemptLoad(&slot.state) != uint32(producerSourceClosing) || + !producerSourceSlotQuiesced(&slot.producerSourceSlot) { + return false, true + } + record, link := &slot.record, &slot.record.link + preferred, physicalSmall, physicalOK := channelPhysicalCompletion(preemptLoad(&slot.physical)) + ready, readyOK := channelOperationReadyAt(source, id.LocalSlot()-1) + schedule := preemptLoad(&p.schedule) + if !physicalOK || physicalSmall != small || preferred != 0 && !preferred.Valid() || + !readyOK || ready || preemptLoad(&slot.generation) != id.Generation || + preemptLoad(&slot.mailbox) != uint32(channelMailboxEmpty) || + preemptLoad(&slot.external) != uint32(channelExternalExposed) || + preemptLoad(&slot.externalLease)&1 != 0 || slot.claim != plan.claim || + record.id != id || record.phase != operationActive || record.cancelRequested || record.quiesced || + !operationCandidateExternallyCommitted(record) || + link.park != state || link.wait != wait || link.operation != record || + link.ticket != wait.ticket || link.caseID != 1 || link.previous != nil || link.next != nil || + state.head != link || !validReadyQueueHeader(p) || p.readyCount == ^uint32(0) || + (schedule != scheduleIdle && schedule != scheduleRequested) { + return true, false + } + previous, next := wait.activePrev, wait.activeNext + if previous == nil { + if p.parkWaitHead != wait { + return true, false + } + } else if previous.activeNext != wait { + return true, false + } + if next == nil { + if p.parkWaitTail != wait { + return true, false + } + } else if next.activePrev != wait { + return true, false + } + + var affectedNext *WaitSetRecord + switch admission { + case ownerLocalCompletionIdle: + if wait.work != waitSetWorkIdle || wait.workNext != nil { + return true, false + } + case ownerLocalCompletionAffectedHead: + if wait.work != waitSetWorkQueued || p.affectedWaitHead != wait { + return true, false + } + affectedNext = wait.workNext + if affectedNext == nil { + if p.affectedWaitTail != wait { + return true, false + } + } else if affectedNext == wait || p.affectedWaitTail == wait || + affectedNext.work != waitSetWorkQueued { + return true, false + } + default: + return true, false + } + + // Every fallible observation is complete above. Producer ingress is sealed + // and quiesced, the claim excludes a resolver, and this P excludes owner + // mutation, so the intermediate Detaching/Ready/Consumed and + // Detached/Quiesced/Taken states are not externally observable. Commit their + // no-fail write halves as one closed transaction and publish Free/Runnable + // only after their complete suffixes are canonical. + if admission == ownerLocalCompletionAffectedHead { + p.affectedWaitHead = affectedNext + if affectedNext == nil { + p.affectedWaitTail = nil + } + } + packet, claim, entry := plan.packet, plan.claim, (*OperationID)(plan.entries) + ticket := wait.ticket + + slot.claim = nil + preemptStore(&slot.external, 0) + preemptStore(&slot.physical, uint32(channelPhysicalIdle)) + *record = OperationRecord{id: id, phase: operationReusable} + source.reserveCursor = 0 + preemptStore(&slot.state, uint32(producerSourceFree)) + preemptStore(&claim.state, uint32(selectClaimOpen)) + + *entry = OperationID{} + *packet = ResumePacket{ + ticket: ticket, caseID: 1, outcome: ParkOutcomeCompleted, + result: ResumeResultChannel, small: small, state: resumePacketMaterialized, + } + *plan = ResumeCleanupPlan{} + *state = ParkState{ + ticket: ticket, phase: parkMaterialized, seed: uint32(preferred), + outcome: ParkOutcomeCompleted, winnerCase: 1, + } + if previous == nil { + p.parkWaitHead = next + } else { + previous.activeNext = next + } + if next == nil { + p.parkWaitTail = previous + } else { + next.activePrev = previous + } + frame.parkWait = nil + *wait = WaitSetRecord{} + g.waiting = false + g.state = GRunnable + appendRunnableUnchecked(p, g) + // This is the inline equivalent of a complete serviceExecutorRunLocal + // reduction. Pay one ready continuation before an unrelated source epoch; + // otherwise the newly parked current G's mandatory initial visit would make + // the runner scan every catalog before dispatching the peer we just woke. + driver.run.blocked = false + driver.run.actionsSinceSource = 0 + driver.run.readyDebt = true + return true, true +} + func ownerLocalCompletionPending(driver *ExecutorDriver) bool { - return driver != nil && (driver.local.head != nil || + // Include a lone tail so malformed selected state enters the exact cold + // validator and fails closed instead of being skipped by the hot selector. + return driver != nil && (driver.local.head != nil || driver.local.tail != nil || driver.local.resolve.phase != publishedEpochResolveIdle) } -func initializeOwnerLocalCompletionResolution( +// initializeOwnerLocalCompletionResolutionAfterHeader consumes the exact +// owner-local FIFO certified by resolveOwnerLocalCompletionStep. It has one +// caller and must remain immediately after that caller's ready/park/affected +// and local-header gate; repeating those O(1) graph checks here made every +// direct handoff validate the same active WaitSetRecord twice before mutation. +func initializeOwnerLocalCompletionResolutionAfterHeader( driver *ExecutorDriver, step *publishedEpochResolveStep, ) bool { if driver == nil || step == nil || driver.local.resolve != (publishedEpochResolveCursor{}) || - !validOwnerLocalCompletionHeader(&driver.local, driver.p) || driver.local.head == nil || - !validAffectedWaitQueueHeader(driver.p) { + driver.local.head == nil || driver.local.tail == nil { return false } head, tail := driver.local.head, driver.local.tail @@ -135,12 +519,15 @@ func resolveOwnerLocalCompletionStep( return false } if cursor.phase == publishedEpochResolveIdle { - if !initializeOwnerLocalCompletionResolution(driver, step) { + if !initializeOwnerLocalCompletionResolutionAfterHeader(driver, step) { return false } - } else if !validPublishedEpochResolveCursor(cursor, p) { + } else if !cursor.directChannel && !validPublishedEpochResolveCursor(cursor, p) { return false } + if handled, directOK := resolveOwnerLocalDirectChannelStep(driver, cursor, step); handled { + return directOK + } switch cursor.phase { case publishedEpochResolveDiscover: @@ -158,3 +545,340 @@ func resolveOwnerLocalCompletionStep( } return ok } + +// ownerLocalDirectChannelPlan recognizes only the compiler-owned one-case +// channel cleanup binding. The plan and ParkState together are the proof that +// this is not a select, timer, poll, worker, or cancellation-only event. A +// failed recognition is an ordinary fallback to the common persisted cursor. +func ownerLocalDirectChannelPlan( + driver *ExecutorDriver, + cursor *publishedEpochResolveCursor, +) (*WaitSetRecord, *ResumeCleanupPlan, *OperationID, *channelOperationSlot, bool) { + if driver == nil || cursor == nil || driver.sources.channel == nil { + return nil, nil, nil, nil, false + } + var wait *WaitSetRecord + var plan *ResumeCleanupPlan + var id *OperationID + if cursor.directChannel { + var stateOK bool + wait, plan, id, stateOK = ownerLocalDirectChannelCleanupState(cursor) + if !stateOK || plan.phase != resumeCleanupConfirm { + return nil, nil, nil, nil, false + } + } else { + if cursor.wait == nil || cursor.wait.resumeKind != resumeBindingCleanup { + return nil, nil, nil, nil, false + } + wait = cursor.wait + if wait.directChannel { + var headerOK bool + plan, headerOK = directChannelBoundResumeHeader(wait) + if !headerOK { + return nil, nil, nil, nil, false + } + } else { + plan = (*ResumeCleanupPlan)(wait.resume) + if !validTrustedResumeCleanupPlanState(wait, plan) { + return nil, nil, nil, nil, false + } + } + if plan.kind != ResumeCleanupChannelDirect || plan.count != 1 || + plan.runtime != 1 || plan.claim == nil { + return nil, nil, nil, nil, false + } + id = resumeCleanupIDAt(plan, 0) + } + if id == nil || !id.Valid() || id.Source() != OperationSourceChannel || + id.Route() != driver.sources.route { + return nil, nil, nil, nil, false + } + slot, slotOK := channelOperationSlotFor(driver.sources.channel, *id) + if !slotOK || preemptLoad(&slot.generation) != id.Generation || + selectClaimLoad(plan.claim) != selectClaimClaimed || slot.record.id != *id { + return nil, nil, nil, nil, false + } + // ApplyOne deliberately destroys the source-to-frame relation before typed + // runtime materialization: it clears slot.claim and DetachParkWaitOperation + // clears the complete ParkLink. Recognize the two sides of that lifetime with + // separate proofs. Requiring the attached shape after materialization would + // silently send every direct operation through the generic five-phase tail. + switch { + case cursor.phase == publishedEpochResolveDiscover && plan.phase == resumeCleanupBound: + if slot.claim != plan.claim || + slot.record.phase != operationActive || slot.record.resolutionApplied || + slot.record.link.wait != wait || slot.record.link.park != &wait.g.park || + slot.record.link.ticket != wait.ticket || + slot.record.link.operation != &slot.record { + return nil, nil, nil, nil, false + } + case cursor.phase == publishedEpochResolvePromote && plan.phase == resumeCleanupConfirm: + if preemptLoad(&slot.state) != uint32(producerSourceClosing) || + !producerSourceSlotQuiesced(&slot.producerSourceSlot) || + preemptLoad(&slot.mailbox) != uint32(channelMailboxEmpty) || + preemptLoad(&slot.external) != uint32(channelExternalExposed) || + preemptLoad(&slot.externalLease)&1 != 0 || slot.claim != nil || + slot.record.phase != operationDetached || !slot.record.resolutionApplied || + slot.record.link != (ParkLink{}) { + return nil, nil, nil, nil, false + } + default: + return nil, nil, nil, nil, false + } + return wait, plan, id, slot, true +} + +// beginOwnerLocalDirectChannelResumeCleanup consumes the exact ordinary +// one-channel winner after ApplyOne has detached it. ownerLocalDirectChannelPlan +// fully audited the immutable frame binding and source generation immediately +// before resolution, and ApplyOne proved the result/disposition transition. +// Repeating validResumeCleanupPlan -> ConsumeParkSet -> validResumeCleanupPlan +// here would traverse that same graph three more times. This gate checks only +// the scalar boundary which those two owner-only operations may have changed, +// then performs the no-fail write half of ConsumeParkSet and beginResumeCleanup. +// Cancellation and every non-direct shape continue through the generic machine. +func beginOwnerLocalDirectChannelResumeCleanup( + wait *WaitSetRecord, + plan *ResumeCleanupPlan, + id *OperationID, + slot *channelOperationSlot, +) bool { + if slot == nil { + return false + } + record := &slot.record + if wait == nil || wait.g == nil || plan == nil || id == nil || + wait.state != waitSetRecordActive || wait.work != waitSetWorkResolving || + wait.resume != unsafe.Pointer(plan) || wait.resumeKind != resumeBindingCleanup || + plan.packet == nil || plan.claim == nil || plan.context == nil || plan.entries == nil || + plan.kind != ResumeCleanupChannelDirect || plan.count != 1 || plan.runtime != 1 || + plan.index != 0 || plan.phase != resumeCleanupBound || plan.ticket != wait.ticket || + plan.outcome != ParkOutcomePending || plan.caseID != 0 || + plan.lease != (OperationResultLease{}) || plan.result != ResumeResultNone || + plan.small != ResumeSmallInvalid || + id != (*OperationID)(unsafe.Add(plan.entries, plan.idOffset)) || + selectClaimLoad(plan.claim) != selectClaimClaimed { + return false + } + packet := plan.packet + if packet.state != resumePacketBound || packet.ticket != wait.ticket || + packet.source != (OperationID{}) || packet.scalar != (ScalarResultPayloadV1{}) || + packet.caseID != 0 || packet.outcome != ParkOutcomePending || + packet.result != ResumeResultNone || packet.small != ResumeSmallInvalid { + return false + } + state := &wait.g.park + if !validParkTicket(wait.ticket) || state.ticket != wait.ticket || state.phase != parkReady || + state.resolving || state.expected != 1 || state.attached != 0 || state.seed != 0 || + state.hasDefault || state.taskCancelKind != TaskCancelNone || + state.taskCancelPhase != taskCancelIdle || state.cancelKind != ParkCancelNone || + state.outcome != ParkOutcomeCompleted || state.winnerCase != 1 || + state.winnerID != *id || state.winnerRecord != record || state.head != nil { + return false + } + if !id.Valid() || id.Source() != OperationSourceChannel || record.id != *id || + record.phase != operationDetached || record.disposition != OperationDispositionWinner || + !record.resolutionApplied || record.cancelRequested || record.quiesced || + operationCandidateMode(record) != OperationCommitReadyThenTryCommit || + operationCandidateState(record) != OperationCommitCommitted || + !operationCandidateIsPublished(record) || record.resultState != operationResultOwned || + record.resultTicket != wait.ticket || record.link != (ParkLink{}) { + return false + } + + record.resultState = operationResultLeased + plan.outcome = ParkOutcomeCompleted + plan.caseID = 1 + plan.lease = OperationResultLease{id: *id, ticket: wait.ticket} + plan.phase = resumeCleanupRuntime + state.winnerRecord = nil + state.seed = 0 + state.phase = parkConsumed + return true +} + +func beginOwnerLocalDirectChannelCompletion( + driver *ExecutorDriver, + cursor *publishedEpochResolveCursor, + step *publishedEpochResolveStep, + wait *WaitSetRecord, + plan *ResumeCleanupPlan, + id *OperationID, + slot *channelOperationSlot, +) bool { + if slot == nil { + return false + } + record := &slot.record + if cursor.phase != publishedEpochResolveDiscover || cursor.link != &record.link || + cursor.claim != nil || cursor.forced != nil || cursor.claimOwned || cursor.hasChannel || + wait.work != waitSetWorkResolving || plan.phase != resumeCleanupBound { + return false + } + resolution, resolved := resolveForcedSinglePark(&wait.g.park, wait.ticket, record) + if !resolved { + return false + } + step.resolution = resolution + cursor.link = wait.g.park.head + cursor.phase = publishedEpochResolveApply + cursor.waitRetry = false + cursor.waitAwait = false + + switch driver.sources.channel.ApplyOne(driver.p, *id, record) { + case OperationApplyDetached: + step.applyVisits = 1 + case OperationApplyRetryBudget: + // A foreign matcher may have acquired an admission immediately before + // this owner reduction. Preserve the already-resolved Detaching state + // and let the common bounded Apply phase perform the retry protocol. + return true + default: + return false + } + if wait.g.park.phase != parkReady || wait.g.park.head != nil || + wait.g.park.attached != 0 { + return false + } + if _, _, progressOK := finishWaitSetApplyProgress(wait, false, false); !progressOK { + return false + } + state := &wait.g.park + directCleanup := state.outcome == ParkOutcomeCompleted && + state.taskCancelKind == TaskCancelNone && state.taskCancelPhase == taskCancelIdle && + state.cancelKind == ParkCancelNone + if directCleanup { + if !beginOwnerLocalDirectChannelResumeCleanup(wait, plan, id, slot) { + return false + } + _, small, physicalOK := channelPhysicalCompletion(preemptLoad(&slot.physical)) + if physicalOK && small != ResumeSmallInvalid { + plan.small = small + plan.phase = resumeCleanupConfirm + } + } else if !beginResumeCleanup(wait, plan) { + return false + } + cursor.link = nil + cursor.phase = publishedEpochResolvePromote + // Task cancellation may suppress a physically completed channel case while + // preserving its result lease for discard. That shape intentionally retains + // the common cleanup machine; certify only the ordinary selected case whose + // scalar continuation is invariant until typed materialization. + cursor.directChannel = directCleanup + return true +} + +func materializeOwnerLocalDirectChannelResult( + driver *ExecutorDriver, + wait *WaitSetRecord, + plan *ResumeCleanupPlan, + id *OperationID, + slot *channelOperationSlot, +) bool { + if driver == nil || plan == nil || wait == nil || wait.g == nil || id == nil || slot == nil || + plan.phase != resumeCleanupConfirm || plan.index != 0 || + wait.work != waitSetWorkResolving || wait.g.park.phase != parkConsumed { + return false + } + sources, p := &driver.sources, driver.p + if p == nil || sources.channel == nil || plan.outcome != ParkOutcomeCompleted || + plan.caseID != 1 || !plan.lease.Valid() || + plan.small == ResumeSmallInvalid || plan.packet.scalar != (ScalarResultPayloadV1{}) { + return false + } + preferred, finished := sources.channel.finishOwnerLocalDirectChannelResult( + p, slot, *id, plan.claim, plan.lease, + ) + if !finished { + return false + } + plan.result = ResumeResultChannel + wait.g.park.seed = uint32(preferred) + plan.lease = OperationResultLease{} + *id = OperationID{} + packet := plan.packet + ticket, outcome, caseID := plan.ticket, plan.outcome, plan.caseID + result, scalar, small := plan.result, packet.scalar, plan.small + materializeConsumedParkStateUnchecked(&wait.g.park, ticket, outcome, caseID) + *packet = ResumePacket{ + ticket: ticket, scalar: scalar, caseID: caseID, outcome: outcome, + result: result, small: small, state: resumePacketMaterialized, + } + wait.resume = unsafe.Pointer(packet) + wait.resumeKind = resumeBindingMaterialized + *plan = ResumeCleanupPlan{} + if !validMaterializedResumePacket(packet) { + return false + } + + g, frame := wait.g, wait.g.active + schedule := preemptLoad(&p.schedule) + if !validMaterializedParkState(&g.park, ticket) || !validReadyQueueHeader(p) || + p.readyCount == ^uint32(0) || frame == nil || frame.parkWait != wait || + g.spawnChild != nil || g.spawnParent != nil || g.spawnP != nil || + (schedule != scheduleIdle && schedule != scheduleRequested) { + return false + } + wait.workNext = nil + promoteReadyWaitSetUnchecked(p, wait) + return true +} + +// finishOwnerLocalDirectChannelCompletion collapses the one-entry common +// cleanup tail after the runtime has removed its typed hchan node. Every +// operation remains the same source-neutral primitive used by select; only +// the per-phase cursor writes and repeated whole-plan validation disappear. +func finishOwnerLocalDirectChannelCompletion( + driver *ExecutorDriver, + cursor *publishedEpochResolveCursor, + step *publishedEpochResolveStep, + wait *WaitSetRecord, + plan *ResumeCleanupPlan, + id *OperationID, + slot *channelOperationSlot, +) bool { + if cursor.phase != publishedEpochResolvePromote || + !materializeOwnerLocalDirectChannelResult(driver, wait, plan, id, slot) { + return false + } + step.promoted = 1 + return advancePublishedEpochWaitAfterCleared(&driver.sources, cursor, driver.p, step) +} + +// resolveOwnerLocalDirectChannelStep returns handled=false for every shape +// which needs the common multi-event/cross-source state machine. +func resolveOwnerLocalDirectChannelStep( + driver *ExecutorDriver, + cursor *publishedEpochResolveCursor, + step *publishedEpochResolveStep, +) (handled, ok bool) { + wait, plan, id, slot, direct := ownerLocalDirectChannelPlan(driver, cursor) + if !direct { + return false, false + } + switch { + case cursor.phase == publishedEpochResolveDiscover && plan.phase == resumeCleanupBound: + if !beginOwnerLocalDirectChannelCompletion( + driver, cursor, step, wait, plan, id, slot, + ) { + return true, false + } + // A committed small result needs no typed runtime hook. The complete + // Discover -> Apply -> Confirm transition above ran in this same owner + // reduction, so finish it before re-entering the generic selector and + // re-auditing the unchanged ready/park/affected queue headers. Large or + // target-typed results remain at Runtime and return through Materialize. + if cursor.directChannel && cursor.phase == publishedEpochResolvePromote && + plan.phase == resumeCleanupConfirm { + return true, finishOwnerLocalDirectChannelCompletion( + driver, cursor, step, wait, plan, id, slot, + ) + } + return true, true + case cursor.phase == publishedEpochResolvePromote && plan.phase == resumeCleanupConfirm: + return true, finishOwnerLocalDirectChannelCompletion(driver, cursor, step, wait, plan, id, slot) + default: + return false, false + } +} diff --git a/runtime/internal/coro/owner_local_manual_test.go b/runtime/internal/coro/owner_local_manual_test.go new file mode 100644 index 0000000000..e1626f1c6a --- /dev/null +++ b/runtime/internal/coro/owner_local_manual_test.go @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coro + +import ( + "runtime" + "testing" + "unsafe" +) + +func TestOwnerLocalManualCompletionDirectlyMaterializesSingleKeyedPark(t *testing.T) { + p := new(P) + driver, registry, manual, handle := bindTestExecutorDriverWithManual(t, p) + waiter := newYieldingTestG(t, "manual-owner-local-direct-waiter") + if !Enqueue(p, waiter.g) { + t.Fatal("enqueue owner-local direct manual waiter") + } + if g, runnable := NextRunnable(p); !runnable || g != waiter.g { + t.Fatalf("dequeue owner-local direct manual waiter = (%p,%t)", g, runnable) + } + waiterAction := beginWaitTestResume(t, p, waiter) + var ( + wait WaitSetRecord + packet ResumePacket + plan ResumeCleanupPlan + operation OperationID + token byte + ) + waiter.frame.header.SuspendReason = uint16(SuspendPark) + waiter.frame.header.Lifecycle = uint16(FrameSuspended) + ticket, id, prepared := PrepareSingleManualPark( + waiter.g, + waiter.handle, + waiter.frame.header, + manual, + &wait, + 1, + 181, + ) + operation = id + if !prepared || !BindWaitSetResumeCleanup( + &wait, + &packet, + &plan, + ResumeCleanupBinding{ + Kind: ResumeCleanupKeyedPark, + Context: unsafe.Pointer(&token), + Entries: unsafe.Pointer(&operation), + Count: 1, + RuntimeCount: 1, + Stride: unsafe.Sizeof(OperationID{}), + }, + ) { + t.Fatal("prepare owner-local direct manual cleanup") + } + if action, parked := Resumed(p, waiter.g, waiterAction); !parked || action.Kind != ActionPark { + t.Fatalf("commit owner-local direct manual park = (%+v,%t)", action, parked) + } + + producer := newYieldingTestG(t, "manual-owner-local-direct-producer") + if !Enqueue(p, producer.g) { + t.Fatal("enqueue owner-local direct manual producer") + } + if g, runnable := NextRunnable(p); !runnable || g != producer.g { + t.Fatalf("dequeue owner-local direct manual producer = (%p,%t)", g, runnable) + } + producerAction := beginWaitTestResume(t, p, producer) + completion, cleanup, local, localOK := BeginOwnerLocalManualCompletionCurrent( + producer.g, + driver, + id, + ) + slot, slotOK := manualOperationSlotFor(manual, id) + if !localOK || !local || !slotOK || cleanup.Kind != ResumeCleanupKeyedPark || + cleanup.Context != unsafe.Pointer(&token) || cleanup.Index != 0 || + cleanup.WinnerCase != 1 || cleanup.Outcome != ParkOutcomeCompleted || + manualOperationMailbox(preemptLoad(&slot.mailbox)) != manualOperationMailboxDelivered || + manual.Pending() || !emptyOwnerLocalCompletion(&driver.local) || + p.affectedWaitHead != nil || p.affectedWaitTail != nil || + registry.ObserveRequested(handle) || plan.phase != resumeCleanupRuntime || + wait.work != waitSetWorkResolving || waiter.g.park.phase != parkConsumed || + preemptWordState(loadGPreempt(producer.g)) != preemptRequested { + t.Fatalf("begin owner-local direct manual completion = (%t,%t) cleanup=%+v slot=%t mailbox=%d pending=%t local=%+v affected=(%p,%p) request=%t plan=%+v wait=%+v park=%+v preempt=%#x", + local, localOK, cleanup, slotOK, + manualOperationMailbox(preemptLoad(&slot.mailbox)), manual.Pending(), driver.local, + p.affectedWaitHead, p.affectedWaitTail, registry.ObserveRequested(handle), + plan, wait, waiter.g.park, loadGPreempt(producer.g)) + } + if !CommitResumeCleanupStep(cleanup, ResumeSmallInvalid) || + !FinishOwnerLocalManualCompletionCurrent(&completion) { + t.Fatal("finish owner-local direct manual completion") + } + if completion != (OwnerLocalManualCompletion{}) || operation != (OperationID{}) || + plan != (ResumeCleanupPlan{}) || wait != (WaitSetRecord{}) || + packet.state != resumePacketMaterialized || packet.outcome != ParkOutcomeCompleted || + packet.caseID != 1 || packet.result != ResumeResultNone || packet.small != ResumeSmallInvalid || + !manualOperationSourceEmpty(manual, p) || registry.ObserveRequested(handle) || + p.readyHead != waiter.g || p.readyTail != waiter.g { + t.Fatalf("owner-local direct manual final state: completion=%+v operation=%+v plan=%+v wait=%+v packet=%+v empty=%t request=%t ready=(%p,%p)", + completion, operation, plan, wait, packet, manualOperationSourceEmpty(manual, p), + registry.ObserveRequested(handle), p.readyHead, p.readyTail) + } + yieldRunningDriverTask(t, p, producer, producerAction) + + if g, runnable := NextRunnable(p); !runnable || g != waiter.g { + t.Fatalf("dequeue owner-local direct manual waiter = (%p,%t)", g, runnable) + } + waiterAction = beginWaitTestResume(t, p, waiter) + outcome, caseID, cancel, result, small, taken := TakeResumePacket( + waiter.g, + ticket, + &packet, + nil, + ) + if !taken || outcome != ParkOutcomeCompleted || caseID != 1 || + cancel != TaskCancelNone || result != ResumeResultNone || small != ResumeSmallInvalid { + t.Fatalf("take owner-local direct manual packet = (%d,%d,%d,%d,%d,%t)", + outcome, caseID, cancel, result, small, taken) + } + finishWaitTestTask(t, p, waiter, waiterAction) + closeTestExecutorDriver(t, driver) + finishReadyDriverTasks(t, p, map[*G]*yieldingTestG{producer.g: producer}) + if !manual.CanRelease() || !registry.CanRelease() { + t.Fatal("owner-local direct manual cleanup retained stable state") + } + runtime.KeepAlive(waiter.frame.memory) + runtime.KeepAlive(producer.frame.memory) +} diff --git a/runtime/internal/coro/park_resolution_v2.go b/runtime/internal/coro/park_resolution_v2.go index 40621949a6..7f98d8c8d0 100644 --- a/runtime/internal/coro/park_resolution_v2.go +++ b/runtime/internal/coro/park_resolution_v2.go @@ -358,6 +358,60 @@ func beginForcedParkSnapshotResolution(state *ParkState, ticket ParkTicket, curs return false } +// resolveForcedSinglePark is the aggregate form of the forced-completion +// state machine for one exact direct operation. The external channel commit +// has already made the physical effect irreversible and claimed the direct +// operation's private SelectClaim, so there is no rank scan, default choice, +// or losing candidate to preserve between scheduler reductions. +// +// Multi-event select never enters this helper. Strong task cancellation is +// still honored exactly like beginForcedParkSnapshotResolution: it suppresses +// delivery while retaining the committed physical result for source cleanup. +// The successful result is consumed immediately by source ApplyOne in both +// callers. ApplyOne's detach gate validates the complete active ParkState +// header before its first mutation, so this aggregate transition does not +// repeat that same post-state audit after deterministic scalar writes. +func resolveForcedSinglePark( + state *ParkState, + ticket ParkTicket, + forced *OperationRecord, +) (CompletionResolution, bool) { + if state == nil || state.resolving || state.phase != parkParked || state.ticket != ticket || + !validParkTicket(ticket) || !validTaskCancelState(state.taskCancelKind, state.taskCancelPhase) || + state.cancelKind > ParkCancelShutdown || state.expected != 1 || state.attached != 1 || + state.hasDefault || state.outcome != ParkOutcomePending || state.winnerCase != 0 || + state.winnerID != (OperationID{}) || state.winnerRecord != nil || state.head == nil || + state.head.previous != nil || state.head.next != nil || state.head.operation != forced || + forced == nil || !validPendingParkResolutionLink(state, ticket, state.head) || + !operationCandidateExternallyCommitted(forced) { + return CompletionResolution{}, false + } + + state.seed = 0 + state.phase = parkDetaching + if state.cancelKind == ParkCancelTaskAbort || state.cancelKind == ParkCancelShutdown { + forced.resultTicket = ParkTicket{} + forced.cancelRequested = true + forced.disposition = OperationDispositionCanceled + state.outcome = ParkOutcomeCanceled + state.winnerCase = 0 + state.winnerID = OperationID{} + state.winnerRecord = nil + return CompletionResolution{WaitSets: 1, Canceled: 1, Losers: 1}, true + } + + if forced.resultState != operationResultOwned || !commitOperationCandidate(forced) { + return CompletionResolution{}, false + } + forced.resultTicket = ticket + forced.disposition = OperationDispositionWinner + state.outcome = ParkOutcomeCompleted + state.winnerCase = forced.link.caseID + state.winnerID = forced.id + state.winnerRecord = forced + return CompletionResolution{WaitSets: 1, Completed: 1, Winners: 1}, true +} + func beginParkSnapshotResolution(state *ParkState, ticket ParkTicket, cursor *parkResolutionCursor, fullAudit bool) bool { if cursor == nil || *cursor != (parkResolutionCursor{}) || state == nil || state.resolving || state.phase != parkParked || state.ticket != ticket || !validParkTicket(ticket) || diff --git a/runtime/internal/coro/park_state_v2.go b/runtime/internal/coro/park_state_v2.go index 990132fcf7..88289513c9 100644 --- a/runtime/internal/coro/park_state_v2.go +++ b/runtime/internal/coro/park_state_v2.go @@ -151,10 +151,15 @@ type ParkLink struct { // irreversible/reservable winners. // All ParkState and ParkLink operations are strictly owner-P-only. type ParkState struct { - ticket ParkTicket - phase parkPhase - hasDefault bool - resolving bool + ticket ParkTicket + phase parkPhase + hasDefault bool + resolving bool + // directChannel occupies the final byte before expected's alignment. It is + // set only on a source-free compact hchan materialization and consumed by + // the immediately resumed compiler prologue; no native/WASM ABI size or + // following-field offset changes. + directChannel bool expected uint32 attached uint32 seed uint32 @@ -191,7 +196,15 @@ func validMaterializedParkHeader(state *ParkState) bool { return false } switch state.outcome { - case ParkOutcomeCompleted, ParkOutcomeDefault: + case ParkOutcomeCompleted: + if state.directChannel { + return state.winnerCase == 1 + } + return state.winnerCase != 0 + case ParkOutcomeDefault: + if state.directChannel { + return false + } return state.winnerCase != 0 case ParkOutcomeCanceled: return state.winnerCase == 0 @@ -202,7 +215,7 @@ func validMaterializedParkHeader(state *ParkState) bool { func validParkState(state *ParkState) bool { if state == nil || state.resolving || !validTaskCancelState(state.taskCancelKind, state.taskCancelPhase) || state.cancelKind > ParkCancelShutdown || - state.attached > state.expected { + state.attached > state.expected || state.directChannel && state.phase != parkMaterialized { return false } links := uint32(0) @@ -325,11 +338,47 @@ func validParkState(state *ParkState) bool { } } -func releasableParkState(state *ParkState) bool { - if !validParkState(state) { +// validReleasableParkState is the complete O(1) validator for phases which +// cannot retain a source link. These phases are on every spawn, dispatch, and +// terminal-G path; routing them through validParkState needlessly enters the +// generic intrusive-list validator even though their exact invariant requires +// head == nil and attached == 0. +func validReleasableParkState(state *ParkState) bool { + if state == nil || state.resolving || + !validTaskCancelState(state.taskCancelKind, state.taskCancelPhase) || + state.cancelKind > ParkCancelShutdown || state.attached > state.expected || state.directChannel { return false } - return state.phase == parkIdle || state.phase == parkConsumed || state.phase == parkDelivered + switch state.phase { + case parkIdle: + return state.ticket == (ParkTicket{}) && state.expected == 0 && state.attached == 0 && + state.seed == 0 && !state.hasDefault && state.cancelKind == ParkCancelNone && + state.outcome == ParkOutcomePending && state.winnerCase == 0 && + state.winnerID == (OperationID{}) && state.winnerRecord == nil && state.head == nil + case parkConsumed: + if !validParkTicket(state.ticket) || state.attached != 0 || state.head != nil { + return false + } + return state.outcome == ParkOutcomeCompleted && !state.hasDefault && + state.cancelKind < ParkCancelTaskAbort && state.winnerID.Valid() && state.winnerRecord == nil || + state.outcome == ParkOutcomeCanceled && !state.hasDefault && + state.cancelKind != ParkCancelNone && state.winnerCase == 0 && + state.winnerID == (OperationID{}) && state.winnerRecord == nil || + state.outcome == ParkOutcomeDefault && state.hasDefault && + state.cancelKind == ParkCancelNone && state.winnerID == (OperationID{}) && + state.winnerRecord == nil + case parkDelivered: + return validParkTicket(state.ticket) && state.expected == 0 && state.attached == 0 && + state.seed == 0 && !state.hasDefault && state.cancelKind == ParkCancelNone && + state.outcome == ParkOutcomePending && state.winnerCase == 0 && + state.winnerID == (OperationID{}) && state.winnerRecord == nil && state.head == nil + default: + return false + } +} + +func releasableParkState(state *ParkState) bool { + return validReleasableParkState(state) } func materializedParkState(state *ParkState, ticket ParkTicket, outcome ParkOutcome, caseID uint32) bool { @@ -350,11 +399,36 @@ func materializedParkState(state *ParkState, ticket ParkTicket, outcome ParkOutc outcome: outcome, winnerCase: caseID, } - return validParkState(state) + return validMaterializedParkState(state, ticket) +} + +func validMaterializedParkState(state *ParkState, ticket ParkTicket) bool { + return state != nil && state.phase == parkMaterialized && state.ticket == ticket && + !state.resolving && validTaskCancelState(state.taskCancelKind, state.taskCancelPhase) && + validMaterializedParkHeader(state) +} + +func materializeConsumedParkStateUnchecked( + state *ParkState, + ticket ParkTicket, + outcome ParkOutcome, + caseID uint32, +) { + kind, phase := state.taskCancelKind, state.taskCancelPhase + preferredRoute := state.seed + *state = ParkState{ + ticket: ticket, + phase: parkMaterialized, + seed: preferredRoute, + taskCancelKind: kind, + taskCancelPhase: phase, + outcome: outcome, + winnerCase: caseID, + } } func deliverMaterializedParkResume(state *ParkState, ticket ParkTicket) bool { - if !validParkState(state) || state.phase != parkMaterialized || state.ticket != ticket { + if !validMaterializedParkState(state, ticket) { return false } kind, phase := state.taskCancelKind, state.taskCancelPhase @@ -364,7 +438,7 @@ func deliverMaterializedParkResume(state *ParkState, ticket ParkTicket) bool { taskCancelKind: kind, taskCancelPhase: phase, } - return validParkState(state) + return true } // BeginParkSet starts one logical N-candidate wait. The two-word ticket only diff --git a/runtime/internal/coro/park_state_v2_test.go b/runtime/internal/coro/park_state_v2_test.go index eee2fb9711..fd19bcae6e 100644 --- a/runtime/internal/coro/park_state_v2_test.go +++ b/runtime/internal/coro/park_state_v2_test.go @@ -183,6 +183,45 @@ func TestOperationIDIsTwoWordPODAndFailsClosedAtExhaustion(t *testing.T) { } } +func TestReleasableParkStateFastValidatorMatchesCompleteInvariant(t *testing.T) { + completedID, ok := MakeOperationID(OperationSourceWait, 1, 1) + if !ok { + t.Fatal("make completed operation ID") + } + valid := []ParkState{ + {}, + {ticket: ParkTicket{generation: 1}, phase: parkConsumed, expected: 3, + outcome: ParkOutcomeCompleted, winnerCase: 2, winnerID: completedID}, + {ticket: ParkTicket{generation: 2}, phase: parkConsumed, expected: 1, + cancelKind: ParkCancelOperation, outcome: ParkOutcomeCanceled}, + {ticket: ParkTicket{generation: 3}, phase: parkConsumed, expected: 2, + hasDefault: true, outcome: ParkOutcomeDefault, winnerCase: 7}, + {ticket: ParkTicket{epoch: 1, generation: 1}, phase: parkDelivered}, + } + for index := range valid { + state := &valid[index] + if !validParkState(state) || !validReleasableParkState(state) || !releasableParkState(state) { + t.Fatalf("valid state %d rejected: %+v", index, *state) + } + } + + invalid := []ParkState{ + {resolving: true}, + {taskCancelKind: TaskCancelAbort}, + {phase: parkIdle, expected: 1}, + {ticket: ParkTicket{generation: 1}, phase: parkConsumed, attached: 1}, + {ticket: ParkTicket{generation: 1}, phase: parkConsumed}, + {ticket: ParkTicket{generation: 1}, phase: parkDelivered, head: &ParkLink{}}, + {ticket: ParkTicket{generation: 1}, phase: parkReady}, + } + for index := range invalid { + state := &invalid[index] + if validReleasableParkState(state) || releasableParkState(state) { + t.Fatalf("invalid state %d accepted: %+v", index, *state) + } + } +} + func TestZeroCandidateParkCanOnlyResumeThroughLogicalCancel(t *testing.T) { var state ParkState ticket, ok := BeginParkSet(&state, 0, 3) diff --git a/runtime/internal/coro/preempt_atomic_host.go b/runtime/internal/coro/preempt_atomic_host.go index d32c096aa7..3e111233d4 100644 --- a/runtime/internal/coro/preempt_atomic_host.go +++ b/runtime/internal/coro/preempt_atomic_host.go @@ -18,7 +18,10 @@ package coro -import "sync/atomic" +import ( + "sync/atomic" + "unsafe" +) func preemptLoad(ptr *uint32) uint32 { return atomic.LoadUint32(ptr) @@ -51,3 +54,15 @@ func preemptLoadWord(ptr *uintptr) uintptr { func preemptStoreWord(ptr *uintptr, value uintptr) { atomic.StoreUintptr(ptr, value) } + +func preemptLoadPointer(ptr *unsafe.Pointer) unsafe.Pointer { + return atomic.LoadPointer(ptr) +} + +func preemptStorePointer(ptr *unsafe.Pointer, value unsafe.Pointer) { + atomic.StorePointer(ptr, value) +} + +func preemptSwapPointer(ptr *unsafe.Pointer, value unsafe.Pointer) unsafe.Pointer { + return atomic.SwapPointer(ptr, value) +} diff --git a/runtime/internal/coro/preempt_atomic_llgo.go b/runtime/internal/coro/preempt_atomic_llgo.go index 6663c76702..0b0aeb7f11 100644 --- a/runtime/internal/coro/preempt_atomic_llgo.go +++ b/runtime/internal/coro/preempt_atomic_llgo.go @@ -18,7 +18,11 @@ package coro -import "github.com/goplus/llgo/runtime/internal/clite/sync/atomic" +import ( + "unsafe" + + "github.com/goplus/llgo/runtime/internal/clite/sync/atomic" +) // Keep these operations as target-lowered sequentially consistent uint32 // atomics. A target without native 32-bit atomics (for example RV32IMC without @@ -55,3 +59,15 @@ func preemptLoadWord(ptr *uintptr) uintptr { func preemptStoreWord(ptr *uintptr, value uintptr) { atomic.Store(ptr, value) } + +func preemptLoadPointer(ptr *unsafe.Pointer) unsafe.Pointer { + return atomic.Load(ptr) +} + +func preemptStorePointer(ptr *unsafe.Pointer, value unsafe.Pointer) { + atomic.Store(ptr, value) +} + +func preemptSwapPointer(ptr *unsafe.Pointer, value unsafe.Pointer) unsafe.Pointer { + return atomic.Exchange(ptr, value) +} diff --git a/runtime/internal/coro/published_epoch_resolution.go b/runtime/internal/coro/published_epoch_resolution.go index 394abd36b2..bfb2d2441e 100644 --- a/runtime/internal/coro/published_epoch_resolution.go +++ b/runtime/internal/coro/published_epoch_resolution.go @@ -53,7 +53,15 @@ type publishedEpochResolveCursor struct { waitAwait bool hasChannel bool claimOwned bool - _ [3]byte + // directChannel is an owner-only certificate installed only after the + // one-case channel fast lane has completed its fully audited resolve, + // detach, and ParkSet consume transaction. It lets the intervening typed + // materialization and final source retirement validate the exact scalar + // continuation instead of re-walking the already destroyed ParkLink graph. + // It occupies existing padding and is cleared before advancing to another + // wait, so the persisted cursor layout does not grow. + directChannel bool + _ [2]byte } type publishedEpochResolveStep struct { @@ -258,6 +266,7 @@ func advancePublishedEpochWaitAfterCleared(sources *ExecutorSourceSet, cursor *p cursor.forced = nil cursor.hasChannel = false cursor.claimOwned = false + cursor.directChannel = false cursor.waitRetry = false cursor.waitAwait = false if next != nil { @@ -535,7 +544,7 @@ func resolvePublishedEpochPromoteStep(sources *ExecutorSourceSet, p *P, cursor * } if wait.resumeKind == resumeBindingCleanup { plan := (*ResumeCleanupPlan)(wait.resume) - if !validResumeCleanupPlan(wait, plan) { + if !validTrustedResumeCleanupPlanState(wait, plan) { return false } switch plan.phase { diff --git a/runtime/internal/coro/resume_cleanup.go b/runtime/internal/coro/resume_cleanup.go index 80caa61c6a..3d7de05642 100644 --- a/runtime/internal/coro/resume_cleanup.go +++ b/runtime/internal/coro/resume_cleanup.go @@ -87,8 +87,11 @@ type ResumeCleanupPlan struct { phase resumeCleanupPhase result ResumeResultKind small uint8 + verified uint8 } +const resumeCleanupPlanVerifiedV1 uint8 = 0xa7 + func validResumeCleanupKind(kind ResumeCleanupKind) bool { return kind >= ResumeCleanupChannelDirect && kind <= ResumeCleanupKeyedPark } @@ -303,6 +306,21 @@ func BindWaitSetResumeCleanup( !validResumeCleanupBindingForWait(record, packet, plan, binding) { return false } + installWaitSetResumeCleanup(record, packet, plan, binding) + return validResumeCleanupPlan(record, plan) +} + +// installWaitSetResumeCleanup is the no-fail write half shared by the general +// audited binding and the fused compiler-owned single-channel transaction. +// Its caller must own the current P and prove all descriptor/storage relations +// before entry; keeping mutation separate prevents the direct path from +// immediately repeating the graph audit it just completed. +func installWaitSetResumeCleanup( + record *WaitSetRecord, + packet *ResumePacket, + plan *ResumeCleanupPlan, + binding ResumeCleanupBinding, +) { *packet = ResumePacket{ ticket: record.ticket, state: resumePacketBound, @@ -319,15 +337,16 @@ func BindWaitSetResumeCleanup( runtime: binding.RuntimeCount, kind: binding.Kind, phase: resumeCleanupBound, + verified: resumeCleanupPlanVerifiedV1, } record.resume = unsafe.Pointer(plan) record.resumeKind = resumeBindingCleanup - return validResumeCleanupPlan(record, plan) } func validResumeCleanupPlan(record *WaitSetRecord, plan *ResumeCleanupPlan) bool { if record == nil || plan == nil || record.resume != unsafe.Pointer(plan) || record.resumeKind != resumeBindingCleanup || plan.packet == nil || + plan.verified != resumeCleanupPlanVerifiedV1 || !validResumeCleanupPacket(plan, record.ticket) || plan.context == nil || plan.ticket != record.ticket || !validResumeCleanupKind(plan.kind) || @@ -350,7 +369,10 @@ func validResumeCleanupPlan(record *WaitSetRecord, plan *ResumeCleanupPlan) bool case resumeCleanupConfirm, resumeCleanupRecycle: return plan.index < plan.count && plan.outcome != ParkOutcomePending && record.g != nil && record.g.park.phase == parkConsumed - case resumeCleanupClaim, resumeCleanupResult, resumeCleanupFinalize: + case resumeCleanupClaim: + return plan.index == plan.count && plan.outcome != ParkOutcomePending && + record.g != nil && record.g.park.phase == parkConsumed + case resumeCleanupResult, resumeCleanupFinalize: return plan.index == 0 && plan.outcome != ParkOutcomePending && record.g != nil && record.g.park.phase == parkConsumed default: @@ -358,8 +380,47 @@ func validResumeCleanupPlan(record *WaitSetRecord, plan *ResumeCleanupPlan) bool } } +// validTrustedResumeCleanupPlanState is the bounded transition validator used +// after BindWaitSetResumeCleanup (or the fused direct-channel binder) has +// audited the immutable descriptor and installed verified. Runtime reductions +// alone may mutate phase/index/outcome/result fields, so repeating range, kind, +// context, and source-shape validation on every one-step reduction only walks +// the same compiler-owned frame graph again. Public binding and whole-G audit +// boundaries continue to call validResumeCleanupPlan. +func validTrustedResumeCleanupPlanState(record *WaitSetRecord, plan *ResumeCleanupPlan) bool { + if record == nil || plan == nil || record.resume != unsafe.Pointer(plan) || + record.resumeKind != resumeBindingCleanup || plan.verified != resumeCleanupPlanVerifiedV1 || + plan.packet == nil || plan.packet.state != resumePacketBound || + plan.packet.ticket != record.ticket || plan.ticket != record.ticket || + plan.outcome > ParkOutcomeDefault || record.g == nil { + return false + } + switch plan.phase { + case resumeCleanupBound: + return plan.index == 0 && plan.caseID == 0 && plan.outcome == ParkOutcomePending && + plan.lease == (OperationResultLease{}) && plan.result == ResumeResultNone && + plan.small == ResumeSmallInvalid && + (record.g.park.phase == parkParked || record.g.park.phase == parkDetaching || + record.g.park.phase == parkReady) + case resumeCleanupRuntime: + return plan.index < plan.runtime && plan.outcome != ParkOutcomePending && + record.g.park.phase == parkConsumed + case resumeCleanupConfirm, resumeCleanupRecycle: + return plan.index < plan.count && plan.outcome != ParkOutcomePending && + record.g.park.phase == parkConsumed + case resumeCleanupClaim: + return plan.index == plan.count && plan.outcome != ParkOutcomePending && + record.g.park.phase == parkConsumed + case resumeCleanupResult, resumeCleanupFinalize: + return plan.index == 0 && plan.outcome != ParkOutcomePending && + record.g.park.phase == parkConsumed + default: + return false + } +} + func beginResumeCleanup(record *WaitSetRecord, plan *ResumeCleanupPlan) bool { - if !validResumeCleanupPlan(record, plan) || plan.phase != resumeCleanupBound { + if !validTrustedResumeCleanupPlanState(record, plan) || plan.phase != resumeCleanupBound { return false } state := &record.g.park @@ -387,7 +448,7 @@ func beginResumeCleanup(record *WaitSetRecord, plan *ResumeCleanupPlan) bool { } plan.outcome, plan.caseID, plan.lease = outcome, caseID, lease plan.phase = resumeCleanupRuntime - return validResumeCleanupPlan(record, plan) + return validTrustedResumeCleanupPlanState(record, plan) } // ResumeCleanupStep is one direct-runtime typed cleanup reduction. The @@ -407,12 +468,26 @@ func pendingResumeCleanupStepForCursor(cursor *publishedEpochResolveCursor) (Res if cursor == nil || cursor.phase != publishedEpochResolvePromote { return ResumeCleanupStep{}, false } + if cursor.directChannel { + _, plan, _, ok := ownerLocalDirectChannelCleanupHeader(cursor) + if !ok || plan.phase != resumeCleanupRuntime { + return ResumeCleanupStep{}, false + } + return ResumeCleanupStep{ + Kind: plan.kind, + Context: plan.context, + Index: plan.index, + WinnerCase: plan.caseID, + Outcome: plan.outcome, + plan: plan, + }, true + } record := cursor.wait if record == nil || record.resumeKind != resumeBindingCleanup { return ResumeCleanupStep{}, false } plan := (*ResumeCleanupPlan)(record.resume) - if !validResumeCleanupPlan(record, plan) || plan.phase != resumeCleanupRuntime { + if !validTrustedResumeCleanupPlanState(record, plan) || plan.phase != resumeCleanupRuntime { return ResumeCleanupStep{}, false } return ResumeCleanupStep{ @@ -572,6 +647,86 @@ func recycleResumeCleanupOperation( } } +func finalizeResumeCleanup(record *WaitSetRecord, plan *ResumeCleanupPlan) bool { + if record == nil || plan == nil || plan.packet == nil || + plan.outcome != ParkOutcomeCompleted && + (plan.result != ResumeResultNone || plan.packet.scalar != (ScalarResultPayloadV1{}) || + plan.small != ResumeSmallInvalid) { + return false + } + packet := plan.packet + ticket, outcome, caseID := plan.ticket, plan.outcome, plan.caseID + result, scalar, small := plan.result, packet.scalar, plan.small + if !materializedParkState(&record.g.park, ticket, outcome, caseID) { + return false + } + *packet = ResumePacket{ + ticket: ticket, + scalar: scalar, + caseID: caseID, + outcome: outcome, + result: result, + small: small, + state: resumePacketMaterialized, + } + record.resume = unsafe.Pointer(packet) + record.resumeKind = resumeBindingMaterialized + *plan = ResumeCleanupPlan{} + return validMaterializedResumePacket(packet) +} + +// finishSingleClaimlessResumeCleanup collapses the source-neutral tail for one +// already materialized worker or manual operation. The verified descriptor +// proves that this fixed-size path has no select claim and exactly one source +// generation. Keep the same Confirm -> Result -> Recycle -> Finalize ordering, +// but resolve the frame-local OperationID once and avoid four scheduler cursor +// transitions. Multi-event cleanup retains the independently bounded machine. +func finishSingleClaimlessResumeCleanup( + sources *ExecutorSourceSet, + p *P, + record *WaitSetRecord, + plan *ResumeCleanupPlan, +) (finalized bool, ok bool) { + id := resumeCleanupIDAt(plan, 0) + if id == nil || *id == (OperationID{}) || + !confirmResumeCleanupOperation(sources, p, *id) { + return false, false + } + if plan.lease.Valid() { + if plan.outcome != ParkOutcomeCompleted || plan.caseID != 1 { + return false, false + } + result, taken := takeResumeCleanupResult( + sources, + p, + *id, + plan.lease, + &plan.packet.scalar, + &plan.small, + ) + if !taken { + return false, false + } + plan.result = result + preferredRoute, routeOK := resumeCleanupCompletionRoute(sources, p, *id) + if !routeOK || !setConsumedParkPreferredRoute( + &record.g.park, + plan.ticket, + preferredRoute, + ) { + return false, false + } + } else if plan.outcome == ParkOutcomeCompleted { + return false, false + } + plan.lease = OperationResultLease{} + if !recycleResumeCleanupOperation(sources, p, *id) { + return false, false + } + *id = OperationID{} + return true, finalizeResumeCleanup(record, plan) +} + // advanceResumeCleanupCore performs one source-neutral bounded reduction after // the runtime has removed every typed queue node. func advanceResumeCleanupCore( @@ -581,9 +736,13 @@ func advanceResumeCleanupCore( plan *ResumeCleanupPlan, ) (finalized bool, ok bool) { if sources == nil || p == nil || !validExecutorSourceSetHeader(sources, p) || - !validResumeCleanupPlan(record, plan) { + !validTrustedResumeCleanupPlanState(record, plan) { return false, false } + if plan.phase == resumeCleanupConfirm && plan.index == 0 && plan.count == 1 && + plan.runtime == 1 && plan.claim == nil { + return finishSingleClaimlessResumeCleanup(sources, p, record, plan) + } switch plan.phase { case resumeCleanupConfirm: if plan.index < plan.count { @@ -594,15 +753,20 @@ func advanceResumeCleanupCore( } plan.index++ if plan.index == plan.count { - plan.index = 0 plan.phase = resumeCleanupClaim } return false, true } case resumeCleanupClaim: - if plan.claim != nil && !sources.channel.ResetSelectClaim(p, plan.claim) { + if plan.claim != nil && !sources.channel.ResetSelectClaimAfterConfirmed( + p, + plan.claim, + plan.index, + plan.count, + ) { return false, false } + plan.index = 0 plan.phase = resumeCleanupResult return false, true case resumeCleanupResult: @@ -660,30 +824,7 @@ func advanceResumeCleanupCore( return false, true } case resumeCleanupFinalize: - if plan.outcome != ParkOutcomeCompleted && - (plan.result != ResumeResultNone || plan.packet.scalar != (ScalarResultPayloadV1{}) || - plan.small != ResumeSmallInvalid) { - return false, false - } - packet := plan.packet - ticket, outcome, caseID := plan.ticket, plan.outcome, plan.caseID - result, scalar, small := plan.result, packet.scalar, plan.small - if !materializedParkState(&record.g.park, ticket, outcome, caseID) { - return false, false - } - *packet = ResumePacket{ - ticket: ticket, - scalar: scalar, - caseID: caseID, - outcome: outcome, - result: result, - small: small, - state: resumePacketMaterialized, - } - record.resume = unsafe.Pointer(packet) - record.resumeKind = resumeBindingMaterialized - *plan = ResumeCleanupPlan{} - return true, validMaterializedResumePacket(packet) + return true, finalizeResumeCleanup(record, plan) } return false, false } diff --git a/runtime/internal/coro/resume_cleanup_test.go b/runtime/internal/coro/resume_cleanup_test.go index c056adbbc4..8c9dc19b52 100644 --- a/runtime/internal/coro/resume_cleanup_test.go +++ b/runtime/internal/coro/resume_cleanup_test.go @@ -93,6 +93,23 @@ func TestTypedResumeCleanupIsBoundedAndPNeutral(t *testing.T) { if !BindWaitSetResumeCleanup(&fixture.wait, &packet, &plan, binding) { t.Fatal("bind typed resume cleanup") } + if plan.verified != resumeCleanupPlanVerifiedV1 || + !validResumeCleanupPlan(&fixture.wait, &plan) || + !validTrustedResumeCleanupPlanState(&fixture.wait, &plan) || + !validTrustedWaitSetResumeBinding(&fixture.wait) { + t.Fatal("bound cleanup plan lacks its one-time descriptor certificate") + } + plan.verified = 0 + if validResumeCleanupPlan(&fixture.wait, &plan) || + validTrustedResumeCleanupPlanState(&fixture.wait, &plan) || + validTrustedWaitSetResumeBinding(&fixture.wait) { + t.Fatal("cleanup validators accepted an uncertified descriptor") + } + plan.verified = resumeCleanupPlanVerifiedV1 + if !validResumeCleanupPlan(&fixture.wait, &plan) || + !validTrustedResumeCleanupPlanState(&fixture.wait, &plan) { + t.Fatal("cleanup validators rejected a restored descriptor certificate") + } }, ) externallyCommitChannelCandidateAtRoute(t, fixture, 1, 7) diff --git a/runtime/internal/coro/resume_packet.go b/runtime/internal/coro/resume_packet.go index ecb2f137b8..6980cb49a4 100644 --- a/runtime/internal/coro/resume_packet.go +++ b/runtime/internal/coro/resume_packet.go @@ -47,6 +47,7 @@ const ( resumeBindingSingle resumeBindingCleanup resumeBindingMaterialized + resumeBindingDirectChannel ) // ResumePacket is stable storage in the direct-parking LLVM coroutine frame. @@ -159,19 +160,100 @@ func validMaterializedResumePacket(packet *ResumePacket) bool { } } +// directChannelBoundResumeHeader is the hot selector for a fused direct +// channel park before source resolution starts. The immutable descriptor was +// fully audited when directChannel was installed; producers can change only +// source atomics and SelectClaim during suspension. The post-suspend commit +// boundary calls validDirectChannelBoundResumeState once before queue effects. +func directChannelBoundResumeHeader(record *WaitSetRecord) (*ResumeCleanupPlan, bool) { + if record == nil || !record.directChannel || record.resume == nil || + record.resumeKind != resumeBindingCleanup || !validParkTicket(record.ticket) { + return nil, false + } + plan := (*ResumeCleanupPlan)(record.resume) + if plan == nil || plan.phase != resumeCleanupBound || + plan.kind != ResumeCleanupChannelDirect || plan.ticket != record.ticket || + plan.verified != resumeCleanupPlanVerifiedV1 || + plan.packet == nil || plan.packet.state != resumePacketBound || + plan.packet.ticket != record.ticket { + return nil, false + } + return plan, true +} + +// validDirectChannelBoundResumeState performs the complete descriptor audit +// exactly once at the post-llvm.coro.suspend queue-commit boundary. +func validDirectChannelBoundResumeState(record *WaitSetRecord, plan *ResumeCleanupPlan) bool { + headerPlan, headerOK := directChannelBoundResumeHeader(record) + if !headerOK || headerPlan != plan || plan.claim == nil || plan.context == nil || + plan.entries == nil || + plan.stride != unsafe.Sizeof(OperationID{}) || plan.idOffset != 0 || + plan.count != 1 || plan.runtime != 1 || plan.index != 0 || + plan.caseID != 0 || plan.outcome != ParkOutcomePending || + plan.lease != (OperationResultLease{}) || plan.result != ResumeResultNone || + plan.small != ResumeSmallInvalid { + return false + } + id := (*OperationID)(plan.entries) + packet := plan.packet + return id.Valid() && id.Source() == OperationSourceChannel && + packet.state == resumePacketBound && packet.ticket == record.ticket && + packet.source == (OperationID{}) && packet.scalar == (ScalarResultPayloadV1{}) && + packet.caseID == 0 && packet.outcome == ParkOutcomePending && + packet.result == ResumeResultNone && packet.small == ResumeSmallInvalid +} + func validWaitSetResumeBinding(record *WaitSetRecord) bool { if record == nil { return false } switch record.resumeKind { case resumeBindingNone: - return record.resume == nil + return !record.directChannel && record.resume == nil case resumeBindingSingle: - return validBoundResumePacket((*ResumePacket)(record.resume), record.ticket) + return !record.directChannel && validBoundResumePacket((*ResumePacket)(record.resume), record.ticket) case resumeBindingCleanup: - return validResumeCleanupPlan(record, (*ResumeCleanupPlan)(record.resume)) + plan := (*ResumeCleanupPlan)(record.resume) + if record.directChannel && plan != nil && plan.phase == resumeCleanupBound { + _, ok := directChannelBoundResumeHeader(record) + return ok + } + return validResumeCleanupPlan(record, plan) case resumeBindingMaterialized: return validMaterializedResumePacket((*ResumePacket)(record.resume)) + case resumeBindingDirectChannel: + return validBoundDirectChannelCompletion(record, (*DirectChannelCompletion)(record.resume)) + default: + return false + } +} + +// validTrustedWaitSetResumeBinding is the active scheduler-record selector. +// Bind/commit performed the complete descriptor audit before the record became +// reachable from P. While active, only the runtime cleanup machine can mutate +// a cleanup plan, so hot queue/cursor checks validate its certified transition +// state without re-walking immutable frame/source descriptors. Full queue and +// whole-G audits continue through validWaitSetResumeBinding. +func validTrustedWaitSetResumeBinding(record *WaitSetRecord) bool { + if record == nil { + return false + } + switch record.resumeKind { + case resumeBindingNone: + return !record.directChannel && record.resume == nil + case resumeBindingSingle: + return !record.directChannel && validBoundResumePacket((*ResumePacket)(record.resume), record.ticket) + case resumeBindingCleanup: + plan := (*ResumeCleanupPlan)(record.resume) + if record.directChannel && plan != nil && plan.phase == resumeCleanupBound { + _, ok := directChannelBoundResumeHeader(record) + return ok + } + return validTrustedResumeCleanupPlanState(record, plan) + case resumeBindingMaterialized: + return validMaterializedResumePacket((*ResumePacket)(record.resume)) + case resumeBindingDirectChannel: + return validBoundDirectChannelCompletion(record, (*DirectChannelCompletion)(record.resume)) default: return false } @@ -210,6 +292,19 @@ func BindSingleWaitSetResumePacket(record *WaitSetRecord, packet *ResumePacket, state.head.next != nil || state.head.operation == nil || state.head.operation.id != source { return false } + installSingleWaitSetResumePacket(record, packet, source) + return true +} + +// installSingleWaitSetResumePacket is the no-fail write half shared by the +// general audited binder and compiler-owned one-event source transactions. +// The latter have already built and audited the exact ParkLink relation in the +// same owner-P interval, so immediately walking it again carries no new fact. +func installSingleWaitSetResumePacket( + record *WaitSetRecord, + packet *ResumePacket, + source OperationID, +) { *packet = ResumePacket{ ticket: record.ticket, source: source, @@ -217,7 +312,6 @@ func BindSingleWaitSetResumePacket(record *WaitSetRecord, packet *ResumePacket, } record.resume = unsafe.Pointer(packet) record.resumeKind = resumeBindingSingle - return true } func materializeManualResume( @@ -437,7 +531,238 @@ func materializeSingleResumePacket(sources *ExecutorSourceSet, p *P, record *Wai return validMaterializedResumePacket(packet) } -// TakeResumePacket is the typed resume prologue for packet-backed parks. It +// TakeDirectChannelResume retains the arbitrary-runner compatibility entry. +// New compiler lowering calls TakeIssuedDirectChannelResume after the bounded +// selector has issued the exact physical resume capability. +func TakeDirectChannelResume( + g *G, + storage *DirectChannelParkStorageV1, +) ( + outcome ParkOutcome, + task TaskCancelKind, + small uint8, + ok bool, +) { + if g == nil || storage == nil || !validParkTicket(storage.Ticket) { + return ParkOutcomePending, TaskCancelNone, ResumeSmallInvalid, false + } + p := g.runP + if p != nil && p.executor != nil && p.executor.run.issued == ActionCheckResume { + return TakeIssuedDirectChannelResume(g, storage) + } + return takeDirectChannelResumeCompatibility( + g, storage.Ticket, &storage.Completion, &storage.Wait, unsafe.Pointer(storage), + ) +} + +// DirectChannelResumeWordV1 is the scalar compiler/runtime result of one exact +// issued channel continuation. Bits 8..9 identify completion or cancellation; +// the low byte contains the physical channel status or TaskCancelKind. Keeping +// this boundary scalar avoids lowering four logical results through an +// aggregate ABI in every generated channel continuation. +type DirectChannelResumeWordV1 uint32 + +const ( + DirectChannelResumeWordInvalidV1 DirectChannelResumeWordV1 = 0 + DirectChannelResumeWordCompletedV1 DirectChannelResumeWordV1 = 1 << 8 + DirectChannelResumeWordCanceledV1 DirectChannelResumeWordV1 = 2 << 8 + DirectChannelResumeWordClassMaskV1 DirectChannelResumeWordV1 = 3 << 8 + DirectChannelResumeWordPayloadMaskV1 DirectChannelResumeWordV1 = 0xff +) + +// TakeIssuedDirectChannelResumeWordV1 is the compiler-owned one-case channel +// resume prologue. DirectChannelCompletion already reduced the physical hchan +// effect into its frame-local terminal word before making G runnable. Checked +// then copied that exact decision into P. This function consumes the adjacent +// completion/decision/ParkState certificates without constructing either the +// generic scalar/poll ResumePacket union or a multi-result return aggregate. +// +// A prompt task stop may suppress an already copied channel value. In that +// case completion and ParkState retain the completed physical fact while the P +// decision carries Canceled plus the task token; the compiler observes only +// the cancellation edge. +func TakeIssuedDirectChannelResumeWordV1( + g *G, + storage *DirectChannelParkStorageV1, +) DirectChannelResumeWordV1 { + if g == nil || storage == nil || !validParkTicket(storage.Ticket) { + return DirectChannelResumeWordInvalidV1 + } + expected := storage.Ticket + completion := &storage.Completion + wait := &storage.Wait + context := unsafe.Pointer(storage) + p := g.runP + if p == nil || p.executor == nil || p.executor.run.issued != ActionCheckResume || + p.current != g || !p.inResume || p.runDecisionTaken { + return DirectChannelResumeWordInvalidV1 + } + decision := &p.runDecision + if decision.g != g || decision.ticket != expected || !decision.materialized || + !decision.directChannel { + return DirectChannelResumeWordInvalidV1 + } + completionState := preemptLoad(&completion.state) + physicalSmallWord := preemptLoad(&completion.small) + if completion.context != context || completion.wait != wait || + completionState != uint32(directChannelCompletionMaterialized) || + physicalSmallWord > uint32(^uint8(0)) { + return DirectChannelResumeWordInvalidV1 + } + park := &g.park + physicalSmall := uint8(physicalSmallWord) + result := DirectChannelResumeWordInvalidV1 + switch decision.outcome { + case ParkOutcomeCompleted: + if decision.caseID != 1 || decision.task != TaskCancelNone || + physicalSmall == ResumeSmallInvalid { + return DirectChannelResumeWordInvalidV1 + } + result = DirectChannelResumeWordCompletedV1 | DirectChannelResumeWordV1(physicalSmall) + case ParkOutcomeCanceled: + if decision.caseID != 0 || !validTaskCancelKind(decision.task) { + return DirectChannelResumeWordInvalidV1 + } + result = DirectChannelResumeWordCanceledV1 | DirectChannelResumeWordV1(decision.task) + default: + return DirectChannelResumeWordInvalidV1 + } + + // BeginIssuedExecutorResumeRuntimeContext copied phase/ticket/directChannel + // into the P-owned decision immediately before llvm.coro.resume. No owner or + // producer can mutate ParkState while this G is running, so replaying those + // three source reads here would not establish a second boundary. The exact + // completion/context correlation above still authenticates the frame-local + // storage which the decision itself does not address. + // + // The direct materialization shape has no source/link payload. Preserve + // its ticket and task-cancellation receipt, and retire only the scalar + // result fields made live by materializeDirectChannelWaitUnchecked. This + // avoids rewriting the entire mostly-zero ParkState on every handoff. + park.phase = parkDelivered + park.directChannel = false + park.seed = 0 + park.cancelKind = ParkCancelNone + park.outcome = ParkOutcomePending + park.winnerCase = 0 + *completion = DirectChannelCompletion{} + p.runDecision = RunDecision{} + p.runDecisionTaken = true + return result +} + +// TakeIssuedDirectChannelResume retains the structured core API for tests and +// compatibility callers. Current compiler lowering consumes the scalar word +// above directly. +func TakeIssuedDirectChannelResume( + g *G, + storage *DirectChannelParkStorageV1, +) ( + outcome ParkOutcome, + task TaskCancelKind, + small uint8, + ok bool, +) { + word := TakeIssuedDirectChannelResumeWordV1(g, storage) + payload := uint8(word & DirectChannelResumeWordPayloadMaskV1) + switch word & DirectChannelResumeWordClassMaskV1 { + case DirectChannelResumeWordCompletedV1: + return ParkOutcomeCompleted, TaskCancelNone, payload, true + case DirectChannelResumeWordCanceledV1: + return ParkOutcomeCanceled, TaskCancelKind(payload), ResumeSmallInvalid, true + default: + return ParkOutcomePending, TaskCancelNone, ResumeSmallInvalid, false + } +} + +func takeDirectChannelResumeCompatibility( + g *G, + expected ParkTicket, + completion *DirectChannelCompletion, + wait *WaitSetRecord, + context unsafe.Pointer, +) ( + outcome ParkOutcome, + task TaskCancelKind, + small uint8, + ok bool, +) { + if g == nil || g.magic != gMagic || !validParkTicket(expected) || + completion == nil || wait == nil || context == nil { + return ParkOutcomePending, TaskCancelNone, ResumeSmallInvalid, false + } + p := g.runP + if p == nil || p.current != g || !p.inResume || g.state != GRunning || + p.runDecisionTaken || p.action.Kind != ActionResume || p.action.Flags != 0 || + p.action.Handle == nil || g.runP != p { + return ParkOutcomePending, TaskCancelNone, ResumeSmallInvalid, false + } + decision := &p.runDecision + if decision.g != g || decision.ticket != expected || !decision.materialized || + !decision.directChannel || + decision.lease != (OperationResultLease{}) { + return ParkOutcomePending, TaskCancelNone, ResumeSmallInvalid, false + } + if completion.context != context || completion.owner == nil || + completion.wait != wait || !completion.route.Valid() || + completion.owner.route != completion.route || + preemptLoad(&completion.state) != uint32(directChannelCompletionMaterialized) || + preemptLoad(&completion.small) > uint32(^uint8(0)) { + return ParkOutcomePending, TaskCancelNone, ResumeSmallInvalid, false + } + physicalSmall := uint8(preemptLoad(&completion.small)) + park := &g.park + if park.phase != parkMaterialized || park.ticket != expected || + !park.directChannel || park.resolving { + return ParkOutcomePending, TaskCancelNone, ResumeSmallInvalid, false + } + resultSmall := uint8(ResumeSmallInvalid) + switch decision.outcome { + case ParkOutcomeCompleted: + if decision.caseID != 1 || decision.task != TaskCancelNone || + park.outcome != ParkOutcomeCompleted || park.winnerCase != 1 || + physicalSmall == ResumeSmallInvalid { + return ParkOutcomePending, TaskCancelNone, ResumeSmallInvalid, false + } + resultSmall = physicalSmall + case ParkOutcomeCanceled: + if decision.caseID != 0 || !validTaskCancelKind(decision.task) || + (park.outcome == ParkOutcomeCompleted && + (park.winnerCase != 1 || physicalSmall == ResumeSmallInvalid)) || + (park.outcome == ParkOutcomeCanceled && + (park.winnerCase != 0 || physicalSmall != ResumeSmallInvalid)) || + (park.outcome != ParkOutcomeCompleted && park.outcome != ParkOutcomeCanceled) { + return ParkOutcomePending, TaskCancelNone, ResumeSmallInvalid, false + } + default: + return ParkOutcomePending, TaskCancelNone, ResumeSmallInvalid, false + } + // prepareRunDecision copied this exact materialized park into the private P + // decision immediately before llvm.coro.resume. The compiler-generated + // continuation reaches this prologue before user code, and no producer owns + // ParkState or the frame completion. The completion owner records where the + // physical fact was materialized; it deliberately does not bind this resume + // to that P because a materialized G is P-neutral and may already have crossed + // a runnable-transfer mailbox. Recheck only the correlation fields needed by + // this consuming mutation; the decision is the full prior certificate. + if !validTaskCancelState(park.taskCancelKind, park.taskCancelPhase) { + return ParkOutcomePending, TaskCancelNone, ResumeSmallInvalid, false + } + // The exact completion/decision/park triple is now consumed. Preserve only the + // sticky task token for the compiler's terminal control path. + outcome, task, small = decision.outcome, decision.task, resultSmall + kind, phase := park.taskCancelKind, park.taskCancelPhase + *park = ParkState{ + ticket: expected, phase: parkDelivered, + taskCancelKind: kind, taskCancelPhase: phase, + } + *completion = DirectChannelCompletion{} + p.runDecision = RunDecision{} + p.runDecisionTaken = true + return outcome, task, small, true +} + +// TakeResumePacket is the typed resume prologue for general packet-backed parks. It // consumes the P-owned logical decision and frame-local result exactly once. // A task stop selected after migration suppresses the copied payload without // consulting or touching the old source. @@ -464,7 +789,7 @@ func TakeResumePacket( return ParkOutcomePending, 0, TaskCancelNone, ResumeResultNone, ResumeSmallInvalid, false } decision := &p.runDecision - if !decision.materialized || decision.g != g || decision.ticket != expected || + if !decision.materialized || decision.directChannel || decision.g != g || decision.ticket != expected || g.park.phase != parkMaterialized || g.park.ticket != expected || packet.outcome != g.park.outcome || packet.caseID != g.park.winnerCase { return ParkOutcomePending, 0, TaskCancelNone, ResumeResultNone, ResumeSmallInvalid, false diff --git a/runtime/internal/coro/resume_packet_test.go b/runtime/internal/coro/resume_packet_test.go index 0c58aed2d5..228dbb987d 100644 --- a/runtime/internal/coro/resume_packet_test.go +++ b/runtime/internal/coro/resume_packet_test.go @@ -73,6 +73,82 @@ func TestResumePacketLayoutIsPointerFreeAndCrossTargetStable(t *testing.T) { } } +func TestTakeIssuedDirectChannelResumeConsumesExactCapabilityOnce(t *testing.T) { + for _, test := range []struct { + name string + decision ParkOutcome + task TaskCancelKind + physicalSmall uint8 + wantSmall uint8 + }{ + { + name: "completed", + decision: ParkOutcomeCompleted, + physicalSmall: 3, + wantSmall: 3, + }, + { + name: "task canceled after completion", + decision: ParkOutcomeCanceled, + task: TaskCancelAbort, + physicalSmall: 3, + wantSmall: ResumeSmallInvalid, + }, + } { + t.Run(test.name, func(t *testing.T) { + ticket := ParkTicket{epoch: 11, generation: 29} + storage := new(DirectChannelParkStorageV1) + storage.Ticket = ticket + storage.Completion.wait = &storage.Wait + storage.Completion.context = unsafe.Pointer(storage) + storage.Completion.state = uint32(directChannelCompletionMaterialized) + storage.Completion.small = uint32(test.physicalSmall) + + g, p, driver := new(G), new(P), new(ExecutorDriver) + g.runP = p + g.park = ParkState{ + ticket: ticket, + phase: parkMaterialized, + directChannel: true, + outcome: ParkOutcomeCompleted, + winnerCase: 1, + } + p.executor = driver + p.current = g + p.inResume = true + p.runDecision = RunDecision{ + g: g, + ticket: ticket, + caseID: 1, + outcome: test.decision, + task: test.task, + materialized: true, + directChannel: true, + } + if test.decision == ParkOutcomeCanceled { + p.runDecision.caseID = 0 + } + driver.p = p + driver.run.issued = ActionCheckResume + + outcome, task, small, ok := TakeIssuedDirectChannelResume(g, storage) + if !ok || outcome != test.decision || task != test.task || small != test.wantSmall { + t.Fatalf("take issued direct resume = (%d,%d,%d,%t), want (%d,%d,%d,true)", + outcome, task, small, ok, test.decision, test.task, test.wantSmall) + } + if g.park.phase != parkDelivered || g.park.directChannel || + storage.Completion != (DirectChannelCompletion{}) || + p.runDecision != (RunDecision{}) || !p.runDecisionTaken { + t.Fatalf("issued direct resume residue: park=%+v completion=%+v decision=%+v taken=%t", + g.park, storage.Completion, p.runDecision, p.runDecisionTaken) + } + if _, _, _, replay := TakeIssuedDirectChannelResume(g, storage); replay { + t.Fatal("issued direct resume capability was consumed twice") + } + }) + } +} + func TestZeroSourceResumePacketMaterializesDefault(t *testing.T) { p, targetP := new(P), new(P) driver, _, _, _ := bindTestExecutorDriverWithManual(t, p) diff --git a/runtime/internal/coro/run_decision.go b/runtime/internal/coro/run_decision.go index bc9f18c293..2f56fb7587 100644 --- a/runtime/internal/coro/run_decision.go +++ b/runtime/internal/coro/run_decision.go @@ -33,7 +33,11 @@ type RunDecision struct { // materialized selects a frame-local ResumePacket rather than an // old-source OperationResultLease. It occupies existing scalar padding. materialized bool - lease OperationResultLease + // directChannel consumes the remaining scalar padding. It is copied only + // from a marked materialized ParkState and lets the compiler's dedicated + // prologue avoid re-proving the generic packet union. + directChannel bool + lease OperationResultLease } func validRunDecision(decision RunDecision) bool { @@ -41,6 +45,7 @@ func validRunDecision(decision RunDecision) bool { return true } if !ValidG(decision.g) || decision.outcome > ParkOutcomeDefault || + decision.directChannel && !decision.materialized || (decision.task != TaskCancelNone && !validTaskCancelKind(decision.task)) { return false } @@ -58,10 +63,19 @@ func validRunDecision(decision RunDecision) bool { return false } switch decision.outcome { - case ParkOutcomeCompleted, ParkOutcomeDefault: + case ParkOutcomeCompleted: + if decision.directChannel { + return decision.task == TaskCancelNone && decision.caseID == 1 + } + return decision.task == TaskCancelNone && decision.caseID != 0 + case ParkOutcomeDefault: + if decision.directChannel { + return false + } return decision.task == TaskCancelNone && decision.caseID != 0 case ParkOutcomeCanceled: - return decision.caseID == 0 + return decision.caseID == 0 && + (!decision.directChannel || validTaskCancelKind(decision.task)) default: return false } @@ -105,10 +119,25 @@ func resumeGateTaken(g *G) bool { // G. A ready park is consumed here, not in a producer callback or PollReady. func prepareRunDecision(p *P, g *G) bool { if p == nil || !ValidG(g) || p.current != g || g.runP != p || g.state != GRunning || - p.runDecision != (RunDecision{}) || p.runDecisionTaken || !gPreemptEnabledAtDepthZero(g) || - !validParkState(&g.park) { + p.runDecision != (RunDecision{}) || p.runDecisionTaken || !gPreemptEnabledAtDepthZero(g) { return false } + return prepareIssuedRunDecision(p, g) +} + +// prepareIssuedRunDecision is the mutation half used only after +// checkedExecutorRun has consumed the private run.issued episode and proved +// the exact P/G/action/frame relation. llvm.coro.done is observational, so no +// independent writer exists between those adjacent gates. Compatibility +// Checked retains prepareRunDecision's complete arbitrary-caller validation. +func prepareIssuedRunDecision(p *P, g *G) bool { + // BeginRunG is the sole producer of this exact P/G/ActionCheckResume + // episode and already audited the runnable ParkState before publishing it. + // No producer can mutate a materialized frame packet or scheduler-owned + // ParkState between that action and Checked. ConsumeTaskParkSet retains its + // complete validator for source-backed Ready parks; releasable phases retain + // their complete O(1) validator below. Repeating the materialized audit here + // would not establish an independent boundary. decision := RunDecision{} if g.park.phase == parkReady { ticket := g.park.ticket @@ -126,11 +155,12 @@ func prepareRunDecision(p *P, g *G) bool { } } else if g.park.phase == parkMaterialized { decision = RunDecision{ - g: g, - ticket: g.park.ticket, - outcome: g.park.outcome, - caseID: g.park.winnerCase, - materialized: true, + g: g, + ticket: g.park.ticket, + outcome: g.park.outcome, + caseID: g.park.winnerCase, + materialized: true, + directChannel: g.park.directChannel, } } else if !releasableParkState(&g.park) { return false @@ -153,9 +183,12 @@ func prepareRunDecision(p *P, g *G) bool { decision.task = kind } } - if !validRunDecision(decision) { - return false - } + // Every non-zero shape above is produced by an exact consuming helper: + // ConsumeTaskParkSet, the materialized ParkState certificate, or + // ClaimTaskCancellation. None accepts caller-supplied decision fields, so a + // second generic union validation here would only replay those branches. + // The compiler prologue still correlates the private slot with G/ticket and + // consumes it exactly once. p.runDecision = decision return true } diff --git a/runtime/internal/coro/run_decision_abi.go b/runtime/internal/coro/run_decision_abi.go index 5241f7faa4..15d7ad1889 100644 --- a/runtime/internal/coro/run_decision_abi.go +++ b/runtime/internal/coro/run_decision_abi.go @@ -49,3 +49,21 @@ func TakeRunDecisionWords( } return uint32(parkOutcome), selectedCase, uint32(task), operation.SourceSlot, operation.Generation, true } + +// TakeRunDecisionWordsCompiler is the compiler-owned scalar gate. A nested +// static child's initial zero-ticket resume directly consumes the adjacent +// pendingInlineStart certificate; ordinary resumes and every non-zero ticket +// retain TakeRunDecisionWords' complete decision validation. +func TakeRunDecisionWordsCompiler( + g *G, + expectedEpoch, expectedGeneration uint32, +) ( + outcome, caseID, taskKind, operationSourceSlot, operationGeneration uint32, + ok bool, +) { + if expectedEpoch == 0 && expectedGeneration == 0 && + takeInlineAwaitInitialDecisionCompiler(g) { + return 0, 0, 0, 0, 0, true + } + return TakeRunDecisionWords(g, expectedEpoch, expectedGeneration) +} diff --git a/runtime/internal/coro/run_slice.go b/runtime/internal/coro/run_slice.go index c60fd264ce..47ff63bd15 100644 --- a/runtime/internal/coro/run_slice.go +++ b/runtime/internal/coro/run_slice.go @@ -22,6 +22,15 @@ package coro // for a hot CPU-only workload. const executorRunSourceQuantum uint8 = 64 +// executorRunSourceBatchQuantum is the bounded number of catalog, resolution, +// and acknowledgement reductions collapsed into one Source host step. The +// resumable A/ack/B transaction remains the single semantic owner; batching +// only avoids re-entering the target/runtime selector between adjacent direct +// core reductions. Typed runtime cleanup is still an explicit Materialize +// boundary, and readyDebt still prevents a completed source epoch from +// starting another epoch before a newly runnable continuation is paid. +const executorRunSourceBatchQuantum uint32 = 16 + // executorRunCursor is cold, scheduler-owner-only continuation state. It has // no callback-visible pointer. readyDebt forces one physical ready action after // a completed source epoch before a hot source can start another A/ack/B epoch. @@ -50,7 +59,8 @@ func validExecutorRunCursor(cursor *executorRunCursor, p *P) bool { func emptyExecutorRunCursor(driver *ExecutorDriver) bool { return driver != nil && driver.run == (executorRunCursor{}) && - emptyOwnerLocalCompletion(&driver.local) + emptyOwnerLocalCompletion(&driver.local) && + executorDirectChannelInboxIdle(driver) } // EnterExecutorRunCompatibility is the only supported stable-idle switch from @@ -63,6 +73,7 @@ func EnterExecutorRunCompatibility(driver *ExecutorDriver) bool { if !validExecutorDriver(driver) || driver.state != executorDriverActive || driver.run.issued != ActionInvalid || driver.poll.phase != executorPollIdle || !emptyOwnerLocalCompletion(&driver.local) || + !executorDirectChannelInboxIdle(driver) || !idleExecutorScheduler(driver.p) { return false } @@ -70,17 +81,40 @@ func EnterExecutorRunCompatibility(driver *ExecutorDriver) bool { return true } +// EnterExecutorRunStandbyCompatibility is the retained-wait variant of +// EnterExecutorRunCompatibility. false,true means a producer won the stable +// idle boundary by publishing a direct-channel completion, so the owner must +// re-enter the bounded runner instead of treating that ordinary race as +// corruption. Owner-local completion and every in-progress reducer field +// remain hard failures; only the lock-free producer inbox is asynchronous. +func EnterExecutorRunStandbyCompatibility(driver *ExecutorDriver) (entered, ok bool) { + if EnterExecutorRunCompatibility(driver) { + return true, true + } + if !validExecutorDriver(driver) || driver.state != executorDriverActive || + driver.run.issued != ActionInvalid || driver.poll.phase != executorPollIdle || + !emptyOwnerLocalCompletion(&driver.local) || + executorDirectChannelInboxIdle(driver) || + !idleExecutorScheduler(driver.p) { + return false, false + } + return false, true +} + // ExecutorRunStepKind is one reduction selected by the unified resumable core. -// Dispatch is separate from Action so budget one always has a stable return -// after dequeue/BeginRunG. Source advances exactly one PollExecutorSlice -// reduction. Action must be completed and committed without returning through -// another host boundary. +// Dispatch remains available when the caller cannot yet prove a managed +// execution lease or has only one budget unit. A lease-holding caller may ask +// the selector to combine that stable transition with the immediately +// following Action; Dispatched records the extra charged reduction. Source +// advances exactly one PollExecutorSlice reduction. Action must be completed +// and committed without returning through another host boundary. type ExecutorRunStepKind uint8 const ( ExecutorRunStepInvalid ExecutorRunStepKind = iota ExecutorRunStepSource ExecutorRunStepMaterialize + ExecutorRunStepDirectChannel ExecutorRunStepDispatch ExecutorRunStepAction ExecutorRunStepDestroyCommit @@ -90,15 +124,33 @@ const ( // ExecutorRunStep carries no callback or interface value. Action handles are // live only for Dispatch/Action. DestroyCommit is always handle-free. type ExecutorRunStep struct { - Kind ExecutorRunStepKind - G *G - Action Action - Poll ExecutorPollProgress - Cleanup ResumeCleanupStep + Kind ExecutorRunStepKind + // Dispatched is valid only for Action. It certifies that this selection also + // dequeued G and completed BeginRunG, so the reducer charges two units and + // applies the command-cancellation gate before the physical operation. + Dispatched bool + G *G + Action Action + Poll ExecutorPollProgress + Cleanup ResumeCleanupStep + Direct *DirectChannelCompletion +} + +// ExecutorRunActionStep is the compact high-frequency half of +// ExecutorRunStep. Poll progress and typed cleanup are cold event payloads; +// carrying their storage through every runnable dequeue and physical resume +// made the common scheduler ABI larger than the state it actually transfers. +// The selector below returns this shape only when the ordinary priority rules +// have already selected Dispatch/Action. Every source, materialize, direct +// completion, destroy receipt, and idle boundary retains ExecutorRunStep. +type ExecutorRunActionStep struct { + Dispatched bool + G *G + Action Action } func executorRunExternalSourceRequested(driver *ExecutorDriver) bool { - return driver.sources.pending(driver.p) || + return executorDirectChannelCompletionPending(driver) || driver.sources.pending(driver.p) || driver.registry.ObserveRequested(driver.handle) || preemptLoad(&driver.p.schedule) != scheduleIdle } @@ -111,11 +163,15 @@ func serviceExecutorRunSource(driver *ExecutorDriver, now int64, withDeadline bo var progress ExecutorPollProgress var ok bool if withDeadline { - _, progress, ok = pollBoundExecutorSliceAt(driver, now, true, 1) + _, progress, ok = pollBoundExecutorSliceAt( + driver, now, true, executorRunSourceBatchQuantum, + ) } else { - _, progress, ok = pollBoundExecutorSliceAt(driver, 0, false, 1) + _, progress, ok = pollBoundExecutorSliceAt( + driver, 0, false, executorRunSourceBatchQuantum, + ) } - if !ok || progress.Used != 1 { + if !ok || progress.Used == 0 || progress.Used > executorRunSourceBatchQuantum { return ExecutorRunStep{}, false } if progress.Complete { @@ -137,13 +193,38 @@ func serviceExecutorRunSource(driver *ExecutorDriver, now int64, withDeadline bo // host step so target-side ready distribution and readyDebt keep one common // boundary, but AtomicResolve identifies that no catalog scan or executor // acknowledgement was performed. +const ownerLocalResolveBatchQuantum uint32 = 16 + func serviceExecutorRunLocal(driver *ExecutorDriver) (ExecutorRunStep, bool) { - var resolved publishedEpochResolveStep - if !resolveOwnerLocalCompletionStep(driver, &resolved) || - resolved.applyVisits < 0 || resolved.promoted < 0 { + var applyVisits, promoted uint32 + complete := false + resolved := &driver.local.scratch + if *resolved != (publishedEpochResolveStep{}) { return ExecutorRunStep{}, false } - complete := resolved.complete + for reduction := uint32(0); reduction < ownerLocalResolveBatchQuantum; reduction++ { + // Typed runtime cleanup is the only boundary which cannot be reduced by + // the target-neutral core. Return it as the next explicit Materialize + // step; the following local service entry continues from the same cursor. + if _, pending := pendingResumeCleanupStepForCursor(&driver.local.resolve); pending { + break + } + if !resolveOwnerLocalCompletionStep(driver, resolved) || + resolved.applyVisits < 0 || resolved.promoted < 0 || + uint64(applyVisits)+uint64(resolved.applyVisits) > uint64(^uint32(0)) || + uint64(promoted)+uint64(resolved.promoted) > uint64(^uint32(0)) { + *resolved = publishedEpochResolveStep{} + return ExecutorRunStep{}, false + } + applyVisits += uint32(resolved.applyVisits) + promoted += uint32(resolved.promoted) + resolvedComplete := resolved.complete + *resolved = publishedEpochResolveStep{} + if resolvedComplete { + complete = true + break + } + } if complete { if ownerLocalCompletionPending(driver) { return ExecutorRunStep{}, false @@ -156,8 +237,8 @@ func serviceExecutorRunLocal(driver *ExecutorDriver) (ExecutorRunStep, bool) { } progress := ExecutorPollProgress{ Used: 1, - ApplyVisits: uint32(resolved.applyVisits), - Promoted: uint32(resolved.promoted), + ApplyVisits: applyVisits, + Promoted: promoted, Complete: complete, More: !complete || runnableForOSThreadOwner(driver.p) || executorRunExternalSourceRequested(driver), AtomicResolve: true, @@ -165,6 +246,20 @@ func serviceExecutorRunLocal(driver *ExecutorDriver) (ExecutorRunStep, bool) { return ExecutorRunStep{Kind: ExecutorRunStepSource, Poll: progress}, true } +// nextExecutorRunLocalStep is deliberately kept out of the common selector. +// Resolving and materializing an owner-local completion is a selected cold +// path; inlining its typed cursor machinery into every ordinary runnable or +// compute reduction expands the scheduler hot loop even when the local FIFO is +// empty. +// +//go:noinline +func nextExecutorRunLocalStep(driver *ExecutorDriver) (ExecutorRunStep, bool) { + if cleanup, pending := pendingResumeCleanupStepForCursor(&driver.local.resolve); pending { + return ExecutorRunStep{Kind: ExecutorRunStepMaterialize, Cleanup: cleanup}, true + } + return serviceExecutorRunLocal(driver) +} + // CommitExecutorRunSourceDistribution closes the optional target-side ready // distribution boundary after one complete Source reduction. Source completion // records readyDebt before returning so a hot source cannot starve a newly @@ -197,18 +292,53 @@ func CommitExecutorRunSourceDistribution(driver *ExecutorDriver, distributed boo return false } -func dispatchExecutorRunReady(driver *ExecutorDriver) (ExecutorRunStep, bool) { +func dispatchExecutorRunReadyAction( + driver *ExecutorDriver, + selected *G, + combineAction bool, +) (ExecutorRunActionStep, bool) { p := driver.p - if !validReadyQueueHeader(p) { - return ExecutorRunStep{}, false + if selected == nil || p == nil { + return ExecutorRunActionStep{}, false } - selected := nextOSThreadRunnable(p) - imported := selected != nil && selected.transferState == runnableTransferGImported - g := dequeueOSThreadRunnable(p) - if g == nil || g != selected { - return ExecutorRunStep{}, false + // nextExecutorRunActionValidated selected this task immediately before this + // call from the same owner-only queue. Preserve the rollback path below for + // a malformed G, but consume the selected queue capability instead of + // replaying the full head/tail/count header relation here. + var g *G + imported := selected.transferState == runnableTransferGImported + ordinaryOwner := p.osThreadLockOwner == nil && p.osThreadSuspend == osThreadSuspendAttached + if ordinaryOwner { + // In the overwhelmingly common attached/unlocked phase, selection is the + // FIFO head. The caller has just selected it, so consume the non-empty + // queue capability without making dequeue repeat the same header checks. + if selected != p.readyHead || p.readyCount == 0 { + return ExecutorRunActionStep{}, false + } + g = dequeueReadyHeadUnchecked(p) + if g != nil && g.transferState == runnableTransferGImported { + g.transferState = runnableTransferGIdle + } + } else { + g, imported = dequeueSelectedOSThreadRunnableWithTransfer(p, selected) + } + if g == nil { + return ExecutorRunActionStep{}, false + } + peerRunnable := p.readyHead != nil + if !ordinaryOwner { + peerRunnable = nextOSThreadRunnable(p) != nil + } + var action Action + var ok bool + if g.runAction == ActionInvalid && g.park.phase == parkMaterialized && g.park.directChannel { + action, ok = beginExecutorRunDirectChannelContinuation(p, g, peerRunnable) + } else if g.runAction == ActionCheckResume || + g.runAction == ActionInvalid && g.park.phase == parkMaterialized { + action, ok = beginExecutorRunContinuation(p, g, peerRunnable) + } else { + action, ok = BeginRunG(p, g) } - action, ok := BeginRunG(p, g) if !ok { // dequeue only clears the selected head's scheduler-owned queue fields. // Restore those exact fields on a fail-closed BeginRunG rejection so a @@ -218,17 +348,108 @@ func dispatchExecutorRunReady(driver *ExecutorDriver) (ExecutorRunStep, bool) { g.transferState = runnableTransferGImported } prependReadyUnchecked(p, g) - return ExecutorRunStep{}, false + return ExecutorRunActionStep{}, false } - return ExecutorRunStep{Kind: ExecutorRunStepDispatch, G: g, Action: action}, true + if combineAction { + driver.run.issued = action.Kind + return ExecutorRunActionStep{Dispatched: true, G: g, Action: action}, true + } + return ExecutorRunActionStep{G: g, Action: action}, true } -func nextExecutorRunStepAt(driver *ExecutorDriver, now int64, withDeadline bool) (ExecutorRunStep, bool) { - if !validExecutorDriverHeader(driver) || driver.state != executorDriverActive || - driver.sources.usesMonotonicTime() != withDeadline || withDeadline && now < 0 || - driver.run.issued != ActionInvalid { +func dispatchExecutorRunReady( + driver *ExecutorDriver, + selected *G, + combineAction bool, +) (ExecutorRunStep, bool) { + action, ok := dispatchExecutorRunReadyAction(driver, selected, combineAction) + if !ok { return ExecutorRunStep{}, false } + kind := ExecutorRunStepDispatch + if combineAction { + kind = ExecutorRunStepAction + } + return ExecutorRunStep{ + Kind: kind, Dispatched: action.Dispatched, G: action.G, Action: action.Action, + }, true +} + +// nextExecutorRunActionValidated applies the same priority order as +// nextExecutorRunStepAtValidated but stops before every cold event reduction. +// selected=false is a side-effect-free request to use the complete selector. +// This keeps one scheduling authority while allowing the production runner to +// avoid constructing the large cold-event union for the overwhelmingly common +// Action path. +func nextExecutorRunActionValidated( + driver *ExecutorDriver, + combineDispatch bool, +) (step ExecutorRunActionStep, selected, ok bool) { + p := driver.p + // A completed same-owner materialization publishes readyDebt only after its + // full wait/frame transaction and an idle-P commit. While this bounded-slice + // capability is retained, no producer can mutate P-local scheduler fields; + // it can only publish another source fact. Fairness requires paying the + // existing debt first, so the ordinary unlocked head may cross directly into + // the adjacent dequeue/action reducer without replaying the complete idle-P + // tuple or probing lower-priority producer queues. + if combineDispatch && driver.run.readyDebt && driver.poll.phase == executorPollIdle && + p.current == nil && p.osThreadLockOwner == nil && + p.osThreadSuspend == osThreadSuspendAttached && p.readyHead != nil { + step, ok = dispatchExecutorRunReadyAction(driver, p.readyHead, true) + return step, ok, ok + } + if p.current != nil { + action, g := p.action, p.current + if action.Kind == ActionCommitDestroy { + return ExecutorRunActionStep{}, false, true + } + if action.Handle == nil || + (action.Kind != ActionCheckResume && action.Kind != ActionCheckDestroy && action.Kind != ActionPanicDestroy) { + return ExecutorRunActionStep{}, false, false + } + driver.run.readyDebt = false + driver.run.issued = action.Kind + return ExecutorRunActionStep{G: g, Action: action}, true, true + } + if p.inResume || p.inlineAwaitDepth != 0 || p.action != (Action{}) || p.runDecision != (RunDecision{}) || + p.runDecisionTaken || p.servicePreemptBudget != 0 { + return ExecutorRunActionStep{}, false, false + } + if !combineDispatch || driver.poll.phase != executorPollIdle { + return ExecutorRunActionStep{}, false, true + } + runnable := nextOSThreadRunnable(p) + // A completed source or same-owner materialization has already selected one + // runnable as its bounded fairness payment. Dispatch that stable owner-local + // fact before probing producer queues again. Concurrent completion facts stay + // published and are observed immediately after this one physical action. + if driver.run.readyDebt && runnable != nil { + step, ok := dispatchExecutorRunReadyAction(driver, runnable, true) + return step, ok, ok + } + if executorDirectChannelCompletionPending(driver) || ownerLocalCompletionPending(driver) { + return ExecutorRunActionStep{}, false, true + } + if executorRunSourceRequested(driver) || + driver.run.actionsSinceSource == executorRunSourceQuantum || + runnable == nil && HasWaiting(p) && !driver.run.blocked { + return ExecutorRunActionStep{}, false, true + } + driver.run.readyDebt = false + if runnable == nil { + return ExecutorRunActionStep{}, false, true + } + step, ok = dispatchExecutorRunReadyAction(driver, runnable, true) + return step, ok, ok +} + +func nextExecutorRunStepAtValidated( + driver *ExecutorDriver, + now int64, + withDeadline bool, + combineDispatch bool, +) (ExecutorRunStep, bool) { p := driver.p if p.current != nil { action, g := p.action, p.current @@ -242,6 +463,11 @@ func nextExecutorRunStepAt(driver *ExecutorDriver, now int64, withDeadline bool) (action.Kind != ActionCheckResume && action.Kind != ActionCheckDestroy && action.Kind != ActionPanicDestroy) { return ExecutorRunStep{}, false } + // Dispatch has already selected this action as payment for any prior + // ready debt. Clear it before opening the no-return physical interval; + // a completion produced by the resumed coroutine may raise a new debt + // which CommitExecutorRunAction must preserve. + driver.run.readyDebt = false driver.run.issued = action.Kind return ExecutorRunStep{Kind: ExecutorRunStepAction, G: g, Action: action}, true } @@ -255,38 +481,209 @@ func nextExecutorRunStepAt(driver *ExecutorDriver, now int64, withDeadline bool) if cleanup, pending := pendingResumeCleanupStepForCursor(&driver.poll.resolve); pending { return ExecutorRunStep{Kind: ExecutorRunStepMaterialize, Cleanup: cleanup}, true } + if withDeadline && now < 0 { + return ExecutorRunStep{}, false + } return serviceExecutorRunSource(driver, now, withDeadline) } + // A compact hchan completion is already a terminal typed fact and has no + // source epoch to acknowledge. Consume it before unrelated catalog work; the + // runtime adapter removes the typed queue node and commits the returned + // frame-local packet in this single explicit reduction. + if executorDirectChannelCompletionPending(driver) { + completion, taken := takeExecutorDirectChannelCompletion(driver) + if !taken { + return ExecutorRunStep{}, false + } + if completion != nil { + return ExecutorRunStep{Kind: ExecutorRunStepDirectChannel, Direct: completion}, true + } + } // An owner-local completion has already published its exact typed source // fact. Resolve it before starting an unrelated external A/ack/B epoch and // before dispatching another G; typed cleanup still returns through the // ordinary direct-runtime Materialize boundary. if ownerLocalCompletionPending(driver) { - if cleanup, pending := pendingResumeCleanupStepForCursor(&driver.local.resolve); pending { - return ExecutorRunStep{Kind: ExecutorRunStepMaterialize, Cleanup: cleanup}, true - } - return serviceExecutorRunLocal(driver) + return nextExecutorRunLocalStep(driver) } + runnable := nextOSThreadRunnable(p) if driver.run.readyDebt { - if runnableForOSThreadOwner(p) { - return dispatchExecutorRunReady(driver) + if runnable != nil { + return dispatchExecutorRunReady(driver, runnable, combineDispatch) } - driver.run.readyDebt = false } if executorRunSourceRequested(driver) || driver.run.actionsSinceSource == executorRunSourceQuantum || - !runnableForOSThreadOwner(p) && HasWaiting(p) && !driver.run.blocked { + runnable == nil && HasWaiting(p) && !driver.run.blocked { + // A negative timestamp is the explicit before-time probe used by the + // native adapter. Selection has not mutated the cursor yet, so its + // caller can sample a fresh clock and retry this exact decision. + if withDeadline && now < 0 { + return ExecutorRunStep{}, false + } + driver.run.readyDebt = false return serviceExecutorRunSource(driver, now, withDeadline) } - if runnableForOSThreadOwner(p) { - return dispatchExecutorRunReady(driver) + driver.run.readyDebt = false + if runnable != nil { + return dispatchExecutorRunReady(driver, runnable, combineDispatch) } return ExecutorRunStep{Kind: ExecutorRunStepIdle}, true } -// NextExecutorRunStep selects one no-deadline runner reduction. It never calls -// PollExecutor, PollReady, or NextRunnable; all source work goes through the -// budget-one PollExecutorSlice cursor. +func nextExecutorRunStepAt(driver *ExecutorDriver, now int64, withDeadline bool) (ExecutorRunStep, bool) { + if !validExecutorDriverHeader(driver) || driver.state != executorDriverActive || + driver.sources.usesMonotonicTime() != withDeadline || + driver.run.issued != ActionInvalid { + return ExecutorRunStep{}, false + } + return nextExecutorRunStepAtValidated(driver, now, withDeadline, false) +} + +// ExecutorRunSliceCapability is a scheduler-owner-only proof that one bounded +// RunSlice entry audited the complete immutable driver/source binding. It is +// deliberately stack-scoped by the runtime adapter: a host return discards it, +// and the next entry must call BeginExecutorRunSlice again. Producers retain +// only source/registry POD identities and cannot manufacture or mutate this +// private P/driver pair. +type ExecutorRunSliceCapability struct { + driver *ExecutorDriver + withDeadline bool +} + +// BeginExecutorRunSlice performs the full cold-boundary audit once before a +// bounded owner loop. Exact source reducers, physical actions, and commits +// continue to validate every mutable transition; only the immutable catalog, +// route, and run-cursor header are not re-read before each adjacent step. +func BeginExecutorRunSlice(driver *ExecutorDriver) (ExecutorRunSliceCapability, bool) { + if !validExecutorDriverHeader(driver) || driver.state != executorDriverActive || + driver.run.issued != ActionInvalid { + return ExecutorRunSliceCapability{}, false + } + return ExecutorRunSliceCapability{ + driver: driver, + withDeadline: driver.sources.usesMonotonicTime(), + }, true +} + +func (capability *ExecutorRunSliceCapability) owner( + withDeadline bool, +) (*ExecutorDriver, bool) { + if capability == nil || capability.driver == nil || + capability.withDeadline != withDeadline { + return nil, false + } + // BeginExecutorRunSlice authenticated the immutable driver/P/source binding + // before this stack-scoped capability was returned. A bounded runner never + // crosses a host boundary while retaining it, and physical/source reducers + // cannot close or rebind the executor. Only the issued no-return interval is + // mutable between adjacent selections, so rechecking the complete binding on + // every hot action would duplicate the entry proof. + driver := capability.driver + return driver, driver.run.issued == ActionInvalid +} + +func (capability *ExecutorRunSliceCapability) nextAction( + combineDispatch bool, +) (ExecutorRunActionStep, bool, bool) { + // This stack-scoped capability already fixed its driver and source-time + // shape at BeginExecutorRunSlice. The compact action probe does not consume + // time, so replaying owner(withDeadline) here was a tautological comparison + // plus an otherwise unused cached P load on every physical resume. Only the + // adjacent issued interval can invalidate another selection. + if capability == nil || capability.driver == nil || + capability.driver.run.issued != ActionInvalid { + return ExecutorRunActionStep{}, false, false + } + return nextExecutorRunActionValidated(capability.driver, combineDispatch) +} + +// NextAction selects an already dispatched physical action without carrying +// the cold source/materialization payload union. selected=false means the +// complete Next/NextBeforeTime/NextAt selector still owns the next reduction. +func (capability *ExecutorRunSliceCapability) NextAction() ( + step ExecutorRunActionStep, + selected bool, + ok bool, +) { + return capability.nextAction(false) +} + +// NextActionCombined additionally combines a ready dequeue with its physical +// action. The caller must hold the same managed-execution lease and two-unit +// budget required by NextCombined/NextAtCombined. +func (capability *ExecutorRunSliceCapability) NextActionCombined() ( + step ExecutorRunActionStep, + selected bool, + ok bool, +) { + return capability.nextAction(true) +} + +// Next selects one step for a no-deadline bounded slice. +func (capability *ExecutorRunSliceCapability) Next() (ExecutorRunStep, bool) { + driver, ok := capability.owner(false) + if !ok { + return ExecutorRunStep{}, false + } + return nextExecutorRunStepAtValidated(driver, 0, false, false) +} + +// NextCombined is Next with the additional proof that the caller already owns +// a managed-execution lease and has at least two budget units. An immediately +// runnable G therefore crosses dequeue/BeginRunG and the issued Action boundary +// in one selector entry. +func (capability *ExecutorRunSliceCapability) NextCombined() (ExecutorRunStep, bool) { + driver, ok := capability.owner(false) + if !ok { + return ExecutorRunStep{}, false + } + return nextExecutorRunStepAtValidated(driver, 0, false, true) +} + +// NextBeforeTime performs the timer-aware before-clock probe without +// re-auditing immutable driver/source structure. +func (capability *ExecutorRunSliceCapability) NextBeforeTime() (ExecutorRunStep, bool) { + driver, ok := capability.owner(true) + if !ok { + return ExecutorRunStep{}, false + } + return nextExecutorRunStepAtValidated(driver, -1, true, false) +} + +// NextBeforeTimeCombined is NextBeforeTime with adjacent dispatch/action +// selection enabled under the caller's retained lease and budget proof. +func (capability *ExecutorRunSliceCapability) NextBeforeTimeCombined() (ExecutorRunStep, bool) { + driver, ok := capability.owner(true) + if !ok { + return ExecutorRunStep{}, false + } + return nextExecutorRunStepAtValidated(driver, -1, true, true) +} + +// NextAt selects one timer-aware step using the host's current monotonic +// sample. The logical A/B epoch still freezes time in its own transaction. +func (capability *ExecutorRunSliceCapability) NextAt(now int64) (ExecutorRunStep, bool) { + driver, ok := capability.owner(true) + if !ok || now < 0 { + return ExecutorRunStep{}, false + } + return nextExecutorRunStepAtValidated(driver, now, true, false) +} + +// NextAtCombined is NextAt with adjacent dispatch/action selection enabled +// under the caller's retained lease and budget proof. +func (capability *ExecutorRunSliceCapability) NextAtCombined(now int64) (ExecutorRunStep, bool) { + driver, ok := capability.owner(true) + if !ok || now < 0 { + return ExecutorRunStep{}, false + } + return nextExecutorRunStepAtValidated(driver, now, true, true) +} + +// NextExecutorRunStep selects one no-deadline runner step. It never calls +// PollExecutor, PollReady, or NextRunnable; source work goes through the +// bounded PollExecutorSlice cursor. func NextExecutorRunStep(driver *ExecutorDriver) (ExecutorRunStep, bool) { if driver == nil || driver.sources.usesMonotonicTime() { return ExecutorRunStep{}, false @@ -294,11 +691,25 @@ func NextExecutorRunStep(driver *ExecutorDriver) (ExecutorRunStep, bool) { return nextExecutorRunStepAt(driver, 0, false) } +// NextExecutorRunStepBeforeTime selects one timer-aware reduction only when +// that reduction does not consume a monotonic sample. A false result is +// deliberately ambiguous: the driver is invalid or the next reduction needs +// fresh time. In either case a native adapter may sample its clock and retry +// through NextExecutorRunStepAt; a time-required rejection does not mutate the +// runner cursor. This keeps ordinary action, channel-cleanup, and ready-queue +// traffic off the platform clock without weakening source/deadline ordering. +func NextExecutorRunStepBeforeTime(driver *ExecutorDriver) (ExecutorRunStep, bool) { + if driver == nil || !driver.sources.usesMonotonicTime() { + return ExecutorRunStep{}, false + } + return nextExecutorRunStepAt(driver, -1, true) +} + // NextExecutorRunStepAt is the deadline-capable counterpart. A fresh sample is // accepted at each source reduction; PollExecutorSliceAt freezes the correct // sample across each logical A or B epoch. func NextExecutorRunStepAt(driver *ExecutorDriver, now int64) (ExecutorRunStep, bool) { - if driver == nil || !driver.sources.usesMonotonicTime() { + if driver == nil || !driver.sources.usesMonotonicTime() || now < 0 { return ExecutorRunStep{}, false } return nextExecutorRunStepAt(driver, now, true) @@ -395,18 +806,137 @@ func completedExecutorRunAction(p *P, g *G, action Action) bool { } } +// validIssuedExecutorRunAction is the commit-side proof for the no-return +// physical interval opened by nextExecutorRunStepAt. The selector already +// audited the immutable registry, source-set, route, and run-cursor binding +// before setting run.issued. No target or coroutine callback can retain the +// private driver or cross a host boundary during that interval, so repeating +// those immutable audits after llvm.coro.resume/destroy only adds hot-path +// work. The mutable P/G/action episode is still checked in full by the exact +// commit reducer below, and the next selector repeats the complete header gate. +func validIssuedExecutorRunAction(driver *ExecutorDriver) bool { + // run.issued is written only after the bounded selector's complete driver, + // source-set, registry, and action audit. Nothing outside this package can + // manufacture or retain that private field, and no host boundary exists + // before the matching commit clears it. The exact P back-pointer and bound + // mode remain the independent owner gate; consume the capability for the + // remaining immutable driver fields instead of replaying them here. + if driver == nil || driver.p == nil || driver.p.executor != driver || + preemptLoad(&driver.p.executorMode) != executorModeBound { + return false + } + switch driver.run.issued { + case ActionCheckResume, ActionCheckDestroy, ActionPanicDestroy: + return true + default: + return false + } +} + +// ResumedExecutorRun consumes the exact ActionResume return for an issued +// executor step. Yield/Park are terminal control receipts constructed entirely +// by Resumed; no runtime or target policy chooses a queue placement for them. +// Commit those adjacent receipts here so the post-resume path does not replay +// completedExecutorRunAction's arbitrary-caller audit. Handle continuations, +// completion, panic, and destroy receipts retain the ordinary explicit commit +// APIs because their placement or terminal policy is selected by the runtime. +func ResumedExecutorRun( + driver *ExecutorDriver, + p *P, + g *G, + action Action, +) (next Action, committed, ok bool) { + // BeginIssuedExecutorResumeRuntimeContext has just consumed this private + // issued episode and opened p.action=ActionResume. No host boundary can + // unbind the driver before the adjacent physical resume returns; only the + // resumed G may publish scheduler state. Keep the exact driver/P identity + // and issued-kind checks, but do not replay the P.executor/executorMode + // binding already certified by the selector and Begin. Stable-boundary + // callers still use CommitExecutorRunAction and its complete owner gate. + if driver == nil || p == nil || driver.p != p || + driver.run.issued != ActionCheckResume { + return Action{}, false, false + } + if g != nil && g.pending.kind == pendingParkSet && g.pending.directChannel && g.active != nil && + g.pending.from == g.active && g.active.handle == action.Handle && + g.active.parkWait != nil && g.active.parkWait.directChannel { + // PrepareCurrentDirectChannelPark installed the pending direct marker + // only after building the complete frame/wait/park graph. The issued + // action and current/inResume relation freeze every owner field until + // this adjacent return from llvm.coro.resume; a peer may change only the + // completion's atomic word. Consume that certificate in one flat commit + // instead of calling the generic Resumed graph validator. An inline child + // may be the deepest active frame while action still names its physical + // outer resume; that ancestry is intentionally left to Resumed. + frame := g.pending.from + if p.current != g || g.runP != p || !p.inResume || p.inlineAwaitDepth != 0 || + p.action != action || !p.runDecisionTaken || frame == nil || frame != g.active || + frame.handle != action.Handle || frame.parkWait != g.active.parkWait || + frame.parkWait.resumeKind != resumeBindingDirectChannel || + !acknowledgeSuspendedGPreempt(g) { + return Action{}, false, false + } + wait := frame.parkWait + g.pending = pendingTransition{} + frame.state = FrameSuspended + p.inResume = false + p.runDecisionTaken = false + g.state = GWaiting + g.runP = nil + p.current = nil + p.servicePreemptBudget = 0 + p.action = Action{} + activateWaitSetRecordUnchecked(p, g, wait) + next, ok = Action{Kind: ActionPark}, true + } else { + next, ok = Resumed(p, g, action) + } + if !ok || (next.Kind != ActionYield && next.Kind != ActionPark) { + return next, false, ok + } + if !validOSThreadPeerActionCommit(p, g) { + return Action{}, false, false + } + // Resumed has just produced the complete stable Yield/Park state and cleared + // P.current/action/service state. Preserve a same-resume ready publication, + // close the issued interval, and apply the exceptional detached-owner debt. + publishedReady := driver.run.readyDebt + driver.run.issued = ActionInvalid + if !publishedReady { + driver.run.readyDebt = false + } else if p.osThreadLockOwner == nil && p.osThreadSuspend == osThreadSuspendAttached { + // The direct-channel handoff normally publishes exactly one peer while + // the physical owner is unattached. Consume the stable queue head here; + // entering nextOSThreadRunnable would only rediscover these two scalar + // affinity facts on every rendezvous. + driver.run.readyDebt = p.readyHead != nil + } else { + driver.run.readyDebt = runnableForOSThreadOwner(p) + } + driver.run.blocked = false + if driver.run.actionsSinceSource < executorRunSourceQuantum { + driver.run.actionsSinceSource++ + } + commitOSThreadPeerAction(p) + return next, true, true +} + // CommitExecutorRunAction closes the no-return physical interval opened by an // Action step. A live continuation is moved to the ready tail; terminal and // yield/park control actions are already stable. The function retains neither // the completed G nor its old handle, so a runtime may reclaim a dynamic G // immediately after a successful ActionComplete commit. func commitExecutorRunAction(driver *ExecutorDriver, g *G, next Action, placement executorRunQueuePlacement) bool { - if !validExecutorDriverHeader(driver) || driver.state != executorDriverActive || - driver.run.issued == ActionInvalid || g == nil || + if !validIssuedExecutorRunAction(driver) || g == nil || !validOSThreadPeerActionCommit(driver.p, g) { return false } p := driver.p + // Dispatch consumes the ready debt which selected this physical action, but + // a same-owner completion may publish a new runnable while llvm.coro.resume + // is executing. Preserve only that newly raised debt across the action + // receipt so its peer runs before an unrelated source epoch. + publishedReady := driver.run.readyDebt committed := false switch next.Kind { case ActionCheckResume, ActionCheckDestroy, ActionPanicDestroy: @@ -431,7 +961,13 @@ func commitExecutorRunAction(driver *ExecutorDriver, g *G, next Action, placemen return false } driver.run.issued = ActionInvalid - driver.run.readyDebt = false + if !publishedReady { + driver.run.readyDebt = false + } else if p.osThreadLockOwner == nil && p.osThreadSuspend == osThreadSuspendAttached { + driver.run.readyDebt = p.readyHead != nil + } else { + driver.run.readyDebt = runnableForOSThreadOwner(p) + } driver.run.blocked = false if driver.run.actionsSinceSource < executorRunSourceQuantum { driver.run.actionsSinceSource++ @@ -499,8 +1035,12 @@ func CommitExecutorRunDomainDestroy(driver *ExecutorDriver, g *G, receipt Action // child destroy plus one root resume per step. Nested non-root cleanup remains // ordinary FIFO work; normal-main return's final root destroy is separate. func CommitExecutorRunCommandBootstrapDirectChildHandoff(driver *ExecutorDriver, g *G, next Action) bool { - if !validExecutorDriver(driver) || driver.state != executorDriverActive || g == nil || - g.root == nil || g.active != g.root || g.panicUnwind || !emptyPanicRecord(&g.panicRecord) { + // This helper is probed for every live command-main action. Reject the + // overwhelmingly common non-bootstrap shape before auditing the complete + // source and owner-local cursors; an exact candidate still receives the + // same full driver validation before any scheduler state is mutated. + if driver == nil || g == nil || g.root == nil || g.active != g.root || + g.panicUnwind || !emptyPanicRecord(&g.panicRecord) { return false } switch next.Kind { @@ -520,6 +1060,9 @@ func CommitExecutorRunCommandBootstrapDirectChildHandoff(driver *ExecutorDriver, default: return false } + if !validExecutorDriver(driver) || driver.state != executorDriverActive { + return false + } return commitExecutorRunAction(driver, g, next, executorRunQueueCommandBootstrapDirectChildHandoff) } diff --git a/runtime/internal/coro/run_slice_test.go b/runtime/internal/coro/run_slice_test.go index ed2e86bad6..ba3f41fde0 100644 --- a/runtime/internal/coro/run_slice_test.go +++ b/runtime/internal/coro/run_slice_test.go @@ -35,7 +35,7 @@ func runnerYieldAction(t *testing.T, driver *ExecutorDriver, step ExecutorRunSte if step.Kind != ExecutorRunStepAction || step.G != task.g || step.Action.Kind != ActionCheckResume { t.Fatalf("runner yield action = %+v", step) } - resume, ok := Checked(driver.p, task.g, step.Action, false) + resume, _, ok := BeginIssuedExecutorResumeRuntimeContext(driver, task.g) if !ok || resume.Kind != ActionResume || resume.Handle != task.handle { t.Fatalf("runner yield check = (%+v, %t)", resume, ok) } @@ -45,8 +45,8 @@ func runnerYieldAction(t *testing.T, driver *ExecutorDriver, step ExecutorRunSte if !PrepareYield(task.g, task.handle, task.frame.header) { t.Fatal("prepare runner yield") } - next, ok := Resumed(driver.p, task.g, resume) - if !ok || next.Kind != ActionYield || !CommitExecutorRunAction(driver, task.g, next) { + next, committed, ok := ResumedExecutorRun(driver, driver.p, task.g, resume) + if !ok || !committed || next.Kind != ActionYield { t.Fatalf("commit runner yield = (%+v, %t)", next, ok) } } @@ -64,6 +64,41 @@ func runnerNextPhysicalAction(t *testing.T, driver *ExecutorDriver, task *yieldi return step } +func TestExecutorRunResumeRuntimeContextDescriptorCapability(t *testing.T) { + for _, test := range []struct { + name string + flags uint32 + anonymous bool + wantNeeds bool + wantOK bool + }{ + {name: "ordinary frame", wantNeeds: true, wantOK: true}, + {name: "anonymous legacy frame", anonymous: true, wantNeeds: true, wantOK: true}, + {name: "context independent", flags: FrameDescriptorNoRuntimeContextV1, wantOK: true}, + {name: "hidden context independent", flags: FrameDescriptorTraceHiddenV1 | FrameDescriptorNoRuntimeContextV1, wantOK: true}, + {name: "unknown capability", flags: 1 << 31}, + } { + t.Run(test.name, func(t *testing.T) { + p := new(P) + driver, _, _ := bindTestExecutorDriver(t, p) + task := newYieldingTestG(t, test.name) + descriptor := (*FrameDescriptorV1)(task.frame.descriptor) + descriptor.Flags = test.flags + if test.anonymous { + descriptor.Function = "" + } + if !Enqueue(p, task.g) { + t.Fatal("enqueue descriptor-capability task") + } + step := runnerNextPhysicalAction(t, driver, task, ActionCheckResume) + _, needs, modeOK := CheckedExecutorRunRuntimeContext(driver, task.g, step.Action, false) + if needs != test.wantNeeds || modeOK != test.wantOK { + t.Fatalf("runtime context mode = (%t, %t), want (%t, %t)", needs, modeOK, test.wantNeeds, test.wantOK) + } + }) + } +} + func queueRunnerCheckDestroy(t *testing.T, driver *ExecutorDriver, task *yieldingTestG) *Frame { t.Helper() step := runnerNextPhysicalAction(t, driver, task, ActionCheckResume) @@ -215,6 +250,211 @@ func TestExecutorRunManagedResumePendingIsObservational(t *testing.T) { runtime.KeepAlive(task.frame.memory) } +func TestExecutorRunSliceCapabilityRetainsExactOwnerAcrossBoundedSteps(t *testing.T) { + p := new(P) + driver, _, _ := bindTestExecutorDriver(t, p) + task := newYieldingTestG(t, "run-slice-capability") + if !Enqueue(p, task.g) { + t.Fatal("enqueue run-slice capability task") + } + run, ok := BeginExecutorRunSlice(driver) + if !ok { + t.Fatal("begin executor run-slice capability") + } + dispatch, ok := run.Next() + if !ok || dispatch.Kind != ExecutorRunStepDispatch || dispatch.G != task.g || + dispatch.Action.Kind != ActionCheckResume || driver.run.issued != ActionInvalid { + t.Fatalf("capability dispatch = (%+v, %t), cursor=%+v", dispatch, ok, driver.run) + } + step, ok := run.Next() + if !ok || step.Kind != ExecutorRunStepAction || step.G != task.g || + step.Action != dispatch.Action || driver.run.issued != ActionCheckResume { + t.Fatalf("capability action = (%+v, %t), cursor=%+v", step, ok, driver.run) + } + runnerYieldAction(t, driver, step, task) + if next, selected := run.Next(); !selected || next.Kind != ExecutorRunStepDispatch || + next.G != task.g || next.Action.Kind != ActionCheckResume { + t.Fatalf("capability reuse after stable reduction = (%+v, %t)", next, selected) + } + runtime.KeepAlive(task.frame.memory) +} + +func TestExecutorRunSliceCompactActionPreservesSelectionAndBudgetBoundaries(t *testing.T) { + if compact, full := unsafe.Sizeof(ExecutorRunActionStep{}), unsafe.Sizeof(ExecutorRunStep{}); compact >= full { + t.Fatalf("compact action ABI size = %d, full step = %d", compact, full) + } + p := new(P) + driver, _, _ := bindTestExecutorDriver(t, p) + task := newYieldingTestG(t, "run-slice-compact-action") + if !Enqueue(p, task.g) { + t.Fatal("enqueue compact-action task") + } + run, ok := BeginExecutorRunSlice(driver) + if !ok { + t.Fatal("begin compact-action run slice") + } + + // A one-unit caller may not combine dequeue with the physical action. The + // compact probe must be observational and leave the complete selector's + // ordinary Dispatch boundary intact. + if step, selected, valid := run.NextAction(); !valid || selected || step != (ExecutorRunActionStep{}) { + t.Fatalf("one-unit compact probe = (%+v, %t, %t)", step, selected, valid) + } + if p.readyHead != task.g || p.readyTail != task.g || p.readyCount != 1 || p.current != nil || + driver.run.issued != ActionInvalid { + t.Fatalf("one-unit compact probe mutated scheduler: head=%p tail=%p count=%d current=%p cursor=%+v", + p.readyHead, p.readyTail, p.readyCount, p.current, driver.run) + } + dispatch, dispatched := run.Next() + if !dispatched || dispatch.Kind != ExecutorRunStepDispatch || dispatch.G != task.g || + dispatch.Action.Kind != ActionCheckResume || driver.run.issued != ActionInvalid { + t.Fatalf("compact fallback dispatch = (%+v, %t), cursor=%+v", dispatch, dispatched, driver.run) + } + action, selected, valid := run.NextAction() + if !valid || !selected || action.Dispatched || action.G != task.g || + action.Action != dispatch.Action || driver.run.issued != ActionCheckResume { + t.Fatalf("compact issued action = (%+v, %t, %t), cursor=%+v", action, selected, valid, driver.run) + } + runnerYieldAction(t, driver, ExecutorRunStep{ + Kind: ExecutorRunStepAction, G: action.G, Action: action.Action, + }, task) + + // A two-unit caller owns the managed-execution lease and may collapse the + // same dequeue/action pair without constructing the cold event union. + action, selected, valid = run.NextActionCombined() + if !valid || !selected || !action.Dispatched || action.G != task.g || + action.Action.Kind != ActionCheckResume || driver.run.issued != ActionCheckResume { + t.Fatalf("combined compact action = (%+v, %t, %t), cursor=%+v", action, selected, valid, driver.run) + } + runnerYieldAction(t, driver, ExecutorRunStep{ + Kind: ExecutorRunStepAction, Dispatched: true, G: action.G, Action: action.Action, + }, task) + runtime.KeepAlive(task.frame.memory) +} + +func TestCurrentExecutorDriverForActiveResumeUsesIssuedCapability(t *testing.T) { + p := new(P) + driver, _, _ := bindTestExecutorDriver(t, p) + task := newYieldingTestG(t, "active-resume-capability") + if !Enqueue(p, task.g) { + t.Fatal("enqueue active-resume-capability task") + } + step := runnerNextPhysicalAction(t, driver, task, ActionCheckResume) + resume, ok := Checked(p, task.g, step.Action, false) + if !ok || resume.Kind != ActionResume { + t.Fatal("enter active-resume capability") + } + takeNormalRunnerDecision(t, task.g) + if current, route, valid := CurrentExecutorDriverForActiveResume(task.g); !valid || current != driver || route != driver.route { + t.Fatalf("active-resume driver = (%p, %d, %t)", current, route, valid) + } + savedExecutor := p.executor + p.executor = nil + if current, route, valid := CurrentExecutorDriverForActiveResume(task.g); valid || current != nil || route != 0 { + t.Fatalf("ownerless active-resume driver = (%p, %d, %t)", current, route, valid) + } + p.executor = savedExecutor + savedIssued := driver.run.issued + driver.run.issued = ActionInvalid + if current, route, valid := CurrentExecutorDriverForActiveResume(task.g); valid || current != nil || route != 0 { + t.Fatalf("unissued active-resume driver = (%p, %d, %t)", current, route, valid) + } + driver.run.issued = savedIssued + task.frame.header.SuspendReason = uint16(SuspendYield) + task.frame.header.Lifecycle = uint16(FrameSuspended) + if !PrepareYield(task.g, task.handle, task.frame.header) { + t.Fatal("prepare active-resume capability yield") + } + next, resumed := Resumed(p, task.g, resume) + if !resumed || next.Kind != ActionYield || !CommitExecutorRunAction(driver, task.g, next) { + t.Fatalf("finish active-resume capability = (%+v, %t)", next, resumed) + } + runtime.KeepAlive(task.frame.memory) +} + +func TestCurrentExecutorDriverForCompilerTaskUsesHiddenTaskCapability(t *testing.T) { + p := new(P) + driver, _, _ := bindTestExecutorDriver(t, p) + task := newYieldingTestG(t, "compiler-task-capability") + if !Enqueue(p, task.g) { + t.Fatal("enqueue compiler-task-capability task") + } + step := runnerNextPhysicalAction(t, driver, task, ActionCheckResume) + resume, ok := Checked(p, task.g, step.Action, false) + if !ok || resume.Kind != ActionResume { + t.Fatal("enter compiler-task capability") + } + takeNormalRunnerDecision(t, task.g) + if current, route, valid := CurrentExecutorDriverForCompilerTask(task.g); !valid || current != driver || route != driver.route { + t.Fatalf("compiler-task driver = (%p, %d, %t)", current, route, valid) + } + // The hidden task is the capability; this boundary does not require the + // host runner's optional issued-action marker. + savedIssued := driver.run.issued + driver.run.issued = ActionInvalid + if current, route, valid := CurrentExecutorDriverForCompilerTask(task.g); !valid || current != driver || route != driver.route { + t.Fatalf("unissued compiler-task driver = (%p, %d, %t)", current, route, valid) + } + driver.run.issued = savedIssued + savedCurrent := p.current + p.current = nil + if current, route, valid := CurrentExecutorDriverForCompilerTask(task.g); valid || current != nil || route != 0 { + t.Fatalf("detached compiler-task driver = (%p, %d, %t)", current, route, valid) + } + p.current = savedCurrent + task.frame.header.SuspendReason = uint16(SuspendYield) + task.frame.header.Lifecycle = uint16(FrameSuspended) + if !PrepareYield(task.g, task.handle, task.frame.header) { + t.Fatal("prepare compiler-task capability yield") + } + next, resumed := Resumed(p, task.g, resume) + if !resumed || next.Kind != ActionYield || !CommitExecutorRunAction(driver, task.g, next) { + t.Fatalf("finish compiler-task capability = (%+v, %t)", next, resumed) + } + runtime.KeepAlive(task.frame.memory) +} + +func TestExecutorRunIssuedCommitRetainsExactOwnerGate(t *testing.T) { + p := new(P) + driver, _, _ := bindTestExecutorDriver(t, p) + task := newYieldingTestG(t, "issued-owner-gate") + if !Enqueue(p, task.g) { + t.Fatal("enqueue issued-owner-gate task") + } + step := runnerNextPhysicalAction(t, driver, task, ActionCheckResume) + resume, ok := Checked(p, task.g, step.Action, false) + if !ok || resume.Kind != ActionResume { + t.Fatal("check issued-owner-gate resume") + } + takeNormalRunnerDecision(t, task.g) + task.frame.header.SuspendReason = uint16(SuspendYield) + task.frame.header.Lifecycle = uint16(FrameSuspended) + if !PrepareYield(task.g, task.handle, task.frame.header) { + t.Fatal("prepare issued-owner-gate yield") + } + next, ok := Resumed(p, task.g, resume) + if !ok || next.Kind != ActionYield { + t.Fatalf("resume issued-owner-gate task = (%+v, %t)", next, ok) + } + + // run.issued alone is not authority: the exact P back-pointer and bound + // executor mode remain mandatory at the commit boundary. + p.executor = nil + if CommitExecutorRunAction(driver, task.g, next) { + t.Fatal("issued commit accepted a missing P owner") + } + p.executor = driver + preemptStore(&p.executorMode, executorModeUnbound) + if CommitExecutorRunAction(driver, task.g, next) { + t.Fatal("issued commit accepted an unbound P") + } + preemptStore(&p.executorMode, executorModeBound) + if !CommitExecutorRunAction(driver, task.g, next) { + t.Fatal("issued commit rejected restored exact owner") + } + runtime.KeepAlive(task.frame.memory) +} + func TestExecutorRunCommandBootstrapDirectChildHandoffPrecedesTwoPeers(t *testing.T) { p := new(P) driver, _, _ := bindTestExecutorDriver(t, p) @@ -511,13 +751,15 @@ func TestExecutorRunStartedEpochPrecedesReadyAndHotSourceAlternates(t *testing.T t.Fatal("prepare hot source runner") } - sourceSteps := uint32(0) + sourceSteps, sourceUsed := uint32(0), uint32(0) for { step, ok := NextExecutorRunStep(driver) - if !ok || step.Kind != ExecutorRunStepSource || step.Poll.Used != 1 { + if !ok || step.Kind != ExecutorRunStepSource || step.Poll.Used == 0 || + step.Poll.Used > executorRunSourceBatchQuantum { t.Fatalf("started epoch step %d = (%+v, %t)", sourceSteps, step, ok) } sourceSteps++ + sourceUsed += step.Poll.Used if step.Poll.Complete { break } @@ -525,8 +767,9 @@ func TestExecutorRunStartedEpochPrecedesReadyAndHotSourceAlternates(t *testing.T t.Fatal("ready G interrupted a started A/ack/B epoch") } } - if want, ok := MinExecutorPollBudget(driver); !ok || sourceSteps != want { - t.Fatalf("budget-one source transaction used %d, want (%d, %t)", sourceSteps, want, ok) + if want, ok := MinExecutorPollBudget(driver); !ok || sourceUsed != want { + t.Fatalf("batched source transaction used %d reductions in %d steps, want (%d, %t)", + sourceUsed, sourceSteps, want, ok) } // Publish the next hot epoch before paying the ready debt. Dispatch and one // complete physical G action must still precede that epoch. @@ -543,12 +786,92 @@ func TestExecutorRunStartedEpochPrecedesReadyAndHotSourceAlternates(t *testing.T } runnerYieldAction(t, driver, step, task) step, ok = NextExecutorRunStep(driver) - if !ok || step.Kind != ExecutorRunStepSource || step.Poll.Used != 1 { + if !ok || step.Kind != ExecutorRunStepSource || step.Poll.Used == 0 || + step.Poll.Used > executorRunSourceBatchQuantum { t.Fatalf("hot source did not alternate after one G action = (%+v, %t)", step, ok) } runtime.KeepAlive(task.frame.memory) } +func TestExecutorRunActionPreservesReadyDebtPublishedDuringResume(t *testing.T) { + p := new(P) + driver, registry, handle := bindTestExecutorDriver(t, p) + current := newYieldingTestG(t, "ready-debt-current") + peer := newYieldingTestG(t, "ready-debt-peer") + if !Enqueue(p, current.g) || registry.Request(handle) != ExecutorRequestPublished { + t.Fatal("prepare ready-debt action") + } + // This debt selected current and is paid when its physical Action starts. + // The pending source remains observable behind that action. + driver.run.readyDebt = true + step := runnerNextPhysicalAction(t, driver, current, ActionCheckResume) + if driver.run.readyDebt || driver.run.issued != ActionCheckResume || + !registry.ObserveRequested(handle) { + t.Fatalf("issued action retained old debt: run=%+v requested=%t", + driver.run, registry.ObserveRequested(handle)) + } + resume, ok := Checked(p, current.g, step.Action, false) + if !ok || resume.Kind != ActionResume { + t.Fatal("check ready-debt current") + } + takeNormalRunnerDecision(t, current.g) + + // Model a same-owner completion produced inside llvm.coro.resume: it makes + // a peer runnable and publishes a new debt before the current Action receipt. + if !Enqueue(p, peer.g) { + t.Fatal("publish peer during physical resume") + } + driver.run.readyDebt = true + current.frame.header.SuspendReason = uint16(SuspendYield) + current.frame.header.Lifecycle = uint16(FrameSuspended) + if !PrepareYield(current.g, current.handle, current.frame.header) { + t.Fatal("prepare current yield") + } + next, resumed := Resumed(p, current.g, resume) + if !resumed || next.Kind != ActionYield || + !CommitExecutorRunAction(driver, current.g, next) { + t.Fatalf("commit action with newly published ready debt = (%+v, %t)", next, resumed) + } + if !driver.run.readyDebt || driver.run.issued != ActionInvalid || + p.readyHead != peer.g || p.readyTail != current.g || + !registry.ObserveRequested(handle) { + t.Fatalf("action receipt lost new debt: run=%+v head=%p tail=%p requested=%t", + driver.run, p.readyHead, p.readyTail, registry.ObserveRequested(handle)) + } + step, ok = NextExecutorRunStep(driver) + if !ok || step.Kind != ExecutorRunStepDispatch || step.G != peer.g { + t.Fatalf("new peer did not precede pending source = (%+v, %t)", step, ok) + } + runtime.KeepAlive(current.frame.memory) + runtime.KeepAlive(peer.frame.memory) +} + +func TestExecutorRunReadyDebtPrecedesNewDirectCompletion(t *testing.T) { + p := new(P) + driver, _, _ := bindTestExecutorDriver(t, p) + task := newYieldingTestG(t, "ready-debt-before-direct") + if !Enqueue(p, task.g) { + t.Fatal("enqueue ready-debt task") + } + driver.run.readyDebt = true + completion := &DirectChannelCompletion{ + owner: driver, + route: driver.route, + } + preemptStore(&completion.state, uint32(directChannelCompletionMatched)) + if !PublishExecutorDirectChannelCompletion(driver, completion) { + t.Fatal("publish direct completion behind ready debt") + } + + step, selected, ok := nextExecutorRunActionValidated(driver, true) + if !ok || !selected || !step.Dispatched || step.G != task.g || + !executorDirectChannelCompletionPending(driver) { + t.Fatalf("compact ready debt/direct completion ordering = (%+v, %t, %t), pending=%t", + step, selected, ok, executorDirectChannelCompletionPending(driver)) + } + runtime.KeepAlive(task.frame.memory) +} + func TestExecutorRunCursorRejectsImplicitLegacySwitch(t *testing.T) { p := new(P) driver, registry, handle := bindTestExecutorDriver(t, p) @@ -586,6 +909,32 @@ func TestExecutorRunCursorRejectsImplicitLegacySwitch(t *testing.T) { runtime.KeepAlive(task.frame.memory) } +func TestExecutorRunStandbyCompatibilityDefersDirectIngress(t *testing.T) { + p := new(P) + driver, _, _ := bindTestExecutorDriver(t, p) + completion := &DirectChannelCompletion{ + owner: driver, + route: driver.route, + state: uint32(directChannelCompletionMatched), + } + if !PublishExecutorDirectChannelCompletion(driver, completion) { + t.Fatal("publish direct completion before standby compatibility") + } + if entered, ok := EnterExecutorRunStandbyCompatibility(driver); !ok || entered { + t.Fatalf("standby compatibility over direct ingress = (%t, %t), want (false, true)", entered, ok) + } + if got, ok := takeExecutorDirectChannelCompletion(driver); !ok || got != completion { + t.Fatalf("take deferred direct completion = (%p, %t), want (%p, true)", got, ok, completion) + } + if !executorDirectChannelInboxIdle(driver) { + t.Fatal("deferred direct completion did not restore idle inbox") + } + if entered, ok := EnterExecutorRunStandbyCompatibility(driver); !ok || !entered { + t.Fatalf("stable standby compatibility = (%t, %t), want (true, true)", entered, ok) + } + closeTestExecutorDriver(t, driver) +} + func TestExecutorRunCursorRejectsExportedPollSlice(t *testing.T) { for _, timed := range []bool{false, true} { name := "plain" @@ -607,7 +956,6 @@ func TestExecutorRunCursorRejectsExportedPollSlice(t *testing.T) { t.Fatal("prepare mixed hot-source/ready-debt cursor") } - requestedBehindA := false now := int64(1) for { var step ExecutorRunStep @@ -621,20 +969,21 @@ func TestExecutorRunCursorRejectsExportedPollSlice(t *testing.T) { if !ok || step.Kind != ExecutorRunStepSource { t.Fatalf("mixed cursor source = (%+v, %t)", step, ok) } - if !requestedBehindA && driver.poll.phase >= executorPollEpochBPublish { - if registry.Request(handle) != ExecutorRequestPublished { - t.Fatal("publish hot source behind acknowledged epoch A") - } - requestedBehindA = true - } if step.Poll.Complete { break } } - if !requestedBehindA || !driver.run.sourceMore || !driver.run.readyDebt || + // A small A/ack/B transaction may complete in one bounded Source + // step. Publish the next hot epoch after completion but before + // ready-debt dispatch; the exported compatibility poll must still + // reject this mixed bounded-runner state. + if registry.Request(handle) != ExecutorRequestPublished { + t.Fatal("publish hot source behind completed batched epoch") + } + if !driver.run.readyDebt || driver.poll != (executorPollTransaction{}) || p.readyHead != task.g { - t.Fatalf("mixed cursor precondition = requested:%t run:%+v poll:%+v head:%p", - requestedBehindA, driver.run, driver.poll, p.readyHead) + t.Fatalf("mixed cursor precondition = run:%+v poll:%+v head:%p", + driver.run, driver.poll, p.readyHead) } beforeRun, beforePoll := driver.run, driver.poll var progress ExecutorPollProgress @@ -680,6 +1029,39 @@ func TestRequestExecutorSourceServiceResumesBlockedOwnerTransaction(t *testing.T } } +func TestNextExecutorRunStepBeforeTimeDefersOnlyTimedSource(t *testing.T) { + p := new(P) + driver, _, _, _ := bindTestExecutorDriverWithTimers(t, p) + task := newYieldingTestG(t, "before-time-ready") + if !Enqueue(p, task.g) { + t.Fatal("enqueue before-time runnable") + } + step, ok := NextExecutorRunStepBeforeTime(driver) + if !ok || step.Kind != ExecutorRunStepDispatch || step.G != task.g { + t.Fatalf("before-time dispatch = (%+v, %t)", step, ok) + } + + // A separate stable driver isolates the time-required source decision. The + // before-time probe must reject it without cursor mutation. + sourceP := new(P) + sourceDriver, registry, _, handle := bindTestExecutorDriverWithTimers(t, sourceP) + if result := registry.Request(handle); result != ExecutorRequestPublished { + t.Fatalf("publish timed source request = %d", result) + } + beforeRun, beforePoll := sourceDriver.run, sourceDriver.poll + if step, ok := NextExecutorRunStepBeforeTime(sourceDriver); ok || step != (ExecutorRunStep{}) { + t.Fatalf("before-time source probe = (%+v, %t)", step, ok) + } + if sourceDriver.run != beforeRun || sourceDriver.poll != beforePoll { + t.Fatalf("before-time source probe mutated state: run=%+v poll=%+v", sourceDriver.run, sourceDriver.poll) + } + step, ok = NextExecutorRunStepAt(sourceDriver, 1) + if !ok || step.Kind != ExecutorRunStepSource { + t.Fatalf("timed retry source = (%+v, %t)", step, ok) + } + runtime.KeepAlive(task.frame.memory) +} + func TestRequestExecutorSourceServiceRejectsUnstableOwner(t *testing.T) { p := new(P) driver, _, _ := bindTestExecutorDriver(t, p) diff --git a/runtime/internal/coro/scheduler.go b/runtime/internal/coro/scheduler.go index 8b44d6ae09..80d897d3ff 100644 --- a/runtime/internal/coro/scheduler.go +++ b/runtime/internal/coro/scheduler.go @@ -217,11 +217,13 @@ const ( // preemptCheckpointStride bounds how many compiler safepoints may pass // before a running G checks atomic G/P/executor requests. Keeping this // deterministic preserves targets without a clock or signal source. - preemptCheckpointStride uint32 = 64 - // servicePreemptSafepointBudget bounds a sole runnable G's complete run - // slice. A known competitor receives one checkpoint stride; otherwise the - // larger service quantum amortizes scheduler/event-source scans. - servicePreemptSafepointBudget uint32 = preemptCheckpointStride * 64 + preemptCheckpointStride uint32 = 2048 + // servicePreemptSafepointBudget bounds the fallback scheduler audit cadence. + // A known competitor receives one checkpoint stride. A bound executor does + // not yield a sole runnable G merely to rediscover empty sources: every + // durable producer must publish its request gate. Unbound compatibility + // mode retains the periodic scheduler handoff as its polling fallback. + servicePreemptSafepointBudget uint32 = 4096 ) // ActionKind identifies either the next compiler-owned handle operation or a @@ -350,7 +352,7 @@ func InitG(g *G) bool { g.runAction != ActionInvalid || g.transferState != runnableTransferGIdle || g.osThreadLockDepth != 0 || g.runnableAffinity != runnableAnyOwner || g.frames != nil || g.active != nil || g.root != nil || - g.pending.kind != pendingNone || g.pending.from != nil || g.pending.target != nil || + g.pending.kind != pendingNone || g.pending.directChannel || g.pending.from != nil || g.pending.target != nil || g.destroyTarget != nil || g.destroyRoot || g.nextReady != nil || g.queued || g.waiting || g.runP != nil || g.park != (ParkState{}) || @@ -433,6 +435,94 @@ func PollPreempt(g *G) bool { return valid && requested } +// PollPreemptCompiler is the trusted generated-code safepoint. The physical +// resume prologue and every scheduler transition already establish or revoke +// the complete frame/source invariants, so this hot hook rechecks only the +// structural resume capability, the immutable P/executor binding, and the +// three request/budget words it actually consumes. +// +// Runtime adapters must call this function only from +// __llgo_coro_preempt_poll_v1. Defensive or manually driven scheduler code +// uses PollPreempt, which retains the complete active-frame validation. +func PollPreemptCompiler(g *G) bool { + if g == nil || g.magic != gMagic { + return false + } + p := g.runP + if p == nil || p.current != g || !p.inResume || g.state != GRunning || + p.action.Kind != ActionResume || p.action.Flags != 0 || p.action.Handle == nil || + !p.runDecisionTaken { + return false + } + word := loadGPreempt(g) + if word != preemptIdle && word != preemptRequested { + return false + } + budget := p.servicePreemptBudget + if budget == 0 || budget > servicePreemptSafepointBudget { + return false + } + + mode := preemptLoad(&p.executorMode) + var driver *ExecutorDriver + var schedule uint32 + switch mode { + case executorModeBound: + driver = p.executor + if driver == nil || driver.magic != executorDriverMagic || + driver.state != executorDriverActive || driver.p != p || + driver.requestGate == nil { + return false + } + gate := preemptLoad(driver.requestGate) + if gate&^executorGateMask != 0 || gate&executorGateClosed != 0 { + return false + } + case executorModeUnbound: + schedule = preemptLoad(&p.schedule) + if schedule != scheduleIdle && schedule != scheduleRequested { + return false + } + default: + return false + } + + requested := compareAndSwapGPreemptStateAtDepthZero(g, preemptRequested, preemptIdle) + if mode == executorModeBound { + gate := preemptLoad(driver.requestGate) + if gate&^executorGateMask != 0 || gate&executorGateClosed != 0 { + return false + } + requested = requested || gate&executorGateRequested != 0 + } else if preemptCompareAndSwap(&p.schedule, scheduleRequested, scheduleIdle) { + requested = true + } + if requested { + return true + } + if budget <= preemptCheckpointStride { + p.servicePreemptBudget = servicePreemptSafepointBudget + return servicePreemptBudgetExpired(p, mode) + } + p.servicePreemptBudget = budget - preemptCheckpointStride + return false +} + +// servicePreemptBudgetExpired keeps the target/locked-owner selection policy +// out of the common compiler-poll body. It runs only once per complete service +// budget, while explicit requests retain the shorter checkpoint bound. +// +//go:noinline +func servicePreemptBudgetExpired(p *P, mode uint32) bool { + if mode != executorModeBound || runnableForOSThreadOwner(p) || HasWaiting(p) || + osThreadSuspendCurrentPeerNeedsService(p) { + return true + } + driver := p.executor + return driver != nil && driver.servicePressure != nil && + preemptLoad(driver.servicePressure) != 0 +} + // acknowledgeSuspendedGPreempt consumes a request which was already satisfied // by an actual Yield/Park suspension. Keeping it on a stable runnable/waiting G // would spuriously prevent P-neutral transfer even though the scheduler has @@ -555,6 +645,15 @@ func dequeue(p *P) *G { if p == nil || p.readyHead == nil || p.readyCount == 0 { return nil } + return dequeueReadyHeadUnchecked(p) +} + +// dequeueReadyHeadUnchecked consumes the queue-head capability held by the +// scheduler owner. Callers must already have proved a non-empty header in the +// same reduction; keeping the mutation suffix here prevents optimized bounded +// selectors from duplicating it while the arbitrary-caller dequeue retains its +// complete fail-closed gate above. +func dequeueReadyHeadUnchecked(p *P) *G { g := p.readyHead p.readyHead = g.nextReady if p.readyHead == nil { @@ -603,16 +702,30 @@ func nextOSThreadRunnable(p *P) *G { return nil } -// dequeueOSThreadRunnable removes the first G this physical owner is permitted -// to execute without reordering any other ready entry. -func dequeueOSThreadRunnable(p *P) *G { +// dequeueOSThreadRunnableWithTransfer selects and removes the first G this +// physical owner may execute in one scheduler-owned pass. imported reports the +// transfer marker consumed by the dequeue so a later BeginRunG rejection can +// restore the exact queue state without selecting the same G a second time. +func dequeueOSThreadRunnableWithTransfer(p *P) (*G, bool) { if p == nil || p.readyHead == nil || p.readyCount == 0 { - return nil + return nil, false } selected := nextOSThreadRunnable(p) + return dequeueSelectedOSThreadRunnableWithTransfer(p, selected) +} + +// dequeueSelectedOSThreadRunnableWithTransfer consumes the exact candidate +// previously selected under this same scheduler-owner reduction. The common +// unlocked case is the ready head and stays O(1); the exceptional detached +// LockOSThread phase may select a later peer and retains the bounded list walk. +func dequeueSelectedOSThreadRunnableWithTransfer(p *P, selected *G) (*G, bool) { + if p == nil || p.readyHead == nil || p.readyCount == 0 { + return nil, false + } if selected == nil { - return nil + return nil, false } + imported := selected.transferState == runnableTransferGImported var previous *G for current := p.readyHead; current != nil; current = current.nextReady { if current != selected { @@ -624,7 +737,7 @@ func dequeueOSThreadRunnable(p *P) *G { if g != nil && g.transferState == runnableTransferGImported { g.transferState = runnableTransferGIdle } - return g + return g, imported } previous.nextReady = current.nextReady if p.readyTail == current { @@ -636,9 +749,16 @@ func dequeueOSThreadRunnable(p *P) *G { if current.transferState == runnableTransferGImported { current.transferState = runnableTransferGIdle } - return current + return current, imported } - return nil + return nil, false +} + +// dequeueOSThreadRunnable retains the compatibility shape for lifecycle and +// shutdown callers which do not need to roll back a consumed transfer marker. +func dequeueOSThreadRunnable(p *P) *G { + g, _ := dequeueOSThreadRunnableWithTransfer(p) + return g } func enqueueParkSet(p *P, g *G) bool { @@ -653,11 +773,19 @@ func enqueueParkSet(p *P, g *G) bool { } func validRunnableParkState(state *ParkState) bool { - if !validParkState(state) { + if state == nil { + return false + } + switch state.phase { + case parkIdle, parkConsumed, parkDelivered: + return validReleasableParkState(state) + case parkMaterialized: + return validMaterializedParkState(state, state.ticket) + case parkReady: + return validParkState(state) + default: return false } - return state.phase == parkIdle || state.phase == parkConsumed || state.phase == parkDelivered || - state.phase == parkReady || state.phase == parkMaterialized } // validRunnableRunAction distinguishes an ordinary runnable suspension from a @@ -859,11 +987,11 @@ func NextRunnableAt(p *P, now int64) (g *G, ok bool) { return dequeueOSThreadRunnable(p), true } -func dispatchPending(g *G, resumed *Frame) (destroy *Frame, yielded bool, ok bool) { +func dispatchPending(g *G, resumed *Frame) (destroy *Frame, yielded, directPark, ok bool) { pending := g.pending g.pending = pendingTransition{} if pending.from != resumed { - return nil, false, false + return nil, false, false, false } switch pending.kind { case pendingAwait: @@ -871,55 +999,63 @@ func dispatchPending(g *G, resumed *Frame) (destroy *Frame, yielded bool, ok boo if child == nil || child.parent != resumed || resumed.header == nil || child.header == nil || resumed.header.Lifecycle != uint16(FrameSuspended) || child.header.Lifecycle != uint16(FrameInitialSuspended) { - return nil, false, false + return nil, false, false, false } resumed.state = FrameSuspended g.active = child - return nil, false, true + return nil, false, false, true case pendingComplete: if pending.target != nil || resumed.header == nil || resumed.header.Lifecycle != uint16(FrameFinalSuspended) || !completionMatchesTerminalFrame(resumed) { - return nil, false, false + return nil, false, false, false } g.active = resumed.parent resumed.state = FrameDestroyPending resumed.header.Lifecycle = uint16(FrameDestroyPending) g.destroyTarget = resumed - return resumed, false, true + return resumed, false, false, true case pendingYield: if pending.target != nil || resumed.header == nil || resumed.header.SuspendReason != uint16(SuspendYield) || resumed.header.Lifecycle != uint16(FrameSuspended) { - return nil, false, false + return nil, false, false, false } resumed.state = FrameSuspended - return nil, true, true + return nil, true, false, true case pendingParkSet: if pending.target != nil || resumed.header == nil || resumed.header.SuspendReason != uint16(SuspendPark) || resumed.header.Lifecycle != uint16(FrameSuspended) || - g.waiting || - !validParkState(&g.park) || g.park.phase != parkParked || - !validCommittedWaitSetRecord(resumed.parkWait, g, resumed) { - return nil, false, false + g.waiting { + return nil, false, false, false + } + wait := resumed.parkWait + direct := wait != nil && wait.directChannel + if direct { + if !validCommittedDirectChannelPark(g, resumed, wait) { + return nil, false, false, false + } + } else if !validParkState(&g.park) || g.park.phase != parkParked || + !validCommittedWaitSetRecord(wait, g, resumed) { + return nil, false, false, false } resumed.state = FrameSuspended - return nil, false, true + return nil, false, direct, true case pendingPanic: if pending.target != nil || resumed.header == nil || resumed.header.SuspendReason != uint16(SuspendPanic) || resumed.header.Lifecycle != uint16(FrameFinalSuspended) || g.panicUnwind || !publishedPanicRecord(&g.panicRecord) { - return nil, false, false + return nil, false, false, false } g.active = resumed.parent resumed.state = FrameDestroyPending resumed.header.Lifecycle = uint16(FrameDestroyPending) g.destroyTarget = resumed g.panicUnwind = true - return resumed, false, true + return resumed, false, false, true default: - return nil, false, false + return nil, false, false, false } } @@ -1004,6 +1140,79 @@ func BeginRunG(p *P, g *G) (Action, bool) { return action, true } +// beginExecutorRunContinuation consumes the private certificate installed by +// pauseExecutorRunAction or materialized park promotion. The executor selector +// has just validated the queue header, selected this G under the exact +// LockOSThread phase, and removed its queue/transfer links. All producer-visible +// work ended before the continuation was published. Rechecking the complete G, +// ParkState, and frame graph here would therefore duplicate the insertion +// boundary rather than establish a new ownership fact. +// +// Keep this unexported: arbitrary callers and compatibility paths must retain +// BeginRunG's complete validator. +func beginExecutorRunDirectChannelContinuation(p *P, g *G, peerRunnable bool) (Action, bool) { + // The selector has just proved an idle P, selected this exact queue member, + // and removed its queue/transfer links. The caller selected this helper only + // after observing the private materialized-direct certificate installed by + // the owner-only no-fail promotion suffix. That suffix also wrote GRunnable, + // waiting=false, runP=nil and the queue fields which dequeue has just + // consumed; replaying those correlated owner-only bytes here is not another + // safety boundary. Recheck only asynchronous preemption and the exact handle + // consumed below. + if p == nil || g == nil || !gPreemptEnabledAtDepthZero(g) || + g.active == nil || g.active.handle == nil { + return Action{}, false + } + schedule := preemptLoad(&p.schedule) + if schedule != scheduleIdle && schedule != scheduleRequested { + return Action{}, false + } + handle := g.active.handle + budget := servicePreemptSafepointBudget + if peerRunnable { + budget = preemptCheckpointStride + } + p.current = g + p.servicePreemptBudget = budget + g.state = GRunning + g.runP = p + action := Action{Kind: ActionCheckResume, Handle: handle} + p.action = action + return action, true +} + +func beginExecutorRunContinuation(p *P, g *G, peerRunnable bool) (Action, bool) { + if p == nil || g == nil || p.current != nil || p.inResume || p.inlineAwaitDepth != 0 || + p.action != (Action{}) || p.runDecision != (RunDecision{}) || p.runDecisionTaken || + p.servicePreemptBudget != 0 || g.magic != gMagic || g.state != GRunnable || + (g.runAction != ActionCheckResume && + (g.runAction != ActionInvalid || g.park.phase != parkMaterialized)) || + g.queued || g.nextReady != nil || g.waiting || + g.runP != nil || g.transferState != runnableTransferGIdle || + !gPreemptEnabledAtDepthZero(g) || g.active == nil || g.active.handle == nil || + g.active.header == nil || + (g.active.state != FrameInitialSuspended && g.active.state != FrameSuspended) { + return Action{}, false + } + schedule := preemptLoad(&p.schedule) + if schedule != scheduleIdle && schedule != scheduleRequested { + return Action{}, false + } + handle := g.active.handle + budget := servicePreemptSafepointBudget + if peerRunnable { + budget = preemptCheckpointStride + } + p.current = g + p.servicePreemptBudget = budget + g.state = GRunning + g.runP = p + g.runAction = ActionInvalid + action := Action{Kind: ActionCheckResume, Handle: handle} + p.action = action + return action, true +} + type executorRunQueuePlacement uint8 const ( @@ -1112,6 +1321,152 @@ func Checked(p *P, g *G, action Action, done bool) (Action, bool) { } } +// CheckedExecutorRun is Checked for the no-return physical interval opened by +// ExecutorRunStepAction. The private issued marker and exact P action were +// established after the scheduler's dequeue/frame audit, and llvm.coro.done is +// observational, so replaying that complete audit before the adjacent +// resume/destroy would not create another ownership boundary. Compiler code +// still has to consume the run decision before it may publish a transition. +func checkedExecutorRun( + driver *ExecutorDriver, + g *G, + action Action, + done bool, + withRuntimeContextMode bool, +) (next Action, needsRuntimeContext bool, ok bool) { + if driver == nil || g == nil || driver.p == nil || + driver.run.issued != action.Kind || action.Flags != 0 || action.Handle == nil { + return Action{}, false, false + } + p := driver.p + if p.current != g || g.runP != p || p.inResume || + p.inlineAwaitDepth != 0 { + return Action{}, false, false + } + switch action.Kind { + case ActionCheckResume: + if done || g.state != GRunning || g.active == nil || + g.active.handle != action.Handle { + return Action{}, false, false + } + if withRuntimeContextMode { + mode := g.active.runtimeContext + if mode == frameRuntimeContextUnknown { + var valid bool + mode, valid = descriptorRuntimeContextMode(g.active.descriptor) + if !valid { + return Action{}, false, false + } + } + needsRuntimeContext = mode != frameRuntimeContextNotRequired + } + if !prepareIssuedRunDecision(p, g) { + return Action{}, false, false + } + g.active.state = FrameActive + p.inResume = true + next = Action{Kind: ActionResume, Handle: action.Handle} + p.action = next + return next, needsRuntimeContext, true + case ActionCheckDestroy: + if !done || g.state != GDispatching || g.destroyTarget == nil || + g.destroyTarget.handle != action.Handle || + g.destroyTarget.state != FrameDestroyPending || + g.park.taskCancelPhase == taskCancelRequested { + return Action{}, false, false + } + next = Action{Kind: ActionDestroy, Handle: action.Handle} + p.action = next + return next, false, true + default: + return Action{}, false, false + } +} + +func CheckedExecutorRun( + driver *ExecutorDriver, + g *G, + action Action, + done bool, +) (Action, bool) { + next, _, ok := checkedExecutorRun(driver, g, action, done, false) + return next, ok +} + +// CheckedExecutorRunRuntimeContext is the hot physical-resume transaction. It +// consumes the publication-time descriptor capability in the same scheduler +// audit which opens ActionResume, avoiding both a second ownership audit and a +// per-resume descriptor/string decode. Unknown legacy frames fail closed by +// validating their live descriptor before any scheduler state is mutated. +func CheckedExecutorRunRuntimeContext( + driver *ExecutorDriver, + g *G, + action Action, + done bool, +) (next Action, needsRuntimeContext bool, ok bool) { + return checkedExecutorRun(driver, g, action, done, true) +} + +// BeginIssuedExecutorResumeRuntimeContext consumes the bounded runner's +// private CheckResume episode without an llvm.coro.done probe. A resume action +// is created only for an initial- or ordinary-suspended frame; a physical +// resume which reaches final suspend returns a destroy action and can never be +// selected as CheckResume again. The issued marker and exact P/G/frame +// correlation below are therefore the stronger scheduler-owned capability. +// +// Compatibility callers which present an independently observed handle keep +// using CheckedExecutorRunRuntimeContext, and every destroy still proves +// llvm.coro.done before invoking llvm.coro.destroy. +func BeginIssuedExecutorResumeRuntimeContext( + driver *ExecutorDriver, + g *G, +) (next Action, needsRuntimeContext bool, ok bool) { + if driver == nil || g == nil || driver.p == nil || + driver.run.issued != ActionCheckResume { + return Action{}, false, false + } + p := driver.p + action := p.action + if p.current != g || g.runP != p || p.action != action || p.inResume || + p.inlineAwaitDepth != 0 || action.Kind != ActionCheckResume || + action.Flags != 0 || action.Handle == nil || g.state != GRunning || g.active == nil || + g.active.handle != action.Handle { + return Action{}, false, false + } + mode := g.active.runtimeContext + if mode == frameRuntimeContextUnknown { + var valid bool + mode, valid = descriptorRuntimeContextMode(g.active.descriptor) + if !valid { + return Action{}, false, false + } + } + needsRuntimeContext = mode != frameRuntimeContextNotRequired + if g.park.phase == parkMaterialized && g.park.directChannel && + g.park.taskCancelKind == TaskCancelNone && g.park.taskCancelPhase == taskCancelIdle { + // The direct-channel materializer published this exact owner-only + // certificate before enqueueing G, and the issued selector removed G + // immediately before this resume. Construct its small decision here; + // the general helper remains authoritative for source-backed parks and + // the cancellation race, which must claim the task token. + p.runDecision = RunDecision{ + g: g, + ticket: g.park.ticket, + caseID: g.park.winnerCase, + outcome: g.park.outcome, + materialized: true, + directChannel: true, + } + } else if !prepareIssuedRunDecision(p, g) { + return Action{}, false, false + } + g.active.state = FrameActive + p.inResume = true + next = Action{Kind: ActionResume, Handle: action.Handle} + p.action = next + return next, needsRuntimeContext, true +} + // Resumed commits the return from a direct llvm.coro.resume call. Coroutine // hooks must have recorded exactly one await or completion transition while // the frame was active. @@ -1125,7 +1480,7 @@ func Resumed(p *P, g *G, action Action) (Action, bool) { } p.inResume = false g.state = GDispatching - destroy, yielded, ok := dispatchPending(g, resumed) + destroy, yielded, directPark, ok := dispatchPending(g, resumed) if !ok { return Action{}, false } @@ -1162,7 +1517,9 @@ func Resumed(p *P, g *G, action Action) (Action, bool) { p.current = nil p.servicePreemptBudget = 0 p.action = Action{} - if !enqueueParkSet(p, g) { + if directPark { + activateWaitSetRecordUnchecked(p, g, g.active.parkWait) + } else if !enqueueParkSet(p, g) { return Action{}, false } return Action{Kind: ActionPark}, true @@ -1485,7 +1842,7 @@ func TerminalG(p *P, g *G) bool { ValidG(g) && gPreemptStateAtDepthZero(g, preemptDisabled) && g.state == GDead && g.root == nil && g.active == nil && g.frames == nil && g.taskControlLeases == 0 && g.runAction == ActionInvalid && g.transferState == runnableTransferGIdle && g.osThreadLockDepth == 0 && - g.pending.kind == pendingNone && g.pending.from == nil && g.pending.target == nil && + g.pending.kind == pendingNone && !g.pending.directChannel && g.pending.from == nil && g.pending.target == nil && g.destroyTarget == nil && !g.destroyRoot && g.nextReady == nil && !g.queued && !g.waiting && g.runP == nil && releasableParkState(&g.park) && g.park.taskCancelKind == TaskCancelNone && diff --git a/runtime/internal/coro/scheduler_preempt_test.go b/runtime/internal/coro/scheduler_preempt_test.go index 0ff46eb719..d5a5b77df5 100644 --- a/runtime/internal/coro/scheduler_preempt_test.go +++ b/runtime/internal/coro/scheduler_preempt_test.go @@ -111,6 +111,26 @@ func TestPreemptPollFailsClosedAndConsumesOnlyActiveRequest(t *testing.T) { runtime.KeepAlive(task.frame.memory) } +func TestCompilerPreemptPollRejectsBrokenOwnerBeforeConsumingRequest(t *testing.T) { + p := new(P) + task := newYieldingTestG(t, "compiler-poll-owner") + _ = beginWaitTestResume(t, p, task) + if !RequestPreempt(task.g) { + t.Fatal("request active compiler-poll G") + } + p.current = nil + if PollPreemptCompiler(task.g) || loadGPreempt(task.g) != preemptRequested || + p.servicePreemptBudget != servicePreemptSafepointBudget { + t.Fatal("compiler poll consumed request or budget through a broken owner") + } + p.current = task.g + if !PollPreemptCompiler(task.g) || loadGPreempt(task.g) != preemptIdle || + p.servicePreemptBudget != servicePreemptSafepointBudget { + t.Fatal("restored compiler poll did not consume the exact G request") + } + runtime.KeepAlive(task.frame.memory) +} + func TestBeginRunGDoesNotImmediatelyPreemptWithoutCompetitor(t *testing.T) { task := newYieldingTestG(t, "single") p := new(P) diff --git a/runtime/internal/coro/scheduler_service_preempt_test.go b/runtime/internal/coro/scheduler_service_preempt_test.go index df5d439abf..97c713cb33 100644 --- a/runtime/internal/coro/scheduler_service_preempt_test.go +++ b/runtime/internal/coro/scheduler_service_preempt_test.go @@ -21,7 +21,7 @@ import ( "testing" ) -func consumeServicePreemptTestBudget(t *testing.T, p *P, g *G) { +func consumeServicePreemptTestBudget(t *testing.T, p *P, g *G, wantYield bool) { t.Helper() if p.servicePreemptBudget != servicePreemptSafepointBudget { t.Fatalf("initial service preemption budget = %d, want %d", p.servicePreemptBudget, servicePreemptSafepointBudget) @@ -35,18 +35,22 @@ func consumeServicePreemptTestBudget(t *testing.T, p *P, g *G) { t.Fatalf("service preemption budget after safepoint %d = %d, want %d", safepoint, p.servicePreemptBudget, want) } } - if !pollCompilerSafepointForTest(t, g) { - t.Fatalf("service preemption did not fire at safepoint %d", servicePreemptSafepointBudget) + if yielded := pollCompilerSafepointForTest(t, g); yielded != wantYield { + t.Fatalf("service preemption at safepoint %d = %t, want %t", + servicePreemptSafepointBudget, yielded, wantYield) } if p.servicePreemptBudget != servicePreemptSafepointBudget { t.Fatalf("fired service preemption budget = %d, want reload %d", p.servicePreemptBudget, servicePreemptSafepointBudget) } } -func runServicePreemptTestQuantum(t *testing.T, p *P, task *yieldingTestG) { +func runServicePreemptTestQuantum(t *testing.T, p *P, task *yieldingTestG, wantServiceYield bool) { t.Helper() action := beginWaitTestResume(t, p, task) - consumeServicePreemptTestBudget(t, p, task.g) + consumeServicePreemptTestBudget(t, p, task.g, wantServiceYield) + if !wantServiceYield && (!RequestPreempt(task.g) || !PollPreemptCompiler(task.g)) { + t.Fatal("explicit request did not end a bound sole-runnable service slice") + } yieldRunningDriverTask(t, p, task, action) if p.servicePreemptBudget != 0 || p.current != nil { t.Fatalf("service yield retained run state: budget=%d current=%p", p.servicePreemptBudget, p.current) @@ -66,11 +70,11 @@ func finishServicePreemptTestTask(t *testing.T, p *P, task *yieldingTestG) { runtime.KeepAlive(task.frame.memory) } -func TestServicePreemptQuantumIsEventSourceIndependent(t *testing.T) { +func TestServicePreemptQuantumUsesBoundRequestContract(t *testing.T) { t.Run("unbound", func(t *testing.T) { p := new(P) task := newYieldingTestG(t, "service-unbound") - runServicePreemptTestQuantum(t, p, task) + runServicePreemptTestQuantum(t, p, task, true) finishServicePreemptTestTask(t, p, task) }) @@ -78,7 +82,7 @@ func TestServicePreemptQuantumIsEventSourceIndependent(t *testing.T) { p := new(P) driver, registry, _ := bindTestExecutorDriver(t, p) task := newYieldingTestG(t, "service-source-empty") - runServicePreemptTestQuantum(t, p, task) + runServicePreemptTestQuantum(t, p, task, false) closeTestExecutorDriver(t, driver) finishServicePreemptTestTask(t, p, task) if !registry.CanRelease() { @@ -90,7 +94,7 @@ func TestServicePreemptQuantumIsEventSourceIndependent(t *testing.T) { p := new(P) driver, registry, timers, _ := bindTestExecutorDriverWithTimers(t, p) task := newYieldingTestG(t, "service-timer-empty") - runServicePreemptTestQuantum(t, p, task) + runServicePreemptTestQuantum(t, p, task, false) closeTestExecutorDriver(t, driver) finishServicePreemptTestTask(t, p, task) if !timers.CanRelease() || !registry.CanRelease() { @@ -98,6 +102,145 @@ func TestServicePreemptQuantumIsEventSourceIndependent(t *testing.T) { } }) + t.Run("bound-waiting-source-audit", func(t *testing.T) { + p := new(P) + driver, registry, _ := bindTestExecutorDriver(t, p) + task := newYieldingTestG(t, "service-waiting-source") + action := beginWaitTestResume(t, p, task) + + // The source-specific park tests cover the complete queue shape. This + // fixture isolates the service policy: a non-empty logical wait set must + // retain periodic source audits even when no runnable peer exists. + wait := new(WaitSetRecord) + p.parkWaitHead, p.parkWaitTail = wait, wait + consumeServicePreemptTestBudget(t, p, task.g, true) + p.parkWaitHead, p.parkWaitTail = nil, nil + + yieldRunningDriverTask(t, p, task, action) + closeTestExecutorDriver(t, driver) + finishServicePreemptTestTask(t, p, task) + if !registry.CanRelease() { + t.Fatal("waiting-source service quantum retained executor state") + } + }) + + t.Run("bound-remote-quota-pressure", func(t *testing.T) { + p := new(P) + driver, registry, _ := bindTestExecutorDriver(t, p) + quota := new(ExecutionQuota) + if !quota.Start(1) || !BindExecutorServicePressure(driver, quota) || + BindExecutorServicePressure(driver, quota) { + t.Fatal("bind shared execution-quota service pressure") + } + if acquired, ok := quota.TryAcquire(1); !acquired || !ok { + t.Fatal("acquire current route execution quota") + } + if acquired, ok := quota.TryAcquire(2); acquired || !ok { + t.Fatalf("publish remote quota contention = (%t, %t)", acquired, ok) + } + + task := newYieldingTestG(t, "service-remote-quota-pressure") + runServicePreemptTestQuantum(t, p, task, true) + + if wake, ok := quota.Release(1); !wake || !ok { + t.Fatalf("release current route quota = (%t, %t)", wake, ok) + } + if acquired, ok := quota.TryAcquire(2); !acquired || !ok { + t.Fatal("remote route did not acquire released execution quota") + } + if _, ok := quota.Release(2); !ok { + t.Fatal("release remote route execution quota") + } + if _, ok := quota.Seal(); !ok || !quota.Quiesced() || !quota.Retire() { + t.Fatal("retire shared execution quota") + } + + closeTestExecutorDriver(t, driver) + finishServicePreemptTestTask(t, p, task) + if !registry.CanRelease() { + t.Fatal("remote-pressure service quantum retained executor state") + } + }) + + t.Run("bound-locked-handoff-current-peer", func(t *testing.T) { + p := new(P) + driver, _, _ := bindTestExecutorDriver(t, p) + owner := newYieldingTestG(t, "service-locked-owner") + peer := newYieldingTestG(t, "service-replacement-peer") + if !Enqueue(p, owner.g) || !Enqueue(p, peer.g) { + t.Fatal("enqueue locked service fixture") + } + kind := commitLockedRunnerYield(t, driver, owner) + if required, ok := PrepareOSThreadSuspendHandoff(driver, owner.g, kind.Kind); !ok || !required { + t.Fatal("prepare locked service handoff") + } + + step := runnerNextPhysicalAction(t, driver, peer, ActionCheckResume) + resume, ok := Checked(p, peer.g, step.Action, false) + if !ok || resume.Kind != ActionResume { + t.Fatal("check replacement service peer") + } + takeNormalRunnerDecision(t, peer.g) + peer.frame.header.SuspendReason = uint16(SuspendNone) + peer.frame.header.Lifecycle = uint16(FrameActive) + consumeServicePreemptTestBudget(t, p, peer.g, true) + peer.frame.header.SuspendReason = uint16(SuspendYield) + peer.frame.header.Lifecycle = uint16(FrameSuspended) + if !PrepareYield(peer.g, peer.handle, peer.frame.header) { + t.Fatal("prepare replacement service peer yield") + } + next, resumed := Resumed(p, peer.g, resume) + if !resumed || next.Kind != ActionYield || + !CommitExecutorRunAction(driver, peer.g, next) { + t.Fatalf("commit replacement service peer = (%+v, %t)", next, resumed) + } + if !RestoreOSThreadSuspendHandoff(driver, owner.g) { + t.Fatal("restore locked owner after service peer action") + } + runtime.KeepAlive(owner.frame.memory) + runtime.KeepAlive(peer.frame.memory) + }) + +} + +func TestCompilerPreemptPollObservesConcurrentExecutorRequest(t *testing.T) { + p := new(P) + driver, registry, executor := bindTestExecutorDriver(t, p) + task := newYieldingTestG(t, "concurrent-executor-request") + action := beginWaitTestResume(t, p, task) + + start := make(chan struct{}) + result := make(chan ExecutorRequestResult, 1) + go func() { + <-start + result <- registry.Request(executor) + }() + close(start) + + observed := false + for attempt := 0; attempt != 100000 && !observed; attempt++ { + observed = PollPreemptCompiler(task.g) + if !observed { + runtime.Gosched() + } + } + if request := <-result; request != ExecutorRequestPublished { + t.Fatalf("concurrent executor request = %d, want published", request) + } + if !observed { + t.Fatal("compiler poll missed a concurrent sticky executor request") + } + + yieldRunningDriverTask(t, p, task, action) + if promoted, ok := PollReady(p); !ok || promoted != 0 || registry.ObserveRequested(executor) { + t.Fatalf("concurrent executor request acknowledgment = (%d, %t), requested=%t", + promoted, ok, registry.ObserveRequested(executor)) + } + closeTestExecutorDriver(t, driver) + finishServicePreemptTestTask(t, p, task) + if !registry.CanRelease() { + t.Fatal("concurrent executor request retained executor state") + } } func TestServicePreemptBudgetRejectsStaleIdleState(t *testing.T) { @@ -124,7 +267,7 @@ func TestExplicitRequestsPrecedeServiceBudget(t *testing.T) { p := new(P) task := newYieldingTestG(t, "service-request-g") action := beginWaitTestResume(t, p, task) - if !RequestPreempt(task.g) || !PollPreempt(task.g) { + if !RequestPreempt(task.g) || !PollPreemptCompiler(task.g) { t.Fatal("G-local request was not observed") } if p.servicePreemptBudget != servicePreemptSafepointBudget { @@ -138,7 +281,7 @@ func TestExplicitRequestsPrecedeServiceBudget(t *testing.T) { p := new(P) task := newYieldingTestG(t, "service-request-p") action := beginWaitTestResume(t, p, task) - if !RequestSchedule(p) || !PollPreempt(task.g) { + if !RequestSchedule(p) || !PollPreemptCompiler(task.g) { t.Fatal("P scheduling request was not observed") } if p.servicePreemptBudget != servicePreemptSafepointBudget { @@ -153,7 +296,7 @@ func TestExplicitRequestsPrecedeServiceBudget(t *testing.T) { driver, registry, executor := bindTestExecutorDriver(t, p) task := newYieldingTestG(t, "service-request-executor") action := beginWaitTestResume(t, p, task) - if registry.Request(executor) != ExecutorRequestPublished || !PollPreempt(task.g) { + if registry.Request(executor) != ExecutorRequestPublished || !PollPreemptCompiler(task.g) { t.Fatal("executor scheduling request was not observed") } if p.servicePreemptBudget != servicePreemptSafepointBudget { diff --git a/runtime/internal/coro/scheduler_spawn_test.go b/runtime/internal/coro/scheduler_spawn_test.go index 78492e4c9f..517ddbccaf 100644 --- a/runtime/internal/coro/scheduler_spawn_test.go +++ b/runtime/internal/coro/scheduler_spawn_test.go @@ -97,8 +97,12 @@ func beginSpawnTestChildResume(t *testing.T, p *P, g *G, frame *testFrame) Actio func yieldSpawnTestG(t *testing.T, p *P, g *G, frame *testFrame, action Action) { t.Helper() - if !PollPreempt(g) { - t.Fatal("spawn commit did not request parent preemption") + // A sole newly spawned child now stays local without forcing the parent to + // yield. Tests which need to hand execution to that child request the + // scheduling point explicitly; a burst with existing ready work still + // coalesces the same request in CommitSpawn. + if !RequestPreempt(g) || !PollPreempt(g) { + t.Fatal("request spawned-parent preemption") } frame.header.SuspendReason = uint16(SuspendYield) frame.header.Lifecycle = uint16(FrameSuspended) @@ -274,18 +278,31 @@ func TestSpawnCommitDiscardedResultAtomicAndTaskReclaim(t *testing.T) { t.Fatal("rejected result layout partially committed spawn") } descriptor.ResultAlign = 8 + p.current = nil + if CommitSpawn(parent.g, child, handle) { + t.Fatal("spawn transaction accepted a lost resume owner") + } + p.current = parent.g + child.spawnP = new(P) + if CommitSpawn(parent.g, child, handle) { + t.Fatal("spawn transaction accepted a mismatched P certificate") + } + child.spawnP = p if !CommitSpawn(parent.g, child, handle) { t.Fatal("commit goroutine root with discarded result") } if parent.g.spawnChild != nil || child.root == nil || child.active != child.root || child.state != GRunnable || child.root.header.ResultSlot != nil || !child.queued || - p.readyHead != child || p.readyTail != child || preemptLoad(preemptAddress(parent.g)) != preemptRequested { + p.readyHead != child || p.readyTail != child || preemptLoad(preemptAddress(parent.g)) != preemptIdle { t.Fatal("committed spawn state is incomplete") } if CommitSpawn(parent.g, child, handle) || p.readyHead != child || p.readyTail != child || child.nextReady != nil { t.Fatal("duplicate spawn commit changed the ready queue") } + if !RequestPreempt(parent.g) { + t.Fatal("request explicit parent yield after sole-child locality check") + } yieldSpawnTestG(t, p, parent.g, parent.frame, parentAction) if got, ok := NextRunnable(p); !ok || got != child { t.Fatalf("spawned child was not first after parent yield: (%p, %t)", got, ok) @@ -321,6 +338,34 @@ func TestSpawnCommitDiscardedResultAtomicAndTaskReclaim(t *testing.T) { runtime.KeepAlive(child) } +func TestCompletedTaskTransfersContextAndStorageAfterOneTerminalAudit(t *testing.T) { + g := new(G) + if !InitG(g) { + t.Fatal("initialize completed task transfer G") + } + g.taskStorage = unsafe.Pointer(g) + g.taskSize = TaskStorageSize() + g.taskState = taskStorageOwned + local := unsafe.Pointer(new(byte)) + if !BindTaskLocal(g, local) { + t.Fatal("bind completed task transfer context") + } + if !disableGPreempt(g) { + t.Fatal("disable completed task transfer preemption") + } + g.state = GDead + + releasedLocal, raw, size, owned, ok := ReleaseCompletedTask(g) + if !ok || !owned || releasedLocal != local || raw != unsafe.Pointer(g) || size != TaskStorageSize() || + g.taskLocal != nil || g.taskStorage != nil || g.taskSize != 0 || g.taskState != taskStorageReleased { + t.Fatalf("completed task transfer = local:%p raw:%p size:%d owned:%t ok:%t state:%d", + releasedLocal, raw, size, owned, ok, g.taskState) + } + if _, _, _, _, ok := ReleaseCompletedTask(g); ok { + t.Fatal("completed task transferred twice") + } +} + func TestSpawnReadyQueuePreservesFIFOAndParentFairness(t *testing.T) { p := new(P) parent := newYieldingTestG(t, "fair-parent") diff --git a/runtime/internal/coro/scheduler_test_helpers_test.go b/runtime/internal/coro/scheduler_test_helpers_test.go index 4d252e7bf9..777ba81692 100644 --- a/runtime/internal/coro/scheduler_test_helpers_test.go +++ b/runtime/internal/coro/scheduler_test_helpers_test.go @@ -57,7 +57,7 @@ func pollCompilerSafepointForTest(t *testing.T, g *G) bool { return false } g.active.header.StateID = preemptCheckpointStride - return PollPreempt(g) + return PollPreemptCompiler(g) } func finishWaitTestTask(t *testing.T, p *P, task *yieldingTestG, action Action) { diff --git a/runtime/internal/coro/shutdown.go b/runtime/internal/coro/shutdown.go index a3341a596b..8d86830448 100644 --- a/runtime/internal/coro/shutdown.go +++ b/runtime/internal/coro/shutdown.go @@ -27,9 +27,16 @@ func CommandMainReturnPoint(p *P, main *G) bool { } func validCancelFrame(frame *Frame, g *G) bool { - return frame != nil && frame.owner == g && frame.handle != nil && frame.header != nil && - frame.storage != nil && frame.rawBase != nil && frame.descriptor != nil && - frame.header.G == unsafe.Pointer(g) && frame.header.Descriptor == frame.descriptor && + if frame == nil || frame.owner != g || frame.handle == nil || frame.header == nil || + frame.descriptor == nil || frame.header.G != unsafe.Pointer(g) || + frame.header.Descriptor != frame.descriptor { + return false + } + if frame.borrowedStorage { + return frame.storage == nil && frame.rawBase == nil && frame.allocationSize == 0 && + frame.header.AllocationBase == unsafe.Pointer(frame) + } + return frame.storage != nil && frame.rawBase != nil && frame.header.AllocationBase == frame.rawBase } @@ -52,7 +59,7 @@ func validCancelFrame(frame *Frame, g *G) bool { func validCancelableReadyG(g *G) bool { if !ValidG(g) || g.state != GRunnable || !g.queued || g.waiting || g.runP != nil || g.root == nil || !releasableParkState(&g.park) || g.park.taskCancelKind != TaskCancelNone || - g.pending.kind != pendingNone || g.pending.from != nil || g.pending.target != nil || + g.pending.kind != pendingNone || g.pending.directChannel || g.pending.from != nil || g.pending.target != nil || g.spawnChild != nil || g.spawnParent != nil || g.spawnP != nil || g.taskControlLeases != 0 || g.taskState != taskStorageOwned || g.taskStorage != unsafe.Pointer(g) || g.taskSize != TaskStorageSize() { diff --git a/runtime/internal/coro/single_park_owner.go b/runtime/internal/coro/single_park_owner.go new file mode 100644 index 0000000000..6e82a30162 --- /dev/null +++ b/runtime/internal/coro/single_park_owner.go @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package coro + +import "unsafe" + +// singleParkPreparation is a stack-only capability for the common compiler +// owned one-event park transaction. Source-specific preflight runs between +// preflightSingleParkPreparation and begin, while no suspension or owner +// transition is possible. Channel, manual, and later single-source adapters +// therefore share one frame/ParkState proof without sharing producer policy. +type singleParkPreparation struct { + g *G + p *P + frame *Frame + wait *WaitSetRecord + ticket ParkTicket + seed uint32 +} + +func preflightSingleParkPreparation( + g *G, + handle unsafe.Pointer, + header *HeaderV1, + wait *WaitSetRecord, + seed uint32, +) (singleParkPreparation, bool) { + if !ValidG(g) || handle == nil || header == nil || wait == nil || + *wait != (WaitSetRecord{}) || !resumeGateTaken(g) || g.runP == nil || + g.pending.kind != pendingNone || g.spawnChild != nil || g.waiting || + !validReusableSingleParkState(&g.park) || g.park.attached != 0 || g.park.head != nil { + return singleParkPreparation{}, false + } + frame := findFrame(g, handle) + if frame == nil || frame != g.active || frame.header != header || frame.state != FrameActive || + frame.parkWait != nil || header.SuspendReason != uint16(SuspendPark) || + header.Lifecycle != uint16(FrameSuspended) { + return singleParkPreparation{}, false + } + ticket, ok := nextParkTicket(g.park.ticket) + if !ok { + return singleParkPreparation{}, false + } + return singleParkPreparation{ + g: g, + p: g.runP, + frame: frame, + wait: wait, + ticket: ticket, + seed: seed ^ ticket.generation*0x9e3779b9 ^ ticket.epoch*0x85ebca6b, + }, true +} + +func (prepared *singleParkPreparation) begin() { + prepared.g.park = ParkState{ + ticket: prepared.ticket, + phase: parkPreparing, + expected: 1, + seed: prepared.seed, + } + prepared.wait.g = prepared.g + prepared.wait.ticket = prepared.ticket + prepared.wait.state = waitSetRecordPreparing +} + +func (prepared *singleParkPreparation) commit(id OperationID, caseID uint32) bool { + state := &prepared.g.park + state.seed = 0 + state.phase = parkParked + if !validPreparedDirectChannelParkState(state, prepared.wait, id, caseID) { + return false + } + prepared.wait.state = waitSetRecordCommitted + prepared.frame.parkWait = prepared.wait + prepared.g.pending = pendingTransition{kind: pendingParkSet, from: prepared.frame} + return true +} diff --git a/runtime/internal/coro/spawn.go b/runtime/internal/coro/spawn.go index 8fad1a9899..9452099b8a 100644 --- a/runtime/internal/coro/spawn.go +++ b/runtime/internal/coro/spawn.go @@ -167,6 +167,23 @@ func BeginSpawn(parent, child *G, storage unsafe.Pointer, size uintptr) bool { return true } +// activeSpawnTransaction consumes the reciprocal parent/child/P links +// published by BeginSpawn as an unforgeable scheduler-owned certificate. The +// compiler's root factory cannot suspend or run another scheduler reduction, +// so the links remain exclusive until CommitSpawn or RollbackSpawn clears +// them. Revalidate the live resume owner because an asynchronous preemption +// request may arrive, but do not repeat every parent and queue audit already +// completed at BeginSpawn. +func activeSpawnTransaction(parent, child *G) (*P, bool) { + if parent == nil || child == nil || parent == child || + parent.spawnChild != child || child.spawnParent != parent || + child.spawnP == nil || parent.runP != child.spawnP || + !resumeGateTaken(parent) { + return nil, false + } + return child.spawnP, true +} + func validDiscardResultSpawnRoot(child *G, handle unsafe.Pointer) (*Frame, bool) { root := findFrame(child, handle) if root == nil || child.frames != root || root.next != nil || root.owner != child || @@ -180,7 +197,7 @@ func validDiscardResultSpawnRoot(child *G, handle unsafe.Pointer) (*Frame, bool) return nil, false } descriptor := (*FrameDescriptorV1)(root.descriptor) - if descriptor.Version != 1 || descriptor.Flags != 0 || + if descriptor.Version != 1 || descriptor.Flags&^frameDescriptorAllowedFlagsV1 != 0 || !validProgramPayloadLayoutV1(descriptor.ResultSize, descriptor.ResultAlign) { return nil, false } @@ -190,17 +207,19 @@ func validDiscardResultSpawnRoot(child *G, handle unsafe.Pointer) (*Frame, bool) // CommitSpawn atomically adopts the independently created root and appends its // G to the current P's ready queue. Every potentially failing check happens // before RequestPreempt and the scheduler-owned stores, so failure never -// exposes a half-adopted or half-enqueued child. The request forces the parent -// through its next compiler safepoint; yielding then places the parent behind -// the newly ready child. +// exposes a half-adopted or half-enqueued child. The first local child does not +// force an immediate parent yield: a parent which is about to park can hand the +// P directly to that child without exporting either continuation or ringing a +// peer doorbell. An already non-empty local queue requests preemption so bursts +// still reach the scheduler promptly and can be shared with idle Ps. func CommitSpawn(parent, child *G, handle unsafe.Pointer) bool { - p, ok := runningSpawnContext(parent) - if !ok || handle == nil || parent.spawnChild != child || child == nil || + p, ok := activeSpawnTransaction(parent, child) + if !ok || handle == nil || !ValidG(child) || child.state != GNew || child.root != nil || child.active != nil || child.pending.kind != pendingNone || child.pending.from != nil || child.pending.target != nil || child.destroyTarget != nil || child.destroyRoot || child.nextReady != nil || child.queued || child.waiting || child.runP != nil || - child.spawnChild != nil || child.spawnParent != parent || child.spawnP != p || + child.spawnChild != nil || child.transferState != runnableTransferGIdle || child.taskState != taskStorageOwned || child.taskStorage != unsafe.Pointer(child) || child.taskSize != TaskStorageSize() || !gPreemptStateAtDepthZero(child, preemptIdle) { @@ -212,8 +231,10 @@ func CommitSpawn(parent, child *G, handle unsafe.Pointer) bool { } // This cannot fail after the complete parent/child/P validation above. It is // intentionally issued before queue publication so no post-publication - // operation can force CommitSpawn to report failure. - if !RequestPreempt(parent) { + // operation can force CommitSpawn to report failure. A sole new child is + // serviced when the parent next parks, yields, or exhausts its ordinary + // compiler-poll quantum; it needs no eager scheduling transaction. + if p.readyCount != 0 && !RequestPreempt(parent) { return false } @@ -233,9 +254,8 @@ func CommitSpawn(parent, child *G, handle unsafe.Pointer) bool { // fail-stop: only the scheduler may destroy that handle, so the exported ABI // aborts instead of trying to free it on the parent executor stack. func RollbackSpawn(parent, child *G) (unsafe.Pointer, uintptr, bool) { - p, ok := runningSpawnContext(parent) - if !ok || parent.spawnChild != child || child == nil || !ValidG(child) || - child.spawnParent != parent || child.spawnP != p || child.spawnChild != nil || + _, ok := activeSpawnTransaction(parent, child) + if !ok || !ValidG(child) || child.spawnChild != nil || child.state != GNew || child.root != nil || child.active != nil || child.frames != nil || child.pending.kind != pendingNone || child.destroyTarget != nil || child.destroyRoot || child.nextReady != nil || child.queued || child.waiting || child.runP != nil || @@ -269,7 +289,7 @@ func ReclaimableG(g *G) bool { g.taskControlLeases == 0 && g.runAction == ActionInvalid && g.transferState == runnableTransferGIdle && g.osThreadLockDepth == 0 && g.root == nil && g.active == nil && g.frames == nil && - g.pending.kind == pendingNone && g.pending.from == nil && g.pending.target == nil && + g.pending.kind == pendingNone && !g.pending.directChannel && g.pending.from == nil && g.pending.target == nil && g.destroyTarget == nil && !g.destroyRoot && g.nextReady == nil && !g.queued && !g.waiting && g.runP == nil && releasableParkState(&g.park) && g.park.taskCancelKind == TaskCancelNone && @@ -312,6 +332,41 @@ func ReleaseTaskStorage(g *G) (raw unsafe.Pointer, size uintptr, ok bool) { return raw, size, true } +// ReleaseCompletedTask transfers both runtime-adapter context and physical +// task storage after one complete terminal audit. Production retirement owns +// these two values as one scheduler-thread transaction: the adapter destroys +// the returned context while the owned allocation is still live, then clears +// and frees that allocation. Clearing taskLocal/taskState here makes duplicate +// transfer fail without repeating ReclaimableG through each release stage. +// +// The narrower ReleaseTaskLocal and ReleaseTaskStorage APIs remain available +// for diagnostics and lifecycle tests which intentionally exercise stages in +// isolation. +func ReleaseCompletedTask(g *G) (local, raw unsafe.Pointer, size uintptr, owned, ok bool) { + if !ReclaimableG(g) || g.taskLocal == nil { + return nil, nil, 0, false, false + } + local = g.taskLocal + switch g.taskState { + case taskStorageStatic: + if g.taskStorage != nil || g.taskSize != 0 { + return nil, nil, 0, false, false + } + case taskStorageOwned: + if g.taskStorage != unsafe.Pointer(g) || g.taskSize != TaskStorageSize() { + return nil, nil, 0, false, false + } + raw, size, owned = g.taskStorage, g.taskSize, true + g.taskStorage = nil + g.taskSize = 0 + g.taskState = taskStorageReleased + default: + return nil, nil, 0, false, false + } + g.taskLocal = nil + return local, raw, size, owned, true +} + // DeadG is a narrow program-driver query. It does not imply that a command // main may safely return: TerminalG must still prove that no ready or parked G // survives. diff --git a/runtime/internal/coro/task_cancel.go b/runtime/internal/coro/task_cancel.go index b6d05f5955..6152345a9b 100644 --- a/runtime/internal/coro/task_cancel.go +++ b/runtime/internal/coro/task_cancel.go @@ -180,7 +180,8 @@ func validRegisteredActiveParkHeader(state *ParkState) bool { func validRegisteredReleasableParkHeader(state *ParkState) bool { if state == nil || state.resolving || !validTaskCancelState(state.taskCancelKind, state.taskCancelPhase) || - state.cancelKind > ParkCancelShutdown || state.attached > state.expected { + state.cancelKind > ParkCancelShutdown || state.attached > state.expected || + state.directChannel && state.phase != parkMaterialized { return false } switch state.phase { @@ -351,11 +352,16 @@ func requestTaskCancellationOwned(p *P, g *G, kind TaskCancelKind, proof taskCan return false } var wait *WaitSetRecord + var direct *DirectChannelCompletion if g.state == GWaiting && g.active != nil && g.active.parkWait != nil { wait = g.active.parkWait if g.park.resolving || g.park.winnerRecord != nil || - proof == taskCancellationProofRegistered && !validRegisteredActiveParkHeader(&g.park) || - !canAppendAffectedWaitSet(p, wait) { + proof == taskCancellationProofRegistered && !validRegisteredActiveParkHeader(&g.park) { + return false + } + if completion, compact := directChannelCompletionForWait(wait); compact { + direct = completion + } else if !canAppendAffectedWaitSet(p, wait) { return false } } else { @@ -397,6 +403,9 @@ func requestTaskCancellationOwned(p *P, g *G, kind TaskCancelKind, proof taskCan } g.park.taskCancelKind = strongest g.park.taskCancelPhase = taskCancelRequested + if direct != nil { + return requestDirectChannelCancellation(direct) + } if wait != nil { appendAffectedWaitSetUnchecked(p, wait) } @@ -473,7 +482,7 @@ func AcknowledgeTaskCancellation(g *G, kind TaskCancelKind) bool { g.runAction != ActionInvalid || g.transferState != runnableTransferGIdle || g.root != nil || g.active != nil || g.frames != nil || g.runP != nil || g.nextReady != nil || g.queued || g.waiting || - g.pending.kind != pendingNone || g.pending.from != nil || g.pending.target != nil || + g.pending.kind != pendingNone || g.pending.directChannel || g.pending.from != nil || g.pending.target != nil || g.destroyTarget != nil || g.destroyRoot || g.spawnChild != nil || g.spawnParent != nil || g.spawnP != nil || !releasableParkState(&g.park) { diff --git a/runtime/internal/coro/timer_park_owner.go b/runtime/internal/coro/timer_park_owner.go index c239794cb0..21e0c4b096 100644 --- a/runtime/internal/coro/timer_park_owner.go +++ b/runtime/internal/coro/timer_park_owner.go @@ -55,16 +55,8 @@ func CanReserveTimerV2(p *P, table *TimerRegistrationTable) bool { if table == nil || p == nil || table.owner != p || !table.route.Valid() { return false } - for index := uint32(0); index < TimerRegistrationConfiguredCapacity(table); index++ { - slot, ok := timerRegistrationSlotAt(table, index) - if !ok { - return false - } - if slot.generation != ^uint32(0) && reusableTimerRegistrationSlot(slot, table.route, index) { - return true - } - } - return false + _, _, ok := nextReusableTimerRegistrationSlot(table) + return ok } // PrepareSingleTimerPark installs one source-aware one-shot timer into the diff --git a/runtime/internal/coro/timer_registration.go b/runtime/internal/coro/timer_registration.go index 1f9bd63b7e..bb0b6d679f 100644 --- a/runtime/internal/coro/timer_registration.go +++ b/runtime/internal/coro/timer_registration.go @@ -102,6 +102,16 @@ type TimerRegistrationTable struct { owner *P route RouteID scanLimit uint32 + // reserveCursor is an owner-only circular allocation hint. Timer parks are + // commonly prepared in bursts; restarting every reservation at slot zero + // makes a dense burst quadratic even though the catalog is already stable + // storage. The cursor changes no handle identity and degrades safely to one + // bounded full-catalog search when the table is fragmented. + reserveCursor uint32 + activeCount uint32 + controlledCount uint32 + minimum int64 + minimumKnown bool } // TimerRegistrationConfiguredCapacity returns the exact number of addressable @@ -123,7 +133,120 @@ func timerRegistrationScanLimit(table *TimerRegistrationTable) (uint32, bool) { return 0, false } capacity := TimerRegistrationConfiguredCapacity(table) - return table.scanLimit, validSourceScanLimit(table.scanLimit, capacity) + return table.scanLimit, validSourceScanLimit(table.scanLimit, capacity) && + validTimerRegistrationIndex(table, capacity) +} + +func validTimerRegistrationIndex(table *TimerRegistrationTable, capacity uint32) bool { + if table == nil || capacity == 0 || table.reserveCursor >= capacity || + table.activeCount > table.scanLimit || table.controlledCount > table.activeCount { + return false + } + if table.activeCount == 0 { + return table.controlledCount == 0 && !table.minimumKnown && table.minimum == 0 + } + return !table.minimumKnown && table.minimum == 0 || table.minimumKnown && table.minimum >= 0 +} + +// timerRegistrationFastDeadline returns an owner-certified future deadline +// when no controlled generation must be inspected. A false skip asks the +// ordinary catalog pass to validate and publish concrete slots. An empty +// active set is also an exact skip even while delivered/canceled generations +// remain attached for resolution or resume cleanup. +func timerRegistrationFastDeadline(table *TimerRegistrationTable, owner *P, now int64) (deadline int64, hasDeadline, skip, ok bool) { + limit, valid := timerRegistrationScanLimit(table) + if !valid || table.owner != owner || now < 0 || limit == 0 && table.activeCount != 0 { + return 0, false, false, false + } + if table.activeCount == 0 { + return 0, false, true, true + } + if table.controlledCount == 0 && table.minimumKnown && now < table.minimum { + return table.minimum, true, true, true + } + return 0, false, false, true +} + +func (table *TimerRegistrationTable) attachActiveTimer(slot *timerRegistrationSlot) bool { + capacity := TimerRegistrationConfiguredCapacity(table) + if !validTimerRegistrationIndex(table, capacity) || slot == nil || slot.state != timerRegistrationActive || + table.activeCount >= capacity { + return false + } + wasEmpty := table.activeCount == 0 + table.activeCount++ + if slot.control != nil { + table.controlledCount++ + } + if wasEmpty { + table.minimum = slot.deadline + table.minimumKnown = true + } else if table.minimumKnown && slot.deadline < table.minimum { + table.minimum = slot.deadline + } + return true +} + +func (table *TimerRegistrationTable) detachActiveTimer(slot *timerRegistrationSlot) bool { + capacity := TimerRegistrationConfiguredCapacity(table) + if !validTimerRegistrationIndex(table, capacity) || slot == nil || slot.state != timerRegistrationActive || + table.activeCount == 0 || slot.control != nil && table.controlledCount == 0 { + return false + } + table.activeCount-- + if slot.control != nil { + table.controlledCount-- + } + if table.activeCount == 0 { + table.minimum = 0 + table.minimumKnown = false + } else if table.minimumKnown && slot.deadline == table.minimum { + // A complete source pass reconstructs the next exact minimum. Until + // then the source remains conservatively non-skippable. + table.minimum = 0 + table.minimumKnown = false + } + return true +} + +func (table *TimerRegistrationTable) commitTimerRegistrationMinimum(deadline int64, hasDeadline bool) bool { + capacity := TimerRegistrationConfiguredCapacity(table) + if !validTimerRegistrationIndex(table, capacity) || hasDeadline && (deadline < 0 || table.activeCount == 0) { + return false + } + if hasDeadline { + table.minimum = deadline + table.minimumKnown = true + } else { + table.minimum = 0 + table.minimumKnown = false + } + return true +} + +// nextReusableTimerRegistrationSlot starts at the owner-only circular hint and +// visits each physical slot at most once. It deliberately does not advance the +// hint: CanReserveTimerV2 may preflight a park before the exact reservation, +// and only a successfully attached generation is allowed to consume the hint. +func nextReusableTimerRegistrationSlot(table *TimerRegistrationTable) (uint32, *timerRegistrationSlot, bool) { + capacity := TimerRegistrationConfiguredCapacity(table) + if !validTimerRegistrationIndex(table, capacity) { + return 0, nil, false + } + for offset := uint32(0); offset < capacity; offset++ { + index := table.reserveCursor + offset + if index >= capacity { + index -= capacity + } + slot, ok := timerRegistrationSlotAt(table, index) + if !ok { + return 0, nil, false + } + if slot.generation != ^uint32(0) && reusableTimerRegistrationSlot(slot, table.route, index) { + return index, slot, true + } + } + return 0, nil, false } func timerRegistrationSlotAt(table *TimerRegistrationTable, index uint32) (*timerRegistrationSlot, bool) { @@ -306,39 +429,44 @@ func (table *TimerRegistrationTable) reserveAndAttachTimerV2( if !validTimerRegistrationController(controller, control, controlWord) { return TimerRegistrationHandle{}, false } - for index := uint32(0); index < TimerRegistrationConfiguredCapacity(table); index++ { - slot, slotOK := timerRegistrationSlotAt(table, index) - if !slotOK || slot.generation == ^uint32(0) || !reusableTimerRegistrationSlot(slot, table.route, index) { - continue - } - slot.state = timerRegistrationInitializing - if !raiseSourceScanLimit(&table.scanLimit, index, TimerRegistrationConfiguredCapacity(table)) { - return TimerRegistrationHandle{}, false - } - desired, idOK := timerRegistrationOperationID(table.route, index, slot.generation+1) - if !idOK || !PrepareOperationAtGeneration(&slot.record, desired) { - slot.state = timerRegistrationFree - continue - } - // Install the shared physical generation before any later failure can - // expose a copied desired ID to reuse. - slot.generation = desired.Generation - if !AttachParkWaitOperation(state, ticket, wait, &slot.record, caseID) { - if !AbortReservedOperation(&slot.record, desired) { - return TimerRegistrationHandle{}, false - } - slot.state = timerRegistrationFree + capacity := TimerRegistrationConfiguredCapacity(table) + index, slot, slotOK := nextReusableTimerRegistrationSlot(table) + if !slotOK { + return TimerRegistrationHandle{}, false + } + slot.state = timerRegistrationInitializing + if !raiseSourceScanLimit(&table.scanLimit, index, capacity) { + return TimerRegistrationHandle{}, false + } + desired, idOK := timerRegistrationOperationID(table.route, index, slot.generation+1) + if !idOK || !PrepareOperationAtGeneration(&slot.record, desired) { + slot.state = timerRegistrationFree + return TimerRegistrationHandle{}, false + } + // Install the shared physical generation before any later failure can + // expose a copied desired ID to reuse. + slot.generation = desired.Generation + if !AttachParkWaitOperation(state, ticket, wait, &slot.record, caseID) { + if !AbortReservedOperation(&slot.record, desired) { return TimerRegistrationHandle{}, false } - slot.p = p - slot.deadline = deadline - slot.controller = controller - slot.control = control - slot.controlWord = controlWord - slot.state = timerRegistrationActive - return TimerRegistrationHandle{Slot: index + 1, Generation: desired.Generation}, true + slot.state = timerRegistrationFree + return TimerRegistrationHandle{}, false + } + slot.p = p + slot.deadline = deadline + slot.controller = controller + slot.control = control + slot.controlWord = controlWord + slot.state = timerRegistrationActive + if !table.attachActiveTimer(slot) { + return TimerRegistrationHandle{}, false } - return TimerRegistrationHandle{}, false + table.reserveCursor = index + 1 + if table.reserveCursor == capacity { + table.reserveCursor = 0 + } + return TimerRegistrationHandle{Slot: index + 1, Generation: desired.Generation}, true } func (table *TimerRegistrationTable) ReserveAndAttachTimerV2( @@ -401,6 +529,15 @@ func (table *TimerRegistrationTable) nextDeadlineFor(owner *P) (deadline int64, if table == nil || table.owner != owner { return 0, false, false } + if _, valid := timerRegistrationScanLimit(table); !valid { + return 0, false, false + } + if table.activeCount == 0 { + return 0, false, true + } + if table.minimumKnown { + return table.minimum, true, true + } limit, valid := timerRegistrationScanLimit(table) if !valid { return 0, false, false @@ -430,6 +567,9 @@ func (table *TimerRegistrationTable) nextDeadlineFor(owner *P) (deadline int64, return 0, false, false } } + if !table.commitTimerRegistrationMinimum(deadline, hasDeadline) { + return 0, false, false + } return deadline, hasDeadline, true } @@ -487,6 +627,9 @@ func (table *TimerRegistrationTable) drainDueSlotFor(owner *P, now int64, index // physical slot Active so it cannot be recycled. return 0, 0, false, false } + if !table.detachActiveTimer(slot) { + return 0, 0, false, false + } slot.state = timerRegistrationDelivered return 1, 0, false, true } @@ -519,6 +662,9 @@ func (table *TimerRegistrationTable) drainDueFor(owner *P, now int64) (completed deadline, hasDeadline = next, true } } + if !table.commitTimerRegistrationMinimum(deadline, hasDeadline) { + return completed, 0, false, false + } return completed, deadline, hasDeadline, true } @@ -568,6 +714,9 @@ func (table *TimerRegistrationTable) ApplyTimerV2One(p *P, id OperationID, recor if disposition != OperationDispositionWinner { // Timer cancellation is the complete source-specific rollback: after // this transition no due delivery can retain or recreate the result. + if slot.state == timerRegistrationActive && !table.detachActiveTimer(slot) { + return OperationApplyInvalid + } slot.state = timerRegistrationCanceled if slot.record.resultState == operationResultOwned && !DiscardUnselectedOperationResult(&slot.record, id) { @@ -633,14 +782,41 @@ func (table *TimerRegistrationTable) RecycleTimerV2(p *P, handle TimerRegistrati slot.controller = 0 slot.control = nil slot.controlWord = 0 + // The exact retired slot is now a proven reusable generation. Publishing it + // as the next owner-local hint keeps sparse churn O(1) and preserves the + // useful one-slot reuse property without imposing a free-list ABI on pages. + table.reserveCursor = handle.Slot - 1 + if handle.Slot == table.scanLimit { + for table.scanLimit != 0 { + index := table.scanLimit - 1 + last, lastOK := timerRegistrationSlotAt(table, index) + if !lastOK { + return false + } + if last.state != timerRegistrationFree { + break + } + if !reusableTimerRegistrationSlot(last, table.route, index) { + return false + } + table.scanLimit-- + } + if table.scanLimit == 0 { + // An empty table should rebuild a compact active prefix on the next + // burst so due scans do not retain a high-water gap from prior use. + table.reserveCursor = 0 + } + } return true } func timerRegistrationTableEmpty(table *TimerRegistrationTable, owner *P) bool { - if table == nil || table.owner != owner { + capacity := TimerRegistrationConfiguredCapacity(table) + if table == nil || table.owner != owner || !validTimerRegistrationIndex(table, capacity) || + table.activeCount != 0 || table.controlledCount != 0 || table.minimumKnown || table.minimum != 0 { return false } - for index := uint32(0); index < TimerRegistrationConfiguredCapacity(table); index++ { + for index := uint32(0); index < capacity; index++ { slot, slotOK := timerRegistrationSlotAt(table, index) if !slotOK || !reusableTimerRegistrationSlot(slot, table.route, index) { return false @@ -656,6 +832,11 @@ func bindTimerRegistrationTableAtRoute(table *TimerRegistrationTable, p *P, rout } table.route = route table.scanLimit = 0 + table.reserveCursor = 0 + table.activeCount = 0 + table.controlledCount = 0 + table.minimum = 0 + table.minimumKnown = false table.owner = p return true } @@ -671,6 +852,11 @@ func unbindTimerRegistrationTable(table *TimerRegistrationTable, p *P) bool { } table.owner = nil table.scanLimit = 0 + table.reserveCursor = 0 + table.activeCount = 0 + table.controlledCount = 0 + table.minimum = 0 + table.minimumKnown = false return true } diff --git a/runtime/internal/coro/timer_registration_v2_test.go b/runtime/internal/coro/timer_registration_v2_test.go index 5515c9e28d..07d2985830 100644 --- a/runtime/internal/coro/timer_registration_v2_test.go +++ b/runtime/internal/coro/timer_registration_v2_test.go @@ -208,8 +208,12 @@ func TestTimerRegistrationV2FutureDeadlineOnlyPublishesWhenDue(t *testing.T) { sources, timers := bindTimerV2TestSources(t, p, nil) park := beginTimerV2TestPark(t, p, "timer-v2-future", 1, 103) handle, attached := timers.ReserveAndAttachTimerV2(p, &park.task.g.park, park.ticket, park.wait, 88, 50) - if !attached { - t.Fatal("reserve future timer V2") + deadline, hasDeadline, skip, fastOK := timerRegistrationFastDeadline(timers, p, 49) + if !attached || timers.activeCount != 1 || timers.controlledCount != 0 || + !timers.minimumKnown || timers.minimum != 50 || !fastOK || !skip || !hasDeadline || deadline != 50 { + t.Fatalf("reserve future timer V2 = (%+v,%t), index=(%d,%d,%d,%t), fast=(%d,%t,%t,%t)", + handle, attached, timers.activeCount, timers.controlledCount, timers.minimum, timers.minimumKnown, + deadline, hasDeadline, skip, fastOK) } commitTimerV2TestPark(t, p, park) @@ -222,6 +226,10 @@ func TestTimerRegistrationV2FutureDeadlineOnlyPublishesWhenDue(t *testing.T) { if scan, ok := sources.publishPass(p, 50, true); !ok || scan.timers != 1 || scan.hasDeadline { t.Fatalf("due future timer scan = (%+v, %t)", scan, ok) } + if _, hasDeadline, skip, fastOK := timerRegistrationFastDeadline(timers, p, 50); !fastOK || !skip || hasDeadline || timers.activeCount != 0 || timers.minimumKnown || timers.minimum != 0 { + t.Fatalf("delivered timer fast index = active:%d minimum:(%d,%t) fast:(%t,%t,%t)", + timers.activeCount, timers.minimum, timers.minimumKnown, hasDeadline, skip, fastOK) + } if promoted, visits, ok := sources.resolvePublishedEpoch(p); !ok || promoted != 1 || visits != 1 { t.Fatalf("due future timer resolve = (%d, %d, %t)", promoted, visits, ok) } @@ -253,6 +261,10 @@ func TestTimerRegistrationControlledV2OwnerScanObservesGenerationChange(t *testi if !attached || handle == (TimerRegistrationHandle{}) { t.Fatal("reserve generation-observed controlled timer V2") } + if _, _, skip, fastOK := timerRegistrationFastDeadline(timers, p, 0); !fastOK || skip || timers.activeCount != 1 || timers.controlledCount != 1 { + t.Fatalf("controlled timer was incorrectly fast-skippable: active=%d controlled=%d skip=(%t,%t)", + timers.activeCount, timers.controlledCount, skip, fastOK) + } commitTimerV2TestPark(t, p, park) // Stop/Reset publishes this word before requesting the route. The owner diff --git a/runtime/internal/coro/wait_set_record.go b/runtime/internal/coro/wait_set_record.go index 02d8340f16..493ac9c267 100644 --- a/runtime/internal/coro/wait_set_record.go +++ b/runtime/internal/coro/wait_set_record.go @@ -66,7 +66,10 @@ type WaitSetRecord struct { state waitSetRecordState work waitSetWorkState resumeKind resumeBindingKind - _ [1]byte + // directChannel certifies the compiler/runtime fused one-channel binding. + // It occupies the record's former tail padding and is cleared with the + // complete record at promotion, so the native/WASM frame ABI is unchanged. + directChannel bool } // PrepareWaitSetRecord binds zero caller storage to one preparing logical @@ -109,7 +112,7 @@ func validAffectedWaitQueueHeader(p *P) bool { func validActiveParkStateHeader(state *ParkState, ticket ParkTicket) bool { if state == nil || state.ticket != ticket || !validParkTicket(ticket) || - state.resolving || + state.resolving || state.directChannel || !validTaskCancelState(state.taskCancelKind, state.taskCancelPhase) || state.cancelKind > ParkCancelShutdown || state.attached > state.expected { return false @@ -157,7 +160,7 @@ func validActiveWaitSetRecordFast(p *P, record *WaitSetRecord) bool { record.g.queued || record.g.nextReady != nil || record.g.runP != nil || record.g.transferState != runnableTransferGIdle || record.g.active == nil || record.g.active.parkWait != record || !validActiveParkStateHeader(&record.g.park, record.ticket) || - !validWaitSetResumeBinding(record) { + !validTrustedWaitSetResumeBinding(record) { return false } if record.activePrev == nil { @@ -174,7 +177,8 @@ func validActiveWaitSetRecordFast(p *P, record *WaitSetRecord) bool { } func validActiveWaitSetRecord(p *P, record *WaitSetRecord) bool { - return validActiveWaitSetRecordFast(p, record) && validParkSetWaitingG(record.g) + return validActiveWaitSetRecordFast(p, record) && validWaitSetResumeBinding(record) && + validParkSetWaitingG(record.g) } // validParkWaitQueue is the allocation-free full audit retained for tests, @@ -299,6 +303,16 @@ func activateWaitSetRecord(p *P, g *G, record *WaitSetRecord) bool { !validParkState(&g.park) || g.park.phase != parkParked { return false } + activateWaitSetRecordUnchecked(p, g, record) + return true +} + +// activateWaitSetRecordUnchecked is the no-fail queue write half. Resumed may +// call it directly only when dispatchPending has just performed the complete +// direct-channel ParkState/record/cleanup audit in the same scheduler-owner +// activation; no callback or producer can mutate these owner-only queues in +// between. Other entry points retain activateWaitSetRecord's full validation. +func activateWaitSetRecordUnchecked(p *P, g *G, record *WaitSetRecord) { record.activePrev = p.parkWaitTail record.state = waitSetRecordActive g.waiting = true @@ -308,6 +322,12 @@ func activateWaitSetRecord(p *P, g *G, record *WaitSetRecord) bool { p.parkWaitTail.activeNext = record } p.parkWaitTail = record + if record.resumeKind == resumeBindingDirectChannel { + // The hchan node has its own exact completion inbox. Unlike a source-backed + // park it cannot have an unobserved catalog fact which requires an initial + // affected-set visit. + return + } // Every newly parked set receives one initial visit. This catches an owner // completion published during preparation without adding an always-live G @@ -315,7 +335,6 @@ func activateWaitSetRecord(p *P, g *G, record *WaitSetRecord) bool { // affected-queue and record-idle preconditions were checked before the // active-list mutation, so there is no fallible step after scheduler commit. appendAffectedWaitSetUnchecked(p, record) - return true } // legacyAffectedWaitSetsClaimlessCompatible audits the complete affected FIFO @@ -436,6 +455,17 @@ func promoteReadyWaitSet(sources *ExecutorSourceSet, p *P, record *WaitSetRecord record.g.park.phase == parkMaterialized && record.resumeKind != resumeBindingMaterialized { return false } + promoteReadyWaitSetUnchecked(p, record) + return true +} + +// promoteReadyWaitSetUnchecked is the no-fail queue transfer half. The direct +// channel completion path calls it only after its exact cursor, active-list, +// ParkState, frame, and ready-queue headers were audited in the same owner +// reduction. Generic and multi-source callers retain promoteReadyWaitSet. +func promoteReadyWaitSetUnchecked(p *P, record *WaitSetRecord) { + g := record.g + frame := g.active previous, next := record.activePrev, record.activeNext if previous == nil { p.parkWaitHead = next @@ -452,7 +482,6 @@ func promoteReadyWaitSet(sources *ExecutorSourceSet, p *P, record *WaitSetRecord g.state = GRunnable appendRunnableUnchecked(p, g) *record = WaitSetRecord{} - return true } // promoteResolvedWaitSets completes the post-source-apply half of one published diff --git a/runtime/internal/coro/worker_operation_source.go b/runtime/internal/coro/worker_operation_source.go index 261f826751..5f5dbf2f7f 100644 --- a/runtime/internal/coro/worker_operation_source.go +++ b/runtime/internal/coro/worker_operation_source.go @@ -101,6 +101,17 @@ type WorkerOperationSource struct { affectedTail uint32 } +// workerOperationReservation is an owner-stack capability for one reusable +// worker slot. It exists only inside a no-suspend park transaction; the +// backend receives the resulting OperationID, never this source or slot. +type workerOperationReservation struct { + source *WorkerOperationSource + owner *P + slot *workerOperationSlot + index uint32 + capacity uint32 +} + // WorkerOperationConfiguredCapacity returns the exact linear slot and scan // capacity. Linear one-based slot identities remain encodable in the frozen // 15-bit OperationID local field. @@ -193,22 +204,51 @@ func ConfigureWorkerOperationPages(source *WorkerOperationSource, pages []Worker return true } -// CanReserveWorkerOperation is the allocation-free source-capacity preflight. -// Physical queue capacity remains an independent target responsibility. -func CanReserveWorkerOperation(p *P, source *WorkerOperationSource) bool { - if !validWorkerOperationOwner(source, p) { - return false - } - for index := uint32(0); index < WorkerOperationConfiguredCapacity(source); index++ { +func (source *WorkerOperationSource) preflightWorkerReservationOwned( + p *P, +) (workerOperationReservation, bool) { + capacity := WorkerOperationConfiguredCapacity(source) + for index := uint32(0); index < capacity; index++ { slot, ok := workerOperationSlotAt(source, index) if !ok { - return false + return workerOperationReservation{}, false } if preemptLoad(&slot.generation) != ^uint32(0) && workerOperationReusableSlot(source, slot, index) { - return true + return workerOperationReservation{ + source: source, owner: p, slot: slot, index: index, capacity: capacity, + }, true } } - return false + return workerOperationReservation{}, false +} + +func validWorkerOperationReservationHeader( + source *WorkerOperationSource, + p *P, + reservation workerOperationReservation, +) bool { + return source != nil && p != nil && reservation.source == source && + reservation.owner == p && reservation.slot != nil && + reservation.capacity != 0 && reservation.index < reservation.capacity +} + +// preflightWorkerOperationReservation authenticates the owner and selects the +// exact reusable slot once. Physical queue capacity remains an independent +// target responsibility. +func preflightWorkerOperationReservation( + p *P, + source *WorkerOperationSource, +) (workerOperationReservation, bool) { + if !validWorkerOperationOwner(source, p) { + return workerOperationReservation{}, false + } + return source.preflightWorkerReservationOwned(p) +} + +// CanReserveWorkerOperation is the allocation-free source-capacity preflight. +func CanReserveWorkerOperation(p *P, source *WorkerOperationSource) bool { + _, ok := preflightWorkerOperationReservation(p, source) + return ok } // AttachWorkerOperationPage publishes one pristine stable page from the owner @@ -287,51 +327,60 @@ func validWorkerOperationLiveSlot(source *WorkerOperationSource, p *P, index uin return ok && slot.record.Matches(id) } -func (source *WorkerOperationSource) reserveAndAttach( - p *P, +func (source *WorkerOperationSource) reserveAndAttachWorkerSlot( state *ParkState, ticket ParkTicket, wait *WaitSetRecord, caseID uint32, + reservation workerOperationReservation, ) (OperationID, bool) { - if !validWorkerOperationOwner(source, p) { + slot, index := reservation.slot, reservation.index + if slot == nil || index >= reservation.capacity || + reservation.capacity != WorkerOperationConfiguredCapacity(source) { return OperationID{}, false } - for index := uint32(0); index < WorkerOperationConfiguredCapacity(source); index++ { - slot, slotOK := workerOperationSlotAt(source, index) - if !slotOK || !workerOperationReusableSlot(source, slot, index) || preemptLoad(&slot.generation) == ^uint32(0) { - continue - } - generation, begun := beginProducerSourceSlot(&slot.producerSourceSlot) - if !begun { - return OperationID{}, false - } - if !raiseSourceScanLimit(&source.scanLimit, index, WorkerOperationConfiguredCapacity(source)) { - return OperationID{}, false - } - id, ok := MakeOperationIDAtRoute(OperationSourceWorker, source.route, index+1, generation) - if !ok || !PrepareOperationAtGeneration(&slot.record, id) { - return OperationID{}, false - } - attached := false - if wait == nil { - attached = AttachParkOperation(state, ticket, &slot.record, caseID) - } else { - attached = AttachParkWaitOperation(state, ticket, wait, &slot.record, caseID) - } - if !attached { - if !AbortReservedOperation(&slot.record, id) || - !resetProducerSourceSlot(&slot.producerSourceSlot, generation) { - return OperationID{}, false - } - return OperationID{}, false - } - if !activateProducerSourceSlot(&slot.producerSourceSlot, generation) { + generation, begun := beginProducerSourceSlot(&slot.producerSourceSlot) + if !begun { + return OperationID{}, false + } + if !raiseSourceScanLimit(&source.scanLimit, index, reservation.capacity) { + return OperationID{}, false + } + id, ok := MakeOperationIDAtRoute(OperationSourceWorker, source.route, index+1, generation) + if !ok || !PrepareOperationAtGeneration(&slot.record, id) { + return OperationID{}, false + } + attached := false + if wait == nil { + attached = AttachParkOperation(state, ticket, &slot.record, caseID) + } else { + attached = AttachParkWaitOperation(state, ticket, wait, &slot.record, caseID) + } + if !attached { + if !AbortReservedOperation(&slot.record, id) || + !resetProducerSourceSlot(&slot.producerSourceSlot, generation) { return OperationID{}, false } - return id, true + return OperationID{}, false + } + if !activateProducerSourceSlot(&slot.producerSourceSlot, generation) { + return OperationID{}, false + } + return id, true +} + +func (source *WorkerOperationSource) reserveAndAttach( + p *P, + state *ParkState, + ticket ParkTicket, + wait *WaitSetRecord, + caseID uint32, +) (OperationID, bool) { + reservation, ok := preflightWorkerOperationReservation(p, source) + if !ok { + return OperationID{}, false } - return OperationID{}, false + return source.reserveAndAttachWorkerSlot(state, ticket, wait, caseID, reservation) } func (source *WorkerOperationSource) ReserveAndAttach( @@ -370,6 +419,30 @@ func (source *WorkerOperationSource) MarkSubmitted(p *P, id OperationID) bool { return true } +// markWorkerReservationSubmitted closes the compiler-owned worker transaction +// against the slot it just reserved. The owner/source relation and complete +// ParkLink graph were already audited before this no-suspend suffix. +func (source *WorkerOperationSource) markWorkerReservationSubmitted( + p *P, + reservation workerOperationReservation, + id OperationID, +) bool { + if !validWorkerOperationReservationHeader(source, p, reservation) || + id.LocalSlot() != reservation.index+1 || id.Route() != source.route || + id.Source() != OperationSourceWorker { + return false + } + slot := reservation.slot + if slot.submitted || preemptLoad(&slot.generation) != id.Generation || + preemptLoad(&slot.state) != uint32(producerSourceActive) || + preemptLoad(&slot.mailbox) != uint32(workerOperationMailboxEmpty) || + !slot.record.Matches(id) || slot.record.phase != operationActive { + return false + } + slot.submitted = true + return true +} + // Post publishes only the first exact-generation result. Later producers are // coalesced and cannot replace its scalar payload. func (source *WorkerOperationSource) Post(id OperationID, payload ScalarResultPayloadV1) WorkerOperationPostResult { @@ -419,6 +492,43 @@ func (source *WorkerOperationSource) Pending() bool { return source != nil && routedProducerPending(&source.routedProducerSource) } +// submittedCompletionState is the owner-side observation used by a physical +// executor immediately before it arms a retained wait. awaiting reports that +// at least one backend owns an exact submitted generation whose completion is +// not yet durable; ready reports the source's durable completion hint. The +// observation never changes source state and never waits for a producer. +func (source *WorkerOperationSource) submittedCompletionState(p *P) (awaiting, ready, ok bool) { + if !validWorkerOperationOwner(source, p) { + return false, false, false + } + ready = source.Pending() + limit, valid := workerOperationScanLimit(source) + if !valid { + return false, false, false + } + for index := uint32(0); index < limit; index++ { + slot, found := workerOperationSlotAt(source, index) + if !found { + return false, false, false + } + if !slot.submitted { + continue + } + if !validWorkerOperationLiveSlot(source, p, index) { + return false, false, false + } + switch workerOperationMailbox(preemptLoad(&slot.mailbox)) { + case workerOperationMailboxEmpty, workerOperationMailboxPosting: + awaiting = true + case workerOperationMailboxPosted, workerOperationMailboxDraining, + workerOperationMailboxDelivered: + default: + return false, false, false + } + } + return awaiting, ready, true +} + func (source *WorkerOperationSource) RequestCancel(p *P, wait *WaitSetRecord) bool { return validWorkerOperationOwner(source, p) && RequestWaitSetCancel(p, wait, ParkCancelOperation) } diff --git a/runtime/internal/coro/worker_operation_source_test.go b/runtime/internal/coro/worker_operation_source_test.go index 4024d075ba..3f9671db4c 100644 --- a/runtime/internal/coro/worker_operation_source_test.go +++ b/runtime/internal/coro/worker_operation_source_test.go @@ -457,6 +457,41 @@ func TestWorkerOperationSourceSubmittedCancellationAwaitsPhysicalCompletion(t *t } } +func TestWorkerOperationSubmittedCompletionState(t *testing.T) { + p := new(P) + source := new(WorkerOperationSource) + if !BindWorkerOperationSource(source, p) { + t.Fatal("bind completion-state worker source") + } + state, ticket, ids := reserveWorkerWaitSet(t, source, p, 74, []uint32{29}) + id := ids[0] + if awaiting, ready, ok := source.submittedCompletionState(p); !ok || awaiting || ready { + t.Fatalf("reserved completion state = (%t, %t, %t)", awaiting, ready, ok) + } + if !source.MarkSubmitted(p, id) { + t.Fatal("mark completion-state worker submitted") + } + if awaiting, ready, ok := source.submittedCompletionState(p); !ok || !awaiting || ready { + t.Fatalf("submitted completion state = (%t, %t, %t)", awaiting, ready, ok) + } + payload := workerPayloadForTest(t, 11, 1100) + if source.Post(id, payload) != WorkerOperationPosted { + t.Fatal("post completion-state worker result") + } + if awaiting, ready, ok := source.submittedCompletionState(p); !ok || awaiting || !ready { + t.Fatalf("posted completion state = (%t, %t, %t)", awaiting, ready, ok) + } + if published, lost, ok := source.PublishPass(p); !ok || published != 1 || lost != 0 { + t.Fatalf("publish completion-state worker = (%d, %d, %t)", published, lost, ok) + } + if awaiting, ready, ok := source.submittedCompletionState(p); !ok || awaiting || ready { + t.Fatalf("delivered completion state = (%t, %t, %t)", awaiting, ready, ok) + } + if !RequestParkCancel(state, ticket, ParkCancelOperation) { + t.Fatal("cancel completion-state worker park") + } +} + func TestWorkerOperationSourceProducerPrefixIsAlignedPOD(t *testing.T) { if unsafe.Offsetof(workerOperationSlot{}.producerSourceSlot) != 0 || unsafe.Offsetof(workerOperationSlot{}.state)%4 != 0 || diff --git a/runtime/internal/coro/worker_park_owner.go b/runtime/internal/coro/worker_park_owner.go index 828b53b60a..cdb9bd9d91 100644 --- a/runtime/internal/coro/worker_park_owner.go +++ b/runtime/internal/coro/worker_park_owner.go @@ -113,6 +113,71 @@ func PrepareCurrentExecutorWorkerPark( return PrepareSingleWorkerPark(g, handle, header, source, wait, caseID, seed) } +// PrepareCurrentExecutorWorkerParkCompiler fuses the compiler-generated +// one-worker transaction through packet binding and backend-submission commit. +// The runtime has already resolved driver with CurrentExecutorWorkerDriver and +// reserved its bounded physical queue in the same no-suspend interval. +// +// Ordinary operation builds and audits one exact ParkLink, selects one source +// slot, and publishes one packet without re-entering the generic N-way park +// layers. A pending task stop retains the complete cancellation-aware path. +func PrepareCurrentExecutorWorkerParkCompiler( + driver *ExecutorDriver, + g *G, + handle unsafe.Pointer, + header *HeaderV1, + wait *WaitSetRecord, + packet *ResumePacket, + caseID uint32, + seed uint32, +) (ParkTicket, OperationID, bool) { + if driver == nil || g == nil || packet == nil || *packet != (ResumePacket{}) || + driver.magic != executorDriverMagic || driver.state != executorDriverActive || + driver.p == nil || driver.p != g.runP || driver.p.executor != driver || + driver.p.current != g || driver.sources.worker == nil || caseID == 0 { + return ParkTicket{}, OperationID{}, false + } + p, source := driver.p, driver.sources.worker + if !validWorkerOperationOwner(source, p) || source.route != driver.route { + return ParkTicket{}, OperationID{}, false + } + + // CommitParkSet must translate a stop which arrived before this hook into + // logical cancellation, so that uncommon shape deliberately keeps every + // generic proof. The packet and submission still become one ABI result. + if g.park.taskCancelKind != TaskCancelNone || g.park.taskCancelPhase != taskCancelIdle { + ticket, id, ok := PrepareSingleWorkerPark( + g, handle, header, source, wait, caseID, seed, + ) + if !ok || !BindSingleWaitSetResumePacket(wait, packet, id) || + !source.MarkSubmitted(p, id) { + return ParkTicket{}, OperationID{}, false + } + return ticket, id, true + } + + prepared, ok := preflightSingleParkPreparation(g, handle, header, wait, seed) + if !ok || prepared.p != p { + return ParkTicket{}, OperationID{}, false + } + reservation, ok := source.preflightWorkerReservationOwned(p) + if !ok { + return ParkTicket{}, OperationID{}, false + } + prepared.begin() + id, ok := source.reserveAndAttachWorkerSlot( + &g.park, prepared.ticket, wait, caseID, reservation, + ) + if !ok || !prepared.commit(id, caseID) { + return ParkTicket{}, OperationID{}, false + } + installSingleWaitSetResumePacket(wait, packet, id) + if !source.markWorkerReservationSubmitted(p, reservation, id) { + return ParkTicket{}, OperationID{}, false + } + return prepared.ticket, id, true +} + // PrepareCurrentExecutorWorkerTimerPark installs one external operation and, // when deadline is non-zero, one absolute timer in the same ParkSet. A zero // deadline keeps the operation controllable without manufacturing an infinite diff --git a/runtime/internal/coro/worker_park_owner_test.go b/runtime/internal/coro/worker_park_owner_test.go index 0ee2a1d88b..9d3c954b3f 100644 --- a/runtime/internal/coro/worker_park_owner_test.go +++ b/runtime/internal/coro/worker_park_owner_test.go @@ -36,25 +36,36 @@ func TestWorkerParkOwnerPrepareCompleteAndFinish(t *testing.T) { } action := beginWaitTestResume(t, p, task) var wait WaitSetRecord + var packet ResumePacket task.frame.header.SuspendReason = uint16(SuspendPark) task.frame.header.Lifecycle = uint16(FrameSuspended) current, currentHandle, currentRoute, currentOK := CurrentExecutorWorkerDriver(task.g) if !currentOK || current != driver || currentHandle != executor || currentRoute != RouteID(1) { t.Fatalf("resolve current worker owner = (%p, %+v, %d, %t)", current, currentHandle, currentRoute, currentOK) } - ticket, id, ok := PrepareCurrentExecutorWorkerPark( - driver, task.g, task.handle, task.frame.header, &wait, 31, 79, + ticket, id, ok := PrepareCurrentExecutorWorkerParkCompiler( + driver, task.g, task.handle, task.frame.header, &wait, &packet, 31, 79, ) - if !ok || !CommitCurrentExecutorWorkerSubmission(driver, task.g, id) { - t.Fatal("prepare worker owner park") + if !ok { + t.Fatal("prepare compiler worker owner park") } if action, ok = Resumed(p, task.g, action); !ok || action.Kind != ActionPark { t.Fatalf("commit worker owner park = (%+v, %t)", action, ok) } + probe, awaiting, ready, probeOK := PrepareExecutorWorkerCompletionProbe(driver) + if !probeOK || !awaiting || ready || !probe.Valid() || probe.Ready() { + t.Fatalf("prepare incomplete worker probe = (%+v, %t, %t, %t)", probe, awaiting, ready, probeOK) + } payload := workerPayloadForTest(t, 11, 111, 222, 0) if workers.Post(id, payload) != WorkerOperationPosted { t.Fatal("post worker owner result") } + if !probe.Ready() { + t.Fatal("worker completion probe missed durable publication") + } + if next, awaiting, ready, ok := PrepareExecutorWorkerCompletionProbe(driver); !ok || awaiting || !ready || !next.Valid() || !next.Ready() { + t.Fatalf("prepare ready worker probe = (%+v, %t, %t, %t)", next, awaiting, ready, ok) + } var complete ExecutorPollProgress for entries := 0; entries < 1000; entries++ { progress, ok := PollExecutorSlice(driver, 1) @@ -76,13 +87,15 @@ func TestWorkerParkOwnerPrepareCompleteAndFinish(t *testing.T) { action = beginWaitTestResume(t, p, task) task.frame.header.SuspendReason = uint16(SuspendPark) task.frame.header.Lifecycle = uint16(FrameSuspended) - outcome, caseID, lease, taskCancel, ok := TakeRunDecision(task.g, ticket) - if !ok || outcome != ParkOutcomeCompleted || caseID != 31 || !lease.Valid() || taskCancel != TaskCancelNone { - t.Fatalf("take worker owner decision = (%d, %d, %+v, %d, %t)", outcome, caseID, lease, taskCancel, ok) - } var got ScalarResultPayloadV1 - if result := FinishCurrentExecutorWorkerPark(driver, task.g, id, lease, false, &got); !result.Finished() || got != payload { - t.Fatalf("finish worker owner result = %+v, want %+v", got, payload) + outcome, caseID, taskCancel, result, small, ok := TakeResumePacket( + task.g, ticket, &packet, &got, + ) + if !ok || outcome != ParkOutcomeCompleted || caseID != 31 || + taskCancel != TaskCancelNone || result != ResumeResultScalar || + small != ResumeSmallInvalid || got != payload { + t.Fatalf("take compiler worker packet = (%d, %d, %d, %d, %d, %+v, %t)", + outcome, caseID, taskCancel, result, small, got, ok) } task.frame.header.SuspendReason = uint16(SuspendFrameComplete) task.frame.header.Lifecycle = uint16(FrameFinalSuspended) diff --git a/runtime/internal/coroalloc/allocator.go b/runtime/internal/coroalloc/allocator.go index bfed637fff..91ff6e4142 100644 --- a/runtime/internal/coroalloc/allocator.go +++ b/runtime/internal/coroalloc/allocator.go @@ -84,10 +84,12 @@ func Ready() bool { return state == bootstrapReady } -// AllocFrame allocates one explicitly owned, GC-visible coroutine frame -// range. A caller cannot accidentally rely on a backend's implicit lazy init. +// AllocFrame allocates one zero-filled, explicitly owned, GC-visible +// coroutine frame range. Zero initialization is a backend contract: BDWGC and +// tinygogc already clear scanned allocations, while libc profiles use calloc. +// A caller cannot accidentally rely on a backend's implicit lazy init. func AllocFrame(size uintptr) unsafe.Pointer { - if !Ready() || size == 0 { + if !backendAllocationsAreZeroed || !Ready() || size == 0 { return nil } return backendAllocFrame(size) @@ -103,14 +105,14 @@ func FreeFrame(ptr unsafe.Pointer, size uintptr) bool { return backendFreeFrame(ptr, size) } -// AllocTask allocates pointer-containing scheduler task storage. It uses the -// same statically selected scanned/root backend as coroutine frames: BDWGC's +// AllocTask allocates zero-filled, pointer-containing scheduler task storage. +// It uses the same statically selected scanned/root backend as coroutine frames: BDWGC's // uncollectable allocation is conservatively scanned, tinygogc sees the task // through the scheduler's static P/parent links, and leaking WebAssembly // profiles have no tracing collector that an ordinary malloc range could hide // pointers from. func AllocTask(size uintptr) unsafe.Pointer { - if !Ready() || size == 0 { + if !backendAllocationsAreZeroed || !Ready() || size == 0 { return nil } return backendAllocFrame(size) diff --git a/runtime/internal/coroalloc/allocator_test.go b/runtime/internal/coroalloc/allocator_test.go index 279603dc35..92cf827362 100644 --- a/runtime/internal/coroalloc/allocator_test.go +++ b/runtime/internal/coroalloc/allocator_test.go @@ -74,6 +74,9 @@ func TestBootstrapStateFailsClosed(t *testing.T) { } func TestSelectedBackendKindIsKnown(t *testing.T) { + if !backendAllocationsAreZeroed { + t.Fatal("selected coroutine allocator does not guarantee zero-filled storage") + } switch backendKind { case "bdwgc", "malloc", "tinygogc": default: diff --git a/runtime/internal/coroalloc/backend_gc.go b/runtime/internal/coroalloc/backend_gc.go index 98d29db259..59d28de64c 100644 --- a/runtime/internal/coroalloc/backend_gc.go +++ b/runtime/internal/coroalloc/backend_gc.go @@ -25,6 +25,7 @@ import ( ) const backendKind = "bdwgc" +const backendAllocationsAreZeroed = true func backendBootstrap() bool { bdwgc.Init() diff --git a/runtime/internal/coroalloc/backend_nogc.go b/runtime/internal/coroalloc/backend_nogc.go index 1f6e4b022d..988ab478b3 100644 --- a/runtime/internal/coroalloc/backend_nogc.go +++ b/runtime/internal/coroalloc/backend_nogc.go @@ -25,13 +25,14 @@ import ( ) const backendKind = "malloc" +const backendAllocationsAreZeroed = true func backendBootstrap() bool { return true } func backendAllocFrame(size uintptr) unsafe.Pointer { - return c.Malloc(size) + return c.Calloc(1, size) } func backendFreeFrame(ptr unsafe.Pointer, size uintptr) bool { diff --git a/runtime/internal/coroalloc/backend_tinygogc.go b/runtime/internal/coroalloc/backend_tinygogc.go index d8feaf07b2..549c56509b 100644 --- a/runtime/internal/coroalloc/backend_tinygogc.go +++ b/runtime/internal/coroalloc/backend_tinygogc.go @@ -25,6 +25,7 @@ import ( ) const backendKind = "tinygogc" +const backendAllocationsAreZeroed = true func backendBootstrap() bool { tinygogc.Init() diff --git a/runtime/internal/coroalloc/backend_webassembly.go b/runtime/internal/coroalloc/backend_webassembly.go index abf28542c6..f1da7f985e 100644 --- a/runtime/internal/coroalloc/backend_webassembly.go +++ b/runtime/internal/coroalloc/backend_webassembly.go @@ -29,13 +29,14 @@ import ( // named targets currently pair this path with nogc; the !nogc form remains // useful to compile the runtime package with the host Go toolchain. const backendKind = "malloc" +const backendAllocationsAreZeroed = true func backendBootstrap() bool { return true } func backendAllocFrame(size uintptr) unsafe.Pointer { - return c.Malloc(size) + return c.Calloc(1, size) } func backendFreeFrame(ptr unsafe.Pointer, size uintptr) bool { diff --git a/runtime/internal/coroalloc/backend_webassembly_test.go b/runtime/internal/coroalloc/backend_webassembly_test.go index 0d8cb74745..dbd9a60e7c 100644 --- a/runtime/internal/coroalloc/backend_webassembly_test.go +++ b/runtime/internal/coroalloc/backend_webassembly_test.go @@ -35,6 +35,11 @@ func TestWasmBackendAllocatesAndFreesWithLibc(t *testing.T) { if ptr == nil { t.Fatal("wasm malloc frame allocation returned nil") } + for offset := uintptr(0); offset < size; offset++ { + if got := *(*byte)(unsafe.Add(ptr, offset)); got != 0 { + t.Fatalf("wasm calloc frame byte %d = %d, want zero", offset, got) + } + } for offset := uintptr(0); offset < size; offset++ { *(*byte)(unsafe.Add(ptr, offset)) = byte(offset + 1) } diff --git a/runtime/internal/corodoorbell/pipe_llgo.go b/runtime/internal/corodoorbell/pipe_llgo.go index 69a92acfcb..17dbd57d36 100644 --- a/runtime/internal/corodoorbell/pipe_llgo.go +++ b/runtime/internal/corodoorbell/pipe_llgo.go @@ -57,8 +57,11 @@ func nativeCDoorbellRead(fd int32, buffer *byte, size uintptr) uint64 // nativeCDoorbellWrite performs exactly one one-byte write to the private // nonblocking write end. EAGAIN is returned to the retained-wake protocol. -// Its exact raw-host use domain is frozen before physical lowering. +// The descriptor capability is established by nativeCDoorbellOpen before the +// Pipe is published, while this declaration supplies the one fact unavailable +// from its C type: the exact call cannot block an executor. // +//llgo:coro contract foreign.v1 scope=declaration progress=executor-safe affinity=any-thread reentry=none memory=borrow-until-return //go:linkname nativeCDoorbellWrite C.__llgo_coro_doorbell_write_v1 func nativeCDoorbellWrite(fd int32, buffer *byte, size uintptr) uint64 diff --git a/runtime/internal/coroworker/_worker/worker.c b/runtime/internal/coroworker/_worker/worker.c index b7d5193e93..f9d972ae31 100644 --- a/runtime/internal/coroworker/_worker/worker.c +++ b/runtime/internal/coroworker/_worker/worker.c @@ -140,6 +140,8 @@ struct llgo_coro_worker_queue_v1 { _Atomic uint32_t producer_state; _Atomic size_t enqueue_position; _Atomic size_t dequeue_position; + /* Exactly one between-job worker may advertise a kernel-free handoff. */ + _Atomic bool handoff_poller; llgo_coro_worker_wake_v1 wake; struct llgo_coro_worker_queue_slot_v1 slots[LLGO_CORO_WORKER_QUEUE_CAPACITY_V1]; }; @@ -158,6 +160,22 @@ extern uint32_t __llgo_coro_native_worker_complete_v1( static void *llgo_coro_worker_main_v1(void *unused); +#if defined(__aarch64__) || defined(__arm__) +#define LLGO_CORO_WORKER_HANDOFF_POLLS_V1 UINT32_C(262144) +#else +#define LLGO_CORO_WORKER_HANDOFF_POLLS_V1 UINT32_C(32768) +#endif + +static inline void llgo_coro_worker_cpu_relax_v1(void) { +#if defined(__aarch64__) || defined(__arm__) + __asm__ volatile("yield" ::: "memory"); +#elif defined(__x86_64__) || defined(__i386__) + __asm__ volatile("pause" ::: "memory"); +#else + atomic_signal_fence(memory_order_seq_cst); +#endif +} + static int llgo_coro_worker_thread_create_v1( pthread_t *thread, void *(*routine)(void *)) { @@ -294,10 +312,17 @@ static bool llgo_coro_worker_queue_publish_reserved_v1( slot->job = *job; atomic_store_explicit(&slot->sequence, reservation + 1, memory_order_release); /* - * Keep producer admission through wake publication. Stop therefore cannot - * seal between the durable cell and its matching semaphore token. + * Only a successful exchange of the exact handoff token may suppress the + * wake. A poller clears that token before sleeping and rechecks after a + * producer claims it, so no stale population count can lose a wake. + * Backlog always emits another token and retains full pool parallelism. + * Keep producer admission through this decision: Stop cannot seal between + * the durable cell and its required wake. */ - if (!llgo_coro_worker_wake_signal_v1(&queue->wake) || + size_t dequeue = atomic_load_explicit(&queue->dequeue_position, memory_order_acquire); + bool direct_handoff = reservation == dequeue && + atomic_exchange_explicit(&queue->handoff_poller, false, memory_order_acq_rel); + if ((!direct_handoff && !llgo_coro_worker_wake_signal_v1(&queue->wake)) || !llgo_coro_worker_queue_leave_producer_v1(queue)) { return false; } @@ -309,7 +334,8 @@ bool __llgo_coro_worker_queue_can_release_v1(void) { return !atomic_load_explicit(&queue->initialized, memory_order_acquire) && atomic_load_explicit(&queue->producer_state, memory_order_relaxed) == 0 && atomic_load_explicit(&queue->enqueue_position, memory_order_relaxed) == 0 && - atomic_load_explicit(&queue->dequeue_position, memory_order_relaxed) == 0; + atomic_load_explicit(&queue->dequeue_position, memory_order_relaxed) == 0 && + !atomic_load_explicit(&queue->handoff_poller, memory_order_relaxed); } bool __llgo_coro_worker_queue_init_v1(void) { @@ -321,6 +347,7 @@ bool __llgo_coro_worker_queue_init_v1(void) { atomic_init(&queue->producer_state, 0); atomic_init(&queue->enqueue_position, 0); atomic_init(&queue->dequeue_position, 0); + atomic_init(&queue->handoff_poller, false); for (size_t index = 0; index < LLGO_CORO_WORKER_QUEUE_CAPACITY_V1; ++index) { atomic_init(&queue->slots[index].sequence, index); memset(&queue->slots[index].job, 0, sizeof(queue->slots[index].job)); @@ -331,6 +358,7 @@ bool __llgo_coro_worker_queue_init_v1(void) { !atomic_is_lock_free(&queue->producer_state) || !atomic_is_lock_free(&queue->enqueue_position) || !atomic_is_lock_free(&queue->dequeue_position) || + !atomic_is_lock_free(&queue->handoff_poller) || !atomic_is_lock_free(&queue->slots[0].sequence) || !llgo_coro_worker_wake_init_v1(&queue->wake)) { return false; @@ -339,10 +367,10 @@ bool __llgo_coro_worker_queue_init_v1(void) { return true; } -bool __llgo_coro_worker_queue_reserve_v1(size_t *reservation) { +size_t __llgo_coro_worker_queue_reserve_v2(void) { struct llgo_coro_worker_queue_v1 *queue = &llgo_coro_worker_queue_v1; - if (reservation == NULL || !llgo_coro_worker_queue_enter_producer_v1(queue)) { - return false; + if (!llgo_coro_worker_queue_enter_producer_v1(queue)) { + return 0; } size_t position = atomic_load_explicit(&queue->enqueue_position, memory_order_relaxed); @@ -350,7 +378,7 @@ bool __llgo_coro_worker_queue_reserve_v1(size_t *reservation) { /* Keep all sequence arithmetic defined instead of depending on wrap. */ if (position > SIZE_MAX - LLGO_CORO_WORKER_QUEUE_CAPACITY_V1) { (void)llgo_coro_worker_queue_leave_producer_v1(queue); - return false; + return 0; } struct llgo_coro_worker_queue_slot_v1 *slot = &queue->slots[position & (LLGO_CORO_WORKER_QUEUE_CAPACITY_V1 - 1)]; @@ -360,35 +388,128 @@ bool __llgo_coro_worker_queue_reserve_v1(size_t *reservation) { if (atomic_compare_exchange_weak_explicit( &queue->enqueue_position, &expected, position + 1, memory_order_relaxed, memory_order_relaxed)) { - *reservation = position; - return true; + /* Zero is the failure sentinel; the opaque public token is + * decoded by cancel/submit before sequence arithmetic. */ + return position + 1; } position = expected; continue; } if (sequence < position) { (void)llgo_coro_worker_queue_leave_producer_v1(queue); - return false; + return 0; } position = atomic_load_explicit(&queue->enqueue_position, memory_order_relaxed); } } -bool __llgo_coro_worker_queue_cancel_reservation_v1(size_t reservation) { +static bool llgo_coro_worker_queue_decode_reservation_v2( + size_t reservation, + size_t *position) { + if (reservation == 0 || position == NULL) { + return false; + } + *position = reservation - 1; + return true; +} + +bool __llgo_coro_worker_queue_cancel_reservation_v2(size_t reservation) { struct llgo_coro_worker_queue_v1 *queue = &llgo_coro_worker_queue_v1; + size_t position; + if (!llgo_coro_worker_queue_decode_reservation_v2(reservation, &position)) { + return false; + } struct llgo_coro_worker_job_v1 canceled; memset(&canceled, 0, sizeof(canceled)); - return llgo_coro_worker_queue_publish_reserved_v1(queue, reservation, &canceled); + return llgo_coro_worker_queue_publish_reserved_v1(queue, position, &canceled); } -bool __llgo_coro_worker_queue_submit_reserved_v1( +bool __llgo_coro_worker_queue_submit_reserved_v4( size_t reservation, - const struct llgo_coro_worker_job_v1 *job) { + uint32_t source_slot, + uint32_t generation, + uintptr_t function, + uintptr_t trace_target, + uint32_t argc, + uintptr_t a0, + uintptr_t a1, + uintptr_t a2, + uintptr_t a3, + uintptr_t a4, + uintptr_t a5, + uintptr_t a6, + uintptr_t a7, + uintptr_t a8) { struct llgo_coro_worker_queue_v1 *queue = &llgo_coro_worker_queue_v1; - if (!llgo_coro_worker_job_valid_v1(job)) { + size_t position; + if (!llgo_coro_worker_queue_decode_reservation_v2(reservation, &position)) { + return false; + } + struct llgo_coro_worker_job_v1 job; + memset(&job, 0, sizeof(job)); + job.source_slot = source_slot; + job.generation = generation; + job.function = function; + job.trace_target = trace_target; + job.argc = argc; + job.args[0] = a0; + job.args[1] = a1; + job.args[2] = a2; + job.args[3] = a3; + job.args[4] = a4; + job.args[5] = a5; + job.args[6] = a6; + job.args[7] = a7; + job.args[8] = a8; + if (!llgo_coro_worker_job_valid_v1(&job)) { return false; } - return llgo_coro_worker_queue_publish_reserved_v1(queue, reservation, job); + return llgo_coro_worker_queue_publish_reserved_v1(queue, position, &job); +} + +enum { + LLGO_CORO_WORKER_QUEUE_TAKE_EMPTY_V1 = 3, + LLGO_CORO_WORKER_QUEUE_TAKE_CANCELED_V1 = 4, +}; + +static uint32_t llgo_coro_worker_queue_try_take_v1( + struct llgo_coro_worker_queue_v1 *queue, + struct llgo_coro_worker_job_v1 *job) { + for (;;) { + size_t position = atomic_load_explicit(&queue->dequeue_position, memory_order_relaxed); + struct llgo_coro_worker_queue_slot_v1 *slot = + &queue->slots[position & (LLGO_CORO_WORKER_QUEUE_CAPACITY_V1 - 1)]; + size_t sequence = atomic_load_explicit(&slot->sequence, memory_order_acquire); + if (sequence == position + 1) { + if (!atomic_compare_exchange_weak_explicit( + &queue->dequeue_position, &position, position + 1, + memory_order_relaxed, memory_order_relaxed)) { + continue; + } + *job = slot->job; + memset(&slot->job, 0, sizeof(slot->job)); + atomic_store_explicit( + &slot->sequence, + position + LLGO_CORO_WORKER_QUEUE_CAPACITY_V1, + memory_order_release); + if (llgo_coro_worker_job_valid_v1(job)) { + return LLGO_CORO_WORKER_QUEUE_TAKE_JOB_V1; + } + return llgo_coro_worker_job_canceled_v1(job) ? + LLGO_CORO_WORKER_QUEUE_TAKE_CANCELED_V1 : + LLGO_CORO_WORKER_QUEUE_TAKE_INVALID_V1; + } + if (position != atomic_load_explicit( + &queue->dequeue_position, memory_order_relaxed)) { + continue; + } + if (llgo_coro_worker_queue_stopping_v1(queue) && + position == atomic_load_explicit(&queue->enqueue_position, memory_order_acquire)) { + return LLGO_CORO_WORKER_QUEUE_TAKE_STOP_V1; + } + /* Empty also covers an earlier reservation which is not published yet. */ + return LLGO_CORO_WORKER_QUEUE_TAKE_EMPTY_V1; + } } uint32_t __llgo_coro_worker_queue_wait_take_v1( @@ -399,44 +520,47 @@ uint32_t __llgo_coro_worker_queue_wait_take_v1( return LLGO_CORO_WORKER_QUEUE_TAKE_INVALID_V1; } for (;;) { - if (!llgo_coro_worker_wake_wait_v1(&queue->wake)) { - return LLGO_CORO_WORKER_QUEUE_TAKE_INVALID_V1; + /* A consumed semaphore token always earns one queue/Stop observation. */ + uint32_t status = llgo_coro_worker_queue_try_take_v1(queue, job); + bool expected = false; + if (status == LLGO_CORO_WORKER_QUEUE_TAKE_EMPTY_V1 && + atomic_compare_exchange_strong_explicit( + &queue->handoff_poller, &expected, true, + memory_order_acq_rel, memory_order_acquire)) { + /* + * Only this one idle worker owns the exact next dequeue cell. + * A producer may consume the token only while publishing that + * cell, so polling avoids a kernel wake without weakening queue + * correctness. The bounded architecture-specific budget is only + * a latency/power tradeoff; expiry falls back to the semaphore. + */ + size_t handoff_position = atomic_load_explicit( + &queue->dequeue_position, memory_order_relaxed); + struct llgo_coro_worker_queue_slot_v1 *handoff_slot = + &queue->slots[handoff_position & + (LLGO_CORO_WORKER_QUEUE_CAPACITY_V1 - 1)]; + for (uint32_t attempt = 0; + attempt < LLGO_CORO_WORKER_HANDOFF_POLLS_V1 && + atomic_load_explicit( + &handoff_slot->sequence, memory_order_acquire) != + handoff_position + 1 && + atomic_load_explicit( + &queue->handoff_poller, memory_order_acquire); + ++attempt) { + llgo_coro_worker_cpu_relax_v1(); + } + (void)atomic_exchange_explicit( + &queue->handoff_poller, false, memory_order_acq_rel); + status = llgo_coro_worker_queue_try_take_v1(queue, job); } - memset(job, 0, sizeof(*job)); - - size_t position = atomic_load_explicit(&queue->dequeue_position, memory_order_relaxed); - for (;;) { - struct llgo_coro_worker_queue_slot_v1 *slot = - &queue->slots[position & (LLGO_CORO_WORKER_QUEUE_CAPACITY_V1 - 1)]; - size_t sequence = atomic_load_explicit(&slot->sequence, memory_order_acquire); - if (sequence == position + 1) { - if (atomic_compare_exchange_weak_explicit( - &queue->dequeue_position, &position, position + 1, - memory_order_relaxed, memory_order_relaxed)) { - *job = slot->job; - memset(&slot->job, 0, sizeof(slot->job)); - atomic_store_explicit( - &slot->sequence, - position + LLGO_CORO_WORKER_QUEUE_CAPACITY_V1, - memory_order_release); - if (llgo_coro_worker_job_valid_v1(job)) { - return LLGO_CORO_WORKER_QUEUE_TAKE_JOB_V1; - } - if (!llgo_coro_worker_job_canceled_v1(job)) { - return LLGO_CORO_WORKER_QUEUE_TAKE_INVALID_V1; - } - break; - } + if (status != LLGO_CORO_WORKER_QUEUE_TAKE_EMPTY_V1) { + if (status == LLGO_CORO_WORKER_QUEUE_TAKE_CANCELED_V1) { continue; } - - position = atomic_load_explicit(&queue->dequeue_position, memory_order_relaxed); - if (llgo_coro_worker_queue_stopping_v1(queue) && - position == atomic_load_explicit(&queue->enqueue_position, memory_order_acquire)) { - return LLGO_CORO_WORKER_QUEUE_TAKE_STOP_V1; - } - /* A later producer may have published before this exact cell. */ - sched_yield(); + return status; + } + if (!llgo_coro_worker_wake_wait_v1(&queue->wake)) { + return LLGO_CORO_WORKER_QUEUE_TAKE_INVALID_V1; } } } @@ -479,6 +603,7 @@ bool __llgo_coro_worker_queue_destroy_after_join_v1(void) { } atomic_store_explicit(&queue->enqueue_position, 0, memory_order_relaxed); atomic_store_explicit(&queue->dequeue_position, 0, memory_order_relaxed); + atomic_store_explicit(&queue->handoff_poller, false, memory_order_relaxed); atomic_store_explicit(&queue->producer_state, 0, memory_order_relaxed); atomic_store_explicit(&queue->initialized, false, memory_order_release); return true; @@ -503,12 +628,14 @@ struct llgo_coro_worker_fault_tls_v1 { uintptr_t trace_target; volatile uintptr_t fault_pc; volatile sig_atomic_t fault; + volatile sig_atomic_t signal_number; volatile sig_atomic_t active; volatile sig_atomic_t handling; }; static _Thread_local struct llgo_coro_worker_fault_tls_v1 llgo_coro_worker_fault_tls_v1; +static _Thread_local int llgo_coro_worker_fault_signals_ready_v1; struct llgo_coro_worker_fault_action_v1 { int signal; @@ -589,6 +716,7 @@ static void llgo_coro_worker_fault_handler_v1( state->fault = signum == SIGFPE ? LLGO_CORO_WORKER_FAULT_DIVIDE_V1 : LLGO_CORO_WORKER_FAULT_MEMORY_V1; + state->signal_number = signum; state->fault_pc = llgo_coro_worker_fault_pc_v1(context); siglongjmp(*state->landing, 1); } @@ -619,6 +747,48 @@ static void llgo_coro_worker_fault_install_v1(void) { llgo_coro_worker_fault_ready_v1 = 1; } +/* + * Internal workers must be able to receive the three synchronous faults that + * the process handler translates. pthreads inherit their creator's mask, so + * normalize it once per physical worker. After a cold siglongjmp, unblock the + * delivered signal again because sigsetjmp deliberately leaves the hot path's + * complete signal-mask snapshot unsaved. + */ +static bool llgo_coro_worker_unblock_fault_signals_v1(int only_signal) { + sigset_t signals; + if (sigemptyset(&signals) != 0) { + return false; + } + if (only_signal != 0) { + if (llgo_coro_worker_fault_action_v1(only_signal) == NULL) { + return false; + } + if (sigaddset(&signals, only_signal) != 0) { + return false; + } + } else { + size_t count = sizeof(llgo_coro_worker_fault_actions_v1) / + sizeof(llgo_coro_worker_fault_actions_v1[0]); + for (size_t index = 0; index < count; ++index) { + if (sigaddset(&signals, llgo_coro_worker_fault_actions_v1[index].signal) != 0) { + return false; + } + } + } + return pthread_sigmask(SIG_UNBLOCK, &signals, NULL) == 0; +} + +static bool llgo_coro_worker_prepare_fault_signals_v1(void) { + if (llgo_coro_worker_fault_signals_ready_v1 != 0) { + return true; + } + if (!llgo_coro_worker_unblock_fault_signals_v1(0)) { + return false; + } + llgo_coro_worker_fault_signals_ready_v1 = 1; + return true; +} + bool __llgo_coro_worker_call_v1( uintptr_t function, uintptr_t trace_target, @@ -634,18 +804,32 @@ bool __llgo_coro_worker_call_v1( struct llgo_coro_worker_fault_tls_v1 *fault_state = &llgo_coro_worker_fault_tls_v1; if (trace_target != 0) { - if (pthread_once( - &llgo_coro_worker_fault_once_v1, - llgo_coro_worker_fault_install_v1) != 0 || - !llgo_coro_worker_fault_ready_v1 || fault_state->active != 0) { + if (fault_state->active != 0) { + return false; + } + /* + * fault_signals_ready is a thread-local certificate produced only + * after pthread_once has installed the process handlers and this + * physical thread has unblocked them. Rechecking pthread_once on + * every foreign call adds an atomic/library boundary to the hottest + * worker path without strengthening that certificate. + */ + if (llgo_coro_worker_fault_signals_ready_v1 == 0 && + (pthread_once( + &llgo_coro_worker_fault_once_v1, + llgo_coro_worker_fault_install_v1) != 0 || + !llgo_coro_worker_fault_ready_v1 || + !llgo_coro_worker_prepare_fault_signals_v1())) { return false; } fault_state->landing = &landing; fault_state->trace_target = trace_target; fault_state->fault_pc = 0; fault_state->fault = LLGO_CORO_WORKER_FAULT_NONE_V1; + fault_state->signal_number = 0; fault_state->handling = 0; - if (sigsetjmp(landing, 1) != 0) { + if (sigsetjmp(landing, 0) != 0) { + int fault_signal = (int)fault_state->signal_number; fault_state->active = 0; fault_state->landing = NULL; result->fault = (uintptr_t)fault_state->fault; @@ -654,9 +838,11 @@ bool __llgo_coro_worker_call_v1( fault_state->trace_target = 0; fault_state->fault_pc = 0; fault_state->fault = LLGO_CORO_WORKER_FAULT_NONE_V1; + fault_state->signal_number = 0; fault_state->handling = 0; - return result->fault == LLGO_CORO_WORKER_FAULT_MEMORY_V1 || - result->fault == LLGO_CORO_WORKER_FAULT_DIVIDE_V1; + return llgo_coro_worker_unblock_fault_signals_v1(fault_signal) && + (result->fault == LLGO_CORO_WORKER_FAULT_MEMORY_V1 || + result->fault == LLGO_CORO_WORKER_FAULT_DIVIDE_V1); } fault_state->active = 1; } @@ -703,6 +889,7 @@ bool __llgo_coro_worker_call_v1( fault_state->active = 0; fault_state->landing = NULL; fault_state->trace_target = 0; + fault_state->signal_number = 0; /* * This leaf returns raw worker-local errno. The compiler-owned diff --git a/runtime/internal/coroworker/_worker/worker.h b/runtime/internal/coroworker/_worker/worker.h index 7152f15521..d499227132 100644 --- a/runtime/internal/coroworker/_worker/worker.h +++ b/runtime/internal/coroworker/_worker/worker.h @@ -65,11 +65,24 @@ int __llgo_coro_worker_create_v1(pthread_t *thread); bool __llgo_coro_worker_queue_init_v1(void); bool __llgo_coro_worker_queue_can_release_v1(void); -bool __llgo_coro_worker_queue_reserve_v1(size_t *reservation); -bool __llgo_coro_worker_queue_cancel_reservation_v1(size_t reservation); -bool __llgo_coro_worker_queue_submit_reserved_v1( +size_t __llgo_coro_worker_queue_reserve_v2(void); +bool __llgo_coro_worker_queue_cancel_reservation_v2(size_t reservation); +bool __llgo_coro_worker_queue_submit_reserved_v4( size_t reservation, - const struct llgo_coro_worker_job_v1 *job); + uint32_t source_slot, + uint32_t generation, + uintptr_t function, + uintptr_t trace_target, + uint32_t argc, + uintptr_t a0, + uintptr_t a1, + uintptr_t a2, + uintptr_t a3, + uintptr_t a4, + uintptr_t a5, + uintptr_t a6, + uintptr_t a7, + uintptr_t a8); uint32_t __llgo_coro_worker_queue_wait_take_v1( struct llgo_coro_worker_job_v1 *job); bool __llgo_coro_worker_queue_stop_v1(uint32_t worker_count); diff --git a/runtime/internal/coroworker/call_llgo.go b/runtime/internal/coroworker/call_llgo.go index e004e3ce57..0e80af87ae 100644 --- a/runtime/internal/coroworker/call_llgo.go +++ b/runtime/internal/coroworker/call_llgo.go @@ -64,21 +64,33 @@ func QueueCanRelease() bool // no-suspend hook. Every live occurrence is in the compiler-verified raw-host // closure, so the declaration publishes no managed executor-safe capability. // -//go:linkname QueueReserve C.__llgo_coro_worker_queue_reserve_v1 -func QueueReserve(reservation *QueueReservation) bool +// Zero is the failure sentinel; successful C reservations are encoded as the +// exact ring position plus one so no Go out-parameter crosses this boundary. +// +//go:linkname QueueReserve C.__llgo_coro_worker_queue_reserve_v2 +func QueueReserve() QueueReservation // QueueCancelReservation publishes an internal tombstone for one unpublished // token. Consumers retire it without exposing an invalid worker job. // -//go:linkname QueueCancelReservation C.__llgo_coro_worker_queue_cancel_reservation_v1 +//go:linkname QueueCancelReservation C.__llgo_coro_worker_queue_cancel_reservation_v2 func QueueCancelReservation(reservation QueueReservation) bool // QueueSubmitReserved release-publishes one POD job and emits one platform -// semaphore signal. sem_post and Mach semaphore_signal never wait for worker -// progress; full capacity was already rejected by QueueReserve. +// semaphore signal. Every argument crosses the Go/C boundary by value and is +// copied into a C-stack Job before queue publication; no Go aggregate address +// crosses the boundary or requires a GC allocation. sem_post and Mach +// semaphore_signal never wait for worker progress; full capacity was already +// rejected by QueueReserve. // -//go:linkname QueueSubmitReserved C.__llgo_coro_worker_queue_submit_reserved_v1 -func QueueSubmitReserved(reservation QueueReservation, job *Job) bool +//go:linkname QueueSubmitReserved C.__llgo_coro_worker_queue_submit_reserved_v4 +func QueueSubmitReserved( + reservation QueueReservation, + sourceSlot, generation uint32, + function, traceTarget uintptr, + argc uint32, + a0, a1, a2, a3, a4, a5, a6, a7, a8 uintptr, +) bool // QueueStop seals producer ingress and emits one terminal wake per raw worker. // It neither drains nor joins workers, but its platform signals may enter libc diff --git a/runtime/internal/coroworker/model.go b/runtime/internal/coroworker/model.go index 43d5ccce07..2fee620d8a 100644 --- a/runtime/internal/coroworker/model.go +++ b/runtime/internal/coroworker/model.go @@ -28,10 +28,10 @@ const ( QueueTakeStop ) -// QueueReservation is the exact monotonically increasing C11 ring position -// owned by one producer. Zero is a valid first reservation; validity is carried -// by QueueReserve's boolean result and the token must be consumed exactly once -// by QueueSubmitReserved or QueueCancelReservation. +// QueueReservation is an opaque nonzero encoding of the exact monotonically +// increasing C11 ring position owned by one producer. Zero reports failed +// admission. A successful token must be consumed exactly once by +// QueueSubmitReserved or QueueCancelReservation. type QueueReservation uintptr // Job is the exact Go view of llgo_coro_worker_job_v1. It is copied by value diff --git a/runtime/internal/coroworker/native_queue_c_test.go b/runtime/internal/coroworker/native_queue_c_test.go index f1e46949db..73fc081eb8 100644 --- a/runtime/internal/coroworker/native_queue_c_test.go +++ b/runtime/internal/coroworker/native_queue_c_test.go @@ -117,7 +117,22 @@ static int submit(size_t reservation, uint32_t generation) { for (uint32_t arg = 0; arg < LLGO_CORO_WORKER_MAX_ARGS_V1; ++arg) { job.args[arg] = ((uintptr_t)generation << 8) + arg; } - return __llgo_coro_worker_queue_submit_reserved_v1(reservation, &job) ? 0 : 1; + return __llgo_coro_worker_queue_submit_reserved_v4( + reservation, + job.source_slot, + job.generation, + job.function, + job.trace_target, + job.argc, + job.args[0], + job.args[1], + job.args[2], + job.args[3], + job.args[4], + job.args[5], + job.args[6], + job.args[7], + job.args[8]) ? 0 : 1; } static void *produce(void *unused) { @@ -129,15 +144,15 @@ static void *produce(void *unused) { return NULL; } size_t reservation; - while (!__llgo_coro_worker_queue_reserve_v1(&reservation)) { + while ((reservation = __llgo_coro_worker_queue_reserve_v2()) == 0) { sched_yield(); } /* Exercise rollback after later producers can already own positions. */ if ((generation & 31) == 0) { - if (!__llgo_coro_worker_queue_cancel_reservation_v1(reservation)) { + if (!__llgo_coro_worker_queue_cancel_reservation_v2(reservation)) { return (void *)(uintptr_t)5; } - while (!__llgo_coro_worker_queue_reserve_v1(&reservation)) { + while ((reservation = __llgo_coro_worker_queue_reserve_v2()) == 0) { sched_yield(); } } @@ -154,17 +169,17 @@ int main(void) { return 10; } size_t reservation; - if (__llgo_coro_worker_queue_reserve_v1(NULL)) { + if (__llgo_coro_worker_queue_cancel_reservation_v2(0)) { return 11; } size_t initial[LLGO_CORO_WORKER_QUEUE_CAPACITY_V1]; for (size_t index = 0; index < LLGO_CORO_WORKER_QUEUE_CAPACITY_V1; ++index) { - if (!__llgo_coro_worker_queue_reserve_v1(&initial[index])) { + if ((initial[index] = __llgo_coro_worker_queue_reserve_v2()) == 0) { return 12; } } - if (__llgo_coro_worker_queue_reserve_v1(&reservation)) { + if (__llgo_coro_worker_queue_reserve_v2() != 0) { return 13; } /* Publish out of order to prove reservation identity is per producer. */ @@ -224,9 +239,9 @@ int main(void) { pthread_t cleanup_worker; if (!__llgo_coro_worker_queue_init_v1() || pthread_create(&cleanup_worker, NULL, consume, NULL) != 0 || - !__llgo_coro_worker_queue_reserve_v1(&reservation) || + (reservation = __llgo_coro_worker_queue_reserve_v2()) == 0 || __llgo_coro_worker_queue_stop_v1(1) || - !__llgo_coro_worker_queue_cancel_reservation_v1(reservation) || + !__llgo_coro_worker_queue_cancel_reservation_v2(reservation) || !__llgo_coro_worker_queue_stop_v1(1)) { return 21; } diff --git a/runtime/internal/lib/runtime/gomaxprocs_coro_llgo.go b/runtime/internal/lib/runtime/gomaxprocs_coro_llgo.go index af68fbe741..9b083f017c 100644 --- a/runtime/internal/lib/runtime/gomaxprocs_coro_llgo.go +++ b/runtime/internal/lib/runtime/gomaxprocs_coro_llgo.go @@ -21,5 +21,12 @@ package runtime import llruntime "github.com/goplus/llgo/runtime/internal/runtime" func GOMAXPROCS(n int) int { - return llruntime.CoroGOMAXPROCS(n) + previous := llruntime.CoroGOMAXPROCS(n) + if n > 0 && previous != n { + // The logical quota changes synchronously above. One explicit scheduler + // boundary makes the next bounded run slice observe its placement policy; + // unrelated channel and timer actions therefore need no epoch poll. + coroSchedulerYield() + } + return previous } diff --git a/runtime/internal/runtime/coro_channel_adapter_test.go b/runtime/internal/runtime/coro_channel_adapter_test.go index a4e3950981..7b2f53639d 100644 --- a/runtime/internal/runtime/coro_channel_adapter_test.go +++ b/runtime/internal/runtime/coro_channel_adapter_test.go @@ -113,6 +113,16 @@ func coroTargetRequestChannelOperationV1(id coro.OperationID) bool { coroTargetRequestExecutorV1(coroProgramExecutorHandleV1State) } +func coroTargetPublishDirectChannelCompletionV1( + owner *coro.ExecutorDriver, + route coro.RouteID, + completion *coro.DirectChannelCompletion, +) bool { + return route == coro.RouteID(1) && + coro.PublishExecutorDirectChannelCompletion(owner, completion) && + coroTargetRequestExecutorV1(coroProgramExecutorHandleV1State) +} + type coroChannelAdapterFrame struct { g *coro.G handle unsafe.Pointer @@ -235,7 +245,10 @@ func resumeCoroChannelAdapterFrame( if !ok || action.Kind != coro.ActionResume { t.Fatalf("activate completed channel adapter G = (%+v, %t)", action, ok) } - status := __llgo_coro_chan_resume_v1(unsafe.Pointer(frame.g), unsafe.Pointer(state)) + status := coroChanResumeCompatibilityV1(unsafe.Pointer(frame.g), unsafe.Pointer(state)) + if *state != (CoroChanParkV1{}) { + t.Fatalf("channel adapter resume retained reusable park state: %+v", *state) + } frame.header.SuspendReason = uint16(coro.SuspendNone) frame.header.Lifecycle = uint16(coro.FrameActive) return action, status @@ -279,6 +292,14 @@ func pollCoroChannelAdapterExecutorProgress(t *testing.T, driver *coro.ExecutorD if !coroMaterializeResumeCleanupStepV1(runStep.Cleanup) { t.Fatalf("materialize channel adapter executor at step %d", step) } + case coro.ExecutorRunStepDirectChannel: + if !coroMaterializeDirectChannelCompletionV1(runStep.Direct) { + t.Fatalf("materialize compact channel adapter executor at step %d", step) + } + atomicResolve = true + if coro.EnterExecutorRunCompatibility(driver) { + return atomicResolve + } default: t.Fatalf("unexpected channel adapter executor step %d at %d", runStep.Kind, step) } @@ -485,6 +506,9 @@ func TestCoroChannelAdapterCleanupCursorBoundsPeerWork(t *testing.T) { } func TestCoroChannelAdapterPairCommitAndResume(t *testing.T) { + if size := unsafe.Sizeof(CoroChanParkV1{}); size > 216 { + t.Fatalf("compact channel park storage size = %d, want <= 216", size) + } coroCurrentTaskRouteTestV1 = 7 defer func() { coroCurrentTaskTestV1 = nil @@ -569,8 +593,16 @@ func TestCoroChannelAdapterPairCommitAndResume(t *testing.T) { sendAction := dequeueCoroChannelAdapterFrame(t, p, pairSender) parkCoroChannelAdapterFrame(t, p, pairSender, sendAction, ch, unsafe.Pointer(&sendValue), pairSenderState, true) pollCoroChannelAdapterExecutor(t, driver) + producerRoute, validProducerRoute := driver.Route() + if !validProducerRoute { + t.Fatal("resolve paired-channel producer route") + } for _, frame := range []*coroChannelAdapterFrame{pairReceiver, pairSender} { - if route, valid := coro.MaterializedRunnablePreferredRoute(frame.g); !valid || route != 7 { + // Compiler channel lowering carries the exact logical task, so the + // compact path derives producer locality from its bound executor rather + // than the host-test current-task shim used by foreign entry points. + if route, valid := coro.MaterializedRunnablePreferredRoute(frame.g); !valid || + route != producerRoute { t.Fatalf("paired-channel materialized producer route = (%d,%t)", route, valid) } } @@ -669,13 +701,16 @@ func TestCoroChannelAdapterPairCommitAndResume(t *testing.T) { } selectedValue := uint32(0xa5b6c7d8) directSendAction := activateCoroChannelAdapterFrame(t, p, directSender) - coroCurrentTaskTestV1 = directSender.g - coroCurrentTaskRouteTestV1 = 1 - if !CoroChanTrySend(selectChannels[1], unsafe.Pointer(&selectedValue), int(unsafe.Sizeof(selectedValue))) { - t.Fatal("same-P direct send did not match channel selector") - } + // Deliberately leave the legacy ambient-current adapter empty and publish a + // mismatched advisory route. The compiler-owned try ABI must derive owner + // locality only from its explicit task argument; otherwise this rendezvous + // falls back to an external source epoch and the atomic-resolve gate below + // fails. coroCurrentTaskTestV1 = nil coroCurrentTaskRouteTestV1 = 7 + if !CoroChanTrySend(unsafe.Pointer(directSender.g), selectChannels[1], unsafe.Pointer(&selectedValue), int(unsafe.Sizeof(selectedValue))) { + t.Fatal("same-P direct send did not match channel selector") + } yieldCoroChannelAdapterFrame(t, p, directSender, directSendAction) if !pollCoroChannelAdapterExecutorProgress(t, driver) { t.Fatal("same-P channel match did not use owner-local completion reduction") @@ -786,6 +821,146 @@ func TestCoroChannelAdapterPairCommitAndResume(t *testing.T) { follow.sendq.first, follow.recvq.first, coroProgramChannelSourceV1State.Pending()) } + // A blocked buffered sender uses the same compact node. Consuming the old + // buffer head must move that sender into the newly free slot, complete it on + // the owner, and leave no generic channel-source generation behind. + bufferedSenderG, bufferedSenderOK := coro.NextRunnable(p) + if !bufferedSenderOK || bufferedSenderG == nil { + t.Fatalf("dequeue compact buffered sender = (%p, %t)", bufferedSenderG, bufferedSenderOK) + } + var bufferedSender *coroChannelAdapterFrame + switch bufferedSenderG { + case receiver.g: + bufferedSender = receiver + case sender.g: + bufferedSender = sender + default: + t.Fatalf("unexpected compact buffered sender G %p", bufferedSenderG) + } + bufferBacking := [1]uint32{0xdecafbad} + compactBuffered := &Chan{ + qcount: 1, dataqsiz: 1, buf: unsafe.Pointer(&bufferBacking[0]), + elemsize: int(unsafe.Sizeof(bufferBacking[0])), + } + compactBuffered.mutex.Init(nil) + bufferedSendValue := uint32(0x11223344) + var bufferedSendState CoroChanParkV1 + bufferedSendAction := activateCoroChannelAdapterFrame(t, p, bufferedSender) + parkCoroChannelAdapterFrame( + t, p, bufferedSender, bufferedSendAction, compactBuffered, + unsafe.Pointer(&bufferedSendValue), &bufferedSendState, true, + ) + if compactBuffered.sendq.first != &bufferedSendState.waiter || + bufferedSendState.waiter.direct != &bufferedSendState.Completion { + t.Fatalf("compact buffered sender not published: %p", compactBuffered.sendq.first) + } + bufferedReceiverG, bufferedReceiverOK := coro.NextRunnable(p) + if !bufferedReceiverOK || bufferedReceiverG == nil || bufferedReceiverG == bufferedSenderG { + t.Fatalf("dequeue compact buffered receiver = (%p, %t), sender=%p", + bufferedReceiverG, bufferedReceiverOK, bufferedSenderG) + } + var bufferedReceiver *coroChannelAdapterFrame + switch bufferedReceiverG { + case receiver.g: + bufferedReceiver = receiver + case sender.g: + bufferedReceiver = sender + default: + t.Fatalf("unexpected compact buffered receiver G %p", bufferedReceiverG) + } + bufferedRecvAction := activateCoroChannelAdapterFrame(t, p, bufferedReceiver) + var bufferedRecvValue uint32 + recvOK, tryOK := CoroChanTryRecv( + unsafe.Pointer(bufferedReceiver.g), compactBuffered, + unsafe.Pointer(&bufferedRecvValue), int(unsafe.Sizeof(bufferedRecvValue)), + ) + if !recvOK || !tryOK || bufferedRecvValue != 0xdecafbad { + t.Fatalf("compact buffered receive = (%#x, %t, %t)", bufferedRecvValue, recvOK, tryOK) + } + yieldCoroChannelAdapterFrame(t, p, bufferedReceiver, bufferedRecvAction) + if !coro.EnterExecutorRunCompatibility(driver) { + t.Fatal("leave compact buffered owner-local completion runner") + } + if compactBuffered.sendq.first != nil || compactBuffered.sendq.last != nil || + compactBuffered.qcount != 1 || bufferBacking[0] != bufferedSendValue || + coroProgramChannelSourceV1State.Pending() { + t.Fatalf("compact buffered refill = queue:(%p,%p) count:%d value:%#x pending:%t", + compactBuffered.sendq.first, compactBuffered.sendq.last, compactBuffered.qcount, + bufferBacking[0], coroProgramChannelSourceV1State.Pending()) + } + bufferedSenderNext, bufferedSenderNextOK := coro.NextRunnable(p) + if !bufferedSenderNextOK || bufferedSenderNext != bufferedSender.g { + t.Fatalf("dequeue completed compact buffered sender = (%p, %t), want %p", + bufferedSenderNext, bufferedSenderNextOK, bufferedSender.g) + } + bufferedSendAction, bufferedSendStatus := resumeCoroChannelAdapterFrame( + t, p, bufferedSender, &bufferedSendState, + ) + if bufferedSendStatus != coroChanResumeSendOK { + t.Fatalf("compact buffered send status = %d, want %d", bufferedSendStatus, coroChanResumeSendOK) + } + yieldCoroChannelAdapterFrame(t, p, bufferedSender, bufferedSendAction) + + // A one-case unbuffered wait uses the compact completion node rather than + // the generic channel source graph. Task cancellation must arbitrate that + // node, detach the typed hchan waiter on the owner, and materialize the same + // compiler-visible abort result without leaving source state behind. + directCanceledG, directCanceledOK := coro.NextRunnable(p) + if !directCanceledOK || directCanceledG == nil { + t.Fatalf("dequeue direct channel waiter for cancellation = (%p, %t)", directCanceledG, directCanceledOK) + } + var directCanceledFrame *coroChannelAdapterFrame + switch directCanceledG { + case receiver.g: + directCanceledFrame = receiver + case sender.g: + directCanceledFrame = sender + default: + t.Fatalf("unexpected direct canceled frame G %p", directCanceledG) + } + directCanceledAction := activateCoroChannelAdapterFrame(t, p, directCanceledFrame) + directCanceledChannel := new(Chan) + directCanceledChannel.elemsize = int(unsafe.Sizeof(uint32(0))) + directCanceledChannel.mutex.Init(nil) + var directCanceledValue uint32 + var directCanceledState CoroChanParkV1 + parkCoroChannelAdapterFrame( + t, p, directCanceledFrame, directCanceledAction, directCanceledChannel, + unsafe.Pointer(&directCanceledValue), &directCanceledState, false, + ) + if directCanceledChannel.recvq.first != &directCanceledState.waiter { + t.Fatalf("direct canceled waiter not published: %p", directCanceledChannel.recvq.first) + } + if !coro.RequestTaskCancellation(p, directCanceledFrame.g, coro.TaskCancelAbort) { + t.Fatal("request direct channel cancellation") + } + pollCoroChannelAdapterExecutor(t, driver) + if directCanceledChannel.recvq.first != nil || directCanceledChannel.recvq.last != nil || + coroProgramChannelSourceV1State.Pending() { + t.Fatalf("direct channel cancellation retained queue/source state: recv=(%p,%p) pending=%t", + directCanceledChannel.recvq.first, directCanceledChannel.recvq.last, + coroProgramChannelSourceV1State.Pending()) + } + directCanceledNext, directCanceledNextOK := coro.NextRunnable(p) + if directCanceledNextOK && directCanceledNext != directCanceledFrame.g { + if !coro.Enqueue(p, directCanceledNext) { + t.Fatalf("rotate unrelated direct cancellation G %p", directCanceledNext) + } + directCanceledNext, directCanceledNextOK = coro.NextRunnable(p) + } + if !directCanceledNextOK || directCanceledNext != directCanceledFrame.g { + t.Fatalf("dequeue canceled direct channel G = (%p, %t), want %p", + directCanceledNext, directCanceledNextOK, directCanceledFrame.g) + } + directCanceledAction, directCanceledStatus := resumeCoroChannelAdapterFrame( + t, p, directCanceledFrame, &directCanceledState, + ) + if directCanceledStatus != coroChanResumeTaskAbort { + t.Fatalf("direct channel cancellation status = %d, want %d", + directCanceledStatus, coroChanResumeTaskAbort) + } + yieldCoroChannelAdapterFrame(t, p, directCanceledFrame, directCanceledAction) + // Claim contention can temporarily leave receivers queued while a sender // uses an available buffer slot. Closing must deliver that buffered value // before publishing the closed zero value to the next receiver. @@ -937,4 +1112,116 @@ func TestCoroChannelAdapterPairCommitAndResume(t *testing.T) { canceledChannels[0].recvq.first, canceledChannels[1].recvq.first, coroProgramChannelSourceV1State.Pending()) } + + // Nil channels have no hchan queue at all. A fresh frame isolates this final + // cancellation-only check from the two task-stop scenarios above. + nilWaitFrame := newCoroChannelAdapterFrame(t) + nilWaitAction := beginCoroChannelAdapterFrame(t, p, nilWaitFrame) + var nilWaitValue uint32 + var nilWaitState CoroChanParkV1 + parkCoroChannelAdapterFrame( + t, p, nilWaitFrame, nilWaitAction, nil, unsafe.Pointer(&nilWaitValue), &nilWaitState, false, + ) + if nilWaitState.waiter.ch != nil || nilWaitState.waiter.direct != &nilWaitState.Completion || + coroProgramChannelSourceV1State.Pending() { + t.Fatalf("compact nil-channel park retained physical/source state: waiter=%+v pending=%t", + nilWaitState.waiter, coroProgramChannelSourceV1State.Pending()) + } + if !coro.RequestTaskCancellation(p, nilWaitFrame.g, coro.TaskCancelAbort) { + t.Fatal("request compact nil-channel cancellation") + } + pollCoroChannelAdapterExecutor(t, driver) + nilWaitNext, nilWaitNextOK := coro.NextRunnable(p) + if !nilWaitNextOK || nilWaitNext != nilWaitFrame.g { + t.Fatalf("dequeue canceled compact nil-channel G = (%p, %t), want %p", + nilWaitNext, nilWaitNextOK, nilWaitFrame.g) + } + nilWaitAction, nilWaitStatus := resumeCoroChannelAdapterFrame(t, p, nilWaitFrame, &nilWaitState) + if nilWaitStatus != coroChanResumeTaskAbort { + t.Fatalf("compact nil-channel status = %d, want %d", nilWaitStatus, coroChanResumeTaskAbort) + } + yieldCoroChannelAdapterFrame(t, p, nilWaitFrame, nilWaitAction) + + // Once the owner has materialized a compact one-case completion, neither its + // direct record nor the physical producer may pin the continuation to that + // owner's P. Exercise the actual typed resume after a runnable transfer; this + // guards the compact path's P-neutral contract independently of select's + // generic ResumePacket migration above. + // Retire prior reusable fixture runnables from this final isolated scenario; + // canceled tasks deliberately retain their task-stop token and cannot form a + // fresh physical park. + for { + g, ok := coro.NextRunnable(p) + if !ok { + t.Fatal("drain prior compact channel fixture runnables") + } + if g == nil { + break + } + } + migrateRecvFrame := newCoroChannelAdapterFrame(t) + migrateRecvG := migrateRecvFrame.g + migrateRecvAction := beginCoroChannelAdapterFrame(t, p, migrateRecvFrame) + migrateChannel := new(Chan) + migrateChannel.elemsize = int(unsafe.Sizeof(uint32(0))) + migrateChannel.mutex.Init(nil) + var migrateRecvValue uint32 + var migrateRecvState CoroChanParkV1 + if status := tryOrParkCoroChanV2( + unsafe.Pointer(migrateRecvFrame.g), migrateRecvFrame.handle, + unsafe.Pointer(migrateRecvFrame.header), unsafe.Pointer(migrateChannel), + unsafe.Pointer(&migrateRecvValue), unsafe.Pointer(&migrateRecvState), + unsafe.Sizeof(migrateRecvValue), 41, 73, false, + ); status != coroChanResumeInvalid { + t.Fatalf("compact try-or-park receive status = %d, want pending", status) + } + if migrateRecvFrame.header.SuspendReason != uint16(coro.SuspendPark) || + migrateRecvFrame.header.Lifecycle != uint16(coro.FrameSuspended) || + migrateRecvFrame.header.StateID != 41 || migrateRecvFrame.header.Line != 73 { + t.Fatalf("compact try-or-park receive header = %+v", *migrateRecvFrame.header) + } + if parked, ok := coro.Resumed(p, migrateRecvFrame.g, migrateRecvAction); !ok || parked.Kind != coro.ActionPark { + t.Fatalf("commit compact try-or-park receive = (%+v, %t)", parked, ok) + } + migrateSendFrame := newCoroChannelAdapterFrame(t) + migrateSendAction := beginCoroChannelAdapterFrame(t, p, migrateSendFrame) + migrateSendValue := uint32(0xc0decafe) + var migrateSendState CoroChanParkV1 + if status := tryOrParkCoroChanV2( + unsafe.Pointer(migrateSendFrame.g), migrateSendFrame.handle, + unsafe.Pointer(migrateSendFrame.header), unsafe.Pointer(migrateChannel), + unsafe.Pointer(&migrateSendValue), unsafe.Pointer(&migrateSendState), + unsafe.Sizeof(migrateSendValue), 42, 74, true, + ); status != coroChanResumeSendOK { + t.Fatalf("compact try-or-park send status = %d, want %d", status, coroChanResumeSendOK) + } + if migrateSendState != (CoroChanParkV1{}) || + migrateSendFrame.header.SuspendReason != uint16(coro.SuspendNone) || + migrateSendFrame.header.Lifecycle != uint16(coro.FrameActive) { + t.Fatalf("ready compact try-or-park mutated slow state: state=%+v header=%+v", + migrateSendState, *migrateSendFrame.header) + } + yieldCoroChannelAdapterFrame(t, p, migrateSendFrame, migrateSendAction) + + directTargetP := new(coro.P) + var directTransfer coro.RunnableTransferMailbox + if !coro.BindRunnableTransferMailbox(&directTransfer, directTargetP) { + t.Fatal("bind compact channel transfer mailbox") + } + directTransferID, directTransferred := coro.PublishPNeutralRunnable(&directTransfer, p, migrateRecvG) + if !directTransferred || !directTransferID.Valid() || + !coro.ImportPNeutralRunnable(&directTransfer, directTargetP, directTransferID) { + t.Fatalf("transfer materialized compact channel G = (%+v, %t)", directTransferID, directTransferred) + } + if next, ok := coro.NextRunnable(directTargetP); !ok || next != migrateRecvG { + t.Fatalf("dequeue transferred compact channel G = (%p, %t), want %p", next, ok, migrateRecvG) + } + migratedAction, status := resumeCoroChannelAdapterFrame( + t, directTargetP, migrateRecvFrame, &migrateRecvState, + ) + if status != coroChanResumeRecvOK || migrateRecvValue != migrateSendValue { + t.Fatalf("migrated compact channel resume = status:%d value:%#x, want status:%d value:%#x", + status, migrateRecvValue, coroChanResumeRecvOK, migrateSendValue) + } + yieldCoroChannelAdapterFrame(t, directTargetP, migrateRecvFrame, migratedAction) } diff --git a/runtime/internal/runtime/coro_channel_request_default.go b/runtime/internal/runtime/coro_channel_request_default.go index d10782f369..0552768eab 100644 --- a/runtime/internal/runtime/coro_channel_request_default.go +++ b/runtime/internal/runtime/coro_channel_request_default.go @@ -29,3 +29,15 @@ func coroTargetRequestChannelOperationV1(id coro.OperationID) bool { coroProgramExecutorHandleV1State != (coro.ExecutorHandle{}) && coroTargetRequestExecutorV1(coroProgramExecutorHandleV1State) } + +func coroTargetPublishDirectChannelCompletionV1( + owner *coro.ExecutorDriver, + route coro.RouteID, + completion *coro.DirectChannelCompletion, +) bool { + return owner == &coroProgramExecutorDriverV1State && route == coro.RouteID(1) && + coroProgramExecutorBoundV1State && + coroProgramExecutorHandleV1State != (coro.ExecutorHandle{}) && + coro.PublishExecutorDirectChannelCompletion(owner, completion) && + coroTargetRequestExecutorV1(coroProgramExecutorHandleV1State) +} diff --git a/runtime/internal/runtime/coro_current_task_route.go b/runtime/internal/runtime/coro_current_task_route.go index dc7c1f11b8..97dfe71a26 100644 --- a/runtime/internal/runtime/coro_current_task_route.go +++ b/runtime/internal/runtime/coro_current_task_route.go @@ -25,23 +25,23 @@ import "github.com/goplus/llgo/runtime/internal/coro" // foreign-thread callbacks, timers, IO reactors, and teardown paths // deliberately return route zero. A synchronous same-G C-to-Go reentry is // still inside that managed resume and may inherit its route. -func coroCurrentTaskV1() (*coro.G, coro.RouteID) { - gp := getg() +func coroCurrentTaskV1() (*coro.G, *coro.ExecutorDriver, coro.RouteID) { + gp := getgIfPresent() if gp == nil || gp.startfn != nil || gp.startarg == nil { - return nil, 0 + return nil, nil, 0 } task := (*coro.G)(gp.startarg) if ctx := (*coroRuntimeContext)(coro.TaskLocal(task)); ctx != gp.context || !validCoroRuntimeTaskContext(task, ctx) { - return nil, 0 + return nil, nil, 0 } - _, _, route, current := coro.CurrentExecutorDriver(task) + driver, route, current := coro.CurrentExecutorDriverForActiveResume(task) if !current { - return nil, 0 + return nil, nil, 0 } - return task, route + return task, driver, route } func coroCurrentTaskRouteV1() coro.RouteID { - _, route := coroCurrentTaskV1() + _, _, route := coroCurrentTaskV1() return route } diff --git a/runtime/internal/runtime/coro_current_task_route_default.go b/runtime/internal/runtime/coro_current_task_route_default.go index b9eab2eaf4..9b775a4bb3 100644 --- a/runtime/internal/runtime/coro_current_task_route_default.go +++ b/runtime/internal/runtime/coro_current_task_route_default.go @@ -23,11 +23,11 @@ import "github.com/goplus/llgo/runtime/internal/coro" // Targets without a multi-route fleet have no useful producer-locality // destination. Keeping this a compile-time zero also avoids treating the // command/host executor's route-1 identity as a migration contract. -func coroCurrentTaskV1() (*coro.G, coro.RouteID) { - return nil, 0 +func coroCurrentTaskV1() (*coro.G, *coro.ExecutorDriver, coro.RouteID) { + return nil, nil, 0 } func coroCurrentTaskRouteV1() coro.RouteID { - _, route := coroCurrentTaskV1() + _, _, route := coroCurrentTaskV1() return route } diff --git a/runtime/internal/runtime/coro_current_task_route_test_adapter.go b/runtime/internal/runtime/coro_current_task_route_test_adapter.go index c44a0811cf..a0908835e7 100644 --- a/runtime/internal/runtime/coro_current_task_route_test_adapter.go +++ b/runtime/internal/runtime/coro_current_task_route_test_adapter.go @@ -25,11 +25,18 @@ var ( coroCurrentTaskRouteTestV1 coro.RouteID ) -func coroCurrentTaskV1() (*coro.G, coro.RouteID) { - return coroCurrentTaskTestV1, coroCurrentTaskRouteTestV1 +func coroCurrentTaskV1() (*coro.G, *coro.ExecutorDriver, coro.RouteID) { + if coroCurrentTaskTestV1 == nil { + return nil, nil, coroCurrentTaskRouteTestV1 + } + driver, _, route, ok := coro.CurrentExecutorDriver(coroCurrentTaskTestV1) + if !ok || route != coroCurrentTaskRouteTestV1 { + return coroCurrentTaskTestV1, nil, coroCurrentTaskRouteTestV1 + } + return coroCurrentTaskTestV1, driver, route } func coroCurrentTaskRouteV1() coro.RouteID { - _, route := coroCurrentTaskV1() + _, _, route := coroCurrentTaskV1() return route } diff --git a/runtime/internal/runtime/coro_execution_quota_native_llgo.go b/runtime/internal/runtime/coro_execution_quota_native_llgo.go index 8cd307ceb0..0180465980 100644 --- a/runtime/internal/runtime/coro_execution_quota_native_llgo.go +++ b/runtime/internal/runtime/coro_execution_quota_native_llgo.go @@ -151,7 +151,7 @@ func CoroGOMAXPROCS(n int) int { if uint64(n) > uint64(coroNativeMaximumLogicalProcsV1) { next = coroNativeMaximumLogicalProcsV1 } - previous, wake, changed := coroNativeFleetV1State.execution.SetLimit(next) + previous, wake, changed := coroNativeFleetSetExecutionLimitV1(next) waiters := uint32(0) waitersOK := true if wake { diff --git a/runtime/internal/runtime/coro_executor_driver_host_llgo.go b/runtime/internal/runtime/coro_executor_driver_host_llgo.go index 0a017e9cbb..09f0c18010 100644 --- a/runtime/internal/runtime/coro_executor_driver_host_llgo.go +++ b/runtime/internal/runtime/coro_executor_driver_host_llgo.go @@ -47,12 +47,21 @@ func coroProgramBindExecutorDriverV1(driver *coro.ExecutorDriver, p *coroP, regi }) } -func coroProgramNextRunStepV1(driver *coro.ExecutorDriver) (coro.ExecutorRunStep, bool) { +func coroProgramNextRunStepV1( + driver *coro.ExecutorDriver, + run *coro.ExecutorRunSliceCapability, + combineDispatch bool, +) (coro.ExecutorRunStep, bool) { now, ok := coroHostClockV1State.Snapshot() if !ok { return coro.ExecutorRunStep{}, false } - step, ok := coro.NextExecutorRunStepAt(driver, now) + var step coro.ExecutorRunStep + if combineDispatch { + step, ok = run.NextAtCombined(now) + } else { + step, ok = run.NextAt(now) + } return step, ok && coroProgramSyncHostOperationCancelsV1(driver) } diff --git a/runtime/internal/runtime/coro_executor_driver_legacy.go b/runtime/internal/runtime/coro_executor_driver_legacy.go index 0697f18c16..2e14d12124 100644 --- a/runtime/internal/runtime/coro_executor_driver_legacy.go +++ b/runtime/internal/runtime/coro_executor_driver_legacy.go @@ -26,8 +26,15 @@ func coroProgramBindExecutorDriverV1(driver *coro.ExecutorDriver, p *coroP, regi }) } -func coroProgramNextRunStepV1(driver *coro.ExecutorDriver) (coro.ExecutorRunStep, bool) { - return coro.NextExecutorRunStep(driver) +func coroProgramNextRunStepV1( + _ *coro.ExecutorDriver, + run *coro.ExecutorRunSliceCapability, + combineDispatch bool, +) (coro.ExecutorRunStep, bool) { + if combineDispatch { + return run.NextCombined() + } + return run.Next() } func coroProgramPrepareExecutorSleepV1(driver *coro.ExecutorDriver) (sleep bool, deadline int64, hasDeadline, ok bool) { diff --git a/runtime/internal/runtime/coro_executor_driver_timer_llgo.go b/runtime/internal/runtime/coro_executor_driver_timer_llgo.go index b21c608977..c8c8307465 100644 --- a/runtime/internal/runtime/coro_executor_driver_timer_llgo.go +++ b/runtime/internal/runtime/coro_executor_driver_timer_llgo.go @@ -43,7 +43,6 @@ func coroProgramBindExecutorDriverV1(driver *coro.ExecutorDriver, p *coroP, regi if coro.TimerRegistrationConfiguredCapacity(&coroProgramTimerTableV1State) != coro.TimerRegistrationPageCapacity || coro.PollOperationConfiguredCapacity(&coroProgramPollSourceV1State) != coro.PollOperationPageCapacity || coro.ManualOperationConfiguredCapacity(&coroProgramManualSourceV2State) != coro.ManualOperationPageCapacity || - coro.WorkerOperationConfiguredCapacity(&coroProgramWorkerSourceV1State) != coro.WorkerOperationPageCapacity || coro.ChannelOperationConfiguredCapacity(&coroProgramChannelSourceV1State) != coro.ChannelOperationPageCapacity || coroNativeTimerCapacityV1 != coroNativeTimerPageCountV1*coro.TimerRegistrationPageCapacity || coroNativePollCapacityV1 != coroNativeSourcePageCountV1*coro.PollOperationPageCapacity || @@ -51,25 +50,51 @@ func coroProgramBindExecutorDriverV1(driver *coro.ExecutorDriver, p *coroP, regi coroNativeWorkerQueueSizeV1 != coroNativeWorkerCapacityV1 { return false } + var worker *coro.WorkerOperationSource + if coroProgramWorkerCapabilityV2() { + if coro.WorkerOperationConfiguredCapacity(&coroProgramWorkerSourceV1State) != coro.WorkerOperationPageCapacity { + return false + } + worker = &coroProgramWorkerSourceV1State + } return coro.BindExecutorSourceCatalog(driver, p, registry, handle, coro.ExecutorSourceCatalog{ Timers: &coroProgramTimerTableV1State, Poll: &coroProgramPollSourceV1State, Manual: &coroProgramManualSourceV2State, - Worker: &coroProgramWorkerSourceV1State, + Worker: worker, Channel: &coroProgramChannelSourceV1State, Control: &coroProgramTaskControlSourceV1State, }) } -func coroProgramNextRunStepV1(driver *coro.ExecutorDriver) (coro.ExecutorRunStep, bool) { +func coroProgramNextRunStepV1( + _ *coro.ExecutorDriver, + run *coro.ExecutorRunSliceCapability, + combineDispatch bool, +) (coro.ExecutorRunStep, bool) { + if combineDispatch { + if step, ok := run.NextBeforeTimeCombined(); ok { + return step, true + } + } else if step, ok := run.NextBeforeTime(); ok { + return step, true + } now, ok := coroclock.MonotonicNano() if !ok { return coro.ExecutorRunStep{}, false } - return coro.NextExecutorRunStepAt(driver, now) + if combineDispatch { + return run.NextAtCombined(now) + } + return run.NextAt(now) } func coroProgramPrepareExecutorSleepV1(driver *coro.ExecutorDriver) (sleep bool, deadline int64, hasDeadline, ok bool) { + if ready, fastOK := coroNativeTryFastWorkerCompletionV1(driver); !fastOK { + return false, 0, false, false + } else if ready { + return false, 0, false, true + } now, ok := coroclock.MonotonicNano() if !ok { return false, 0, false, false diff --git a/runtime/internal/runtime/coro_executor_driver_worker_llgo.go b/runtime/internal/runtime/coro_executor_driver_worker_llgo.go index 4b04dda7a4..dfe4933c88 100644 --- a/runtime/internal/runtime/coro_executor_driver_worker_llgo.go +++ b/runtime/internal/runtime/coro_executor_driver_worker_llgo.go @@ -26,23 +26,40 @@ import "github.com/goplus/llgo/runtime/internal/coro" // managed blocking foreign call always parks its G and completes through the // pipe; it must never fall back to running on the executor thread. func coroProgramBindExecutorDriverV1(driver *coro.ExecutorDriver, p *coroP, registry *coro.ExecutorRegistry, handle coro.ExecutorHandle) bool { - if coro.WorkerOperationConfiguredCapacity(&coroProgramWorkerSourceV1State) != coro.WorkerOperationPageCapacity || - coroNativeWorkerCapacityV1 != coroRuntimeWorkerCapacityV1 || + workerEnabled := coroProgramWorkerCapabilityV2() + if workerEnabled && (coro.WorkerOperationConfiguredCapacity(&coroProgramWorkerSourceV1State) != coro.WorkerOperationPageCapacity || + coroNativeWorkerCapacityV1 != coroRuntimeWorkerCapacityV1) || coroNativeWorkerQueueSizeV1 != coroNativeWorkerCapacityV1 { return false } + var worker *coro.WorkerOperationSource + if workerEnabled { + worker = &coroProgramWorkerSourceV1State + } return coro.BindExecutorSourceCatalog(driver, p, registry, handle, coro.ExecutorSourceCatalog{ - Worker: &coroProgramWorkerSourceV1State, + Worker: worker, Channel: &coroProgramChannelSourceV1State, Control: &coroProgramTaskControlSourceV1State, }) } -func coroProgramNextRunStepV1(driver *coro.ExecutorDriver) (coro.ExecutorRunStep, bool) { - return coro.NextExecutorRunStep(driver) +func coroProgramNextRunStepV1( + _ *coro.ExecutorDriver, + run *coro.ExecutorRunSliceCapability, + combineDispatch bool, +) (coro.ExecutorRunStep, bool) { + if combineDispatch { + return run.NextCombined() + } + return run.Next() } func coroProgramPrepareExecutorSleepV1(driver *coro.ExecutorDriver) (sleep bool, deadline int64, hasDeadline, ok bool) { + if ready, fastOK := coroNativeTryFastWorkerCompletionV1(driver); !fastOK { + return false, 0, false, false + } else if ready { + return false, 0, false, true + } sleep, ok = coro.PrepareExecutorSleep(driver) return sleep, 0, false, ok } diff --git a/runtime/internal/runtime/coro_frame.go b/runtime/internal/runtime/coro_frame.go index 36acc12af7..bf8b64fd79 100644 --- a/runtime/internal/runtime/coro_frame.go +++ b/runtime/internal/runtime/coro_frame.go @@ -50,6 +50,26 @@ func __llgo_coro_frame_publish_v1(g, handle, header, storage unsafe.Pointer) { } } +//go:noinline +//export __llgo_coro_frame_publish_v3 +func __llgo_coro_frame_publish_v3( + g, handle, header, storage, metadata, descriptor, resultSlot unsafe.Pointer, +) { + if !coro.PublishFrameV3( + (*coro.G)(g), handle, (*coro.HeaderV1)(header), storage, metadata, + descriptor, resultSlot, + ) { + coroRuntimeAbort("invalid initialized borrowable coroutine frame publication") + } +} + +//export __llgo_coro_frame_destroy_commit_v2 +func __llgo_coro_frame_destroy_commit_v2(g, handle unsafe.Pointer) { + if !coro.CommitFrameDestroyCompiler((*coro.G)(g), handle) { + coroRuntimeAbort("invalid borrowable coroutine frame destruction") + } +} + //export __llgo_coro_await_prepare_v1 func __llgo_coro_await_prepare_v1(g, parent, child unsafe.Pointer) { if !coro.PrepareAwait((*coro.G)(g), parent, child) { @@ -59,7 +79,7 @@ func __llgo_coro_await_prepare_v1(g, parent, child unsafe.Pointer) { //export __llgo_coro_await_prepare_v2 func __llgo_coro_await_prepare_v2(g, parent, child unsafe.Pointer) { - if !coro.PrepareAwaitCompletion((*coro.G)(g), parent, child) { + if !coro.PrepareAwaitCompletionCompiler((*coro.G)(g), parent, child) { coroRuntimeAbort("invalid coroutine child completion handoff") } } @@ -70,7 +90,7 @@ func __llgo_coro_await_consume_v1(g, parent, typeOut, dataOut unsafe.Pointer) ui coroRuntimeAbort("invalid coroutine child outcome output") } task := (*coro.G)(g) - snapshot, ok := coro.ConsumeAwaitCompletion(task, parent) + snapshot, ok := coro.ConsumeAwaitCompletionCompiler(task, parent) if !ok { coroRuntimeAbort("invalid coroutine child outcome consume") } @@ -82,7 +102,7 @@ func __llgo_coro_await_consume_v1(g, parent, typeOut, dataOut unsafe.Pointer) ui //export __llgo_coro_preempt_poll_v1 func __llgo_coro_preempt_poll_v1(g unsafe.Pointer) bool { - return coro.PollPreempt((*coro.G)(g)) + return coro.PollPreemptCompiler((*coro.G)(g)) } //export __llgo_coro_yield_prepare_v1 @@ -107,7 +127,7 @@ func __llgo_coro_complete_prepare_v2(g, handle, header unsafe.Pointer, status ui task := (*coro.G)(g) frameHeader := (*coro.HeaderV1)(header) completion := coro.CompletionStatus(status) - if !coro.PrepareCompleteStatus( + if !coro.PrepareCompleteStatusCompiler( task, handle, frameHeader, completion, ) { coroRuntimeAbort("invalid coroutine terminal completion handoff") @@ -132,7 +152,6 @@ func __llgo_coro_frame_free_v1(g, storage unsafe.Pointer, size, align uintptr, d task == &coroProgramGV1State && coro.RetainPanicTraceFrame(task, raw, total) { return } - coro.Zero(raw, total) if !coroalloc.FreeFrame(raw, total) { coroRuntimeAbort("coroutine frame release failed") } diff --git a/runtime/internal/runtime/coro_keyed_park.go b/runtime/internal/runtime/coro_keyed_park.go index c44702747e..c3bff26272 100644 --- a/runtime/internal/runtime/coro_keyed_park.go +++ b/runtime/internal/runtime/coro_keyed_park.go @@ -31,6 +31,29 @@ const ( coroKeyedResumeShutdownV2 ) +// raiseScanLimit publishes the shortest prefix which can contain an Active +// slot. The prefix is monotonic: retirement never needs to coordinate a +// downward move with concurrent registration or claim. This keeps the common +// one-waiter semaphore path O(1), while a registry which has actually reached +// N simultaneous slots retains the original bounded O(N) FIFO selection. +func (registry *coroKeyedRegistryV2) raiseScanLimit(needed uint32) bool { + if registry == nil || needed == 0 || needed > uint32(len(registry.slots)) { + return false + } + for { + limit := coroKeyedAtomicLoadUint32(®istry.scanLimit) + if limit > uint32(len(registry.slots)) { + return false + } + if limit >= needed { + return true + } + if coroKeyedAtomicCompareAndSwapUint32(®istry.scanLimit, limit, needed) { + return true + } + } +} + func (registry *coroKeyedRegistryV2) register( kind coroKeyedParkKindV2, key uintptr, @@ -54,6 +77,10 @@ func (registry *coroKeyedRegistryV2) register( if !coroKeyedAtomicCompareAndSwapUint32(&slot.control, control, registering) { continue } + if !registry.raiseScanLimit(uint32(index) + 1) { + _ = coroKeyedAtomicCompareAndSwapUint32(&slot.control, registering, control) + return coroKeyedRegistryHandleV2{}, false + } sequence := coroKeyedAtomicAddUint32(®istry.sequence, 1) coroKeyedAtomicStoreUint32((*uint32)(unsafe.Pointer(&slot.kind)), uint32(kind)) coroKeyedAtomicStoreUint32(&slot.logical, logical) @@ -155,9 +182,13 @@ func (registry *coroKeyedRegistryV2) claimOne( return coroKeyedRegistryHandleV2{}, coro.OperationID{}, false } for { + limit := coroKeyedAtomicLoadUint32(®istry.scanLimit) + if limit > uint32(len(registry.slots)) { + return coroKeyedRegistryHandleV2{}, coro.OperationID{}, false + } selected := -1 var candidate coroKeyedRegistrySnapshotV2 - for index := range registry.slots { + for index := uint32(0); index < limit; index++ { slot := ®istry.slots[index] control := coroKeyedAtomicLoadUint32(&slot.control) if coroKeyedRegistryControlStateV2(control) != coroKeyedRegistryActiveV2 { @@ -169,7 +200,7 @@ func (registry *coroKeyedRegistryV2) claimOne( continue } if selected < 0 || coroKeyedRegistrySequenceLessV2(snapshot.sequence, candidate.sequence) { - selected, candidate = index, snapshot + selected, candidate = int(index), snapshot } } if selected < 0 { @@ -261,6 +292,21 @@ func coroKeyedPostClaimedV2(handle coroKeyedRegistryHandleV2, operation coro.Ope case coroKeyedRegistryPublishRetiredV2: return true case coroKeyedRegistryPublishReadyV2: + current, driver, route := coroCurrentTaskV1() + if current != nil && driver != nil && route == operation.Route() { + completion, cleanup, local, localOK := coro.BeginOwnerLocalManualCompletionCurrent( + current, + driver, + operation, + ) + if !localOK { + return coroProgramKeyedRegistryV2State.publicationRetired(handle) + } + if local { + return coroMaterializePrivateResumeCleanupStepV1(cleanup) && + coro.FinishOwnerLocalManualCompletionCurrent(&completion) + } + } if coroTargetPostKeyedOperationV2(operation) { return true } @@ -319,37 +365,44 @@ func __llgo_coro_keyed_park_v2(g, handle, header, storage unsafe.Pointer) { return } task := (*coro.G)(g) - driver, wantExecutor, wantRoute, ok := coro.CurrentExecutorManualDriver(task) - if !ok || !ensureCoroManualOperationCapacityV1(driver, task, coroRuntimeManualCapacityV1) { + driver, wantExecutor, wantRoute, reservation, current, reserved := + coro.CurrentExecutorManualReservation(task) + if current && !reserved { + if !ensureCoroManualOperationCapacityV1(driver, task, coroRuntimeManualCapacityV1) { + coroKeyedAbortV2("cannot resolve coroutine keyed Park V2 owner") + return + } + var retryDriver *coro.ExecutorDriver + retryDriver, wantExecutor, wantRoute, reservation, current, reserved = + coro.CurrentExecutorManualReservation(task) + if retryDriver != driver { + current = false + } + } + if !current || !reserved { coroKeyedAbortV2("cannot resolve coroutine keyed Park V2 owner") return } - ticket, operation, executor, prepared := coro.PrepareCurrentExecutorManualPark( - driver, task, handle, (*coro.HeaderV1)(header), &state.wait, 1, 1, + ticket, operation, executor, prepared := coro.PrepareCurrentExecutorManualCleanupParkReserved( + driver, + task, + handle, + (*coro.HeaderV1)(header), + &state.wait, + reservation, + &state.packet, + &state.cleanup, + unsafe.Pointer(state), + &state.operation, + 1, + 1, ) if !prepared || executor != wantExecutor || operation.Route() != wantRoute { coroKeyedAbortV2("cannot prepare coroutine keyed Park V2 source") return } state.ticket = ticket - state.operation = operation state.magic = coroKeyedParkActiveMagicV2 - if !coro.BindWaitSetResumeCleanup( - &state.wait, - &state.packet, - &state.cleanup, - coro.ResumeCleanupBinding{ - Kind: coro.ResumeCleanupKeyedPark, - Context: unsafe.Pointer(state), - Entries: unsafe.Pointer(&state.operation), - Count: 1, - RuntimeCount: 1, - Stride: unsafe.Sizeof(coro.OperationID{}), - }, - ) { - coroKeyedAbortV2("cannot bind coroutine keyed Park V2 cleanup") - return - } // Publish the scalar key only after the complete P-neutral cleanup // descriptor is bound. A concurrent release can post the Manual fact // immediately, but the current owner cannot suspend this G with a partially diff --git a/runtime/internal/runtime/coro_keyed_registry_test.go b/runtime/internal/runtime/coro_keyed_registry_test.go index 7bcde854f2..a40d7fe3ff 100644 --- a/runtime/internal/runtime/coro_keyed_registry_test.go +++ b/runtime/internal/runtime/coro_keyed_registry_test.go @@ -92,6 +92,35 @@ func TestCoroKeyedRegistryFIFOAndExactLogicalSelectionV2(t *testing.T) { } } +func TestCoroKeyedRegistryScanLimitTracksOnlyPublishedHighWaterV2(t *testing.T) { + registry := new(coroKeyedRegistryV2) + firstID := keyedRegistryOperation(t, 1, 1) + first, ok := registry.register(coroKeyedParkSemaphoreV2, 0x11, 0, firstID) + if !ok || first.Slot != 1 || coroKeyedAtomicLoadUint32(®istry.scanLimit) != 1 { + t.Fatalf("first keyed scan prefix = handle:%+v ok:%t limit:%d", + first, ok, coroKeyedAtomicLoadUint32(®istry.scanLimit)) + } + secondID := keyedRegistryOperation(t, 2, 1) + second, ok := registry.register(coroKeyedParkSemaphoreV2, 0x11, 0, secondID) + if !ok || second.Slot != 2 || coroKeyedAtomicLoadUint32(®istry.scanLimit) != 2 { + t.Fatalf("second keyed scan prefix = handle:%+v ok:%t limit:%d", + second, ok, coroKeyedAtomicLoadUint32(®istry.scanLimit)) + } + if !registry.retire(second, secondID) || + coroKeyedAtomicLoadUint32(®istry.scanLimit) != 2 { + t.Fatalf("retirement rewrote monotonic scan prefix = %d", + coroKeyedAtomicLoadUint32(®istry.scanLimit)) + } + claimed, operation, found := registry.claimOne(coroKeyedParkSemaphoreV2, 0x11, 0, false) + if !found || claimed != first || operation != firstID { + t.Fatalf("bounded-prefix keyed claim = %+v/%+v/%t", claimed, operation, found) + } + if registry.finishPost(claimed, operation) != coroKeyedRegistryPublishReadyV2 || + !registry.retire(claimed, operation) { + t.Fatal("finish bounded-prefix keyed claim") + } +} + func TestCoroKeyedRegistryConcurrentClaimIsSingleOwnerV2(t *testing.T) { registry := new(coroKeyedRegistryV2) id := keyedRegistryOperation(t, 1, 1) diff --git a/runtime/internal/runtime/coro_native_fleet.go b/runtime/internal/runtime/coro_native_fleet.go index 6dc6e7dc43..69523af4eb 100644 --- a/runtime/internal/runtime/coro_native_fleet.go +++ b/runtime/internal/runtime/coro_native_fleet.go @@ -143,6 +143,9 @@ func (domain *coroNativeFleetDomainV1) workerOwnerV1() *coro.WorkerOperationSour if domain.adopted { return domain.owners.sources.Worker } + if _, bound := domain.worker.Route(); !bound { + return nil + } return &domain.worker } @@ -159,7 +162,7 @@ func (domain *coroNativeFleetDomainV1) channelOwnerV1() *coro.ChannelOperationSo func validCoroNativeFleetAdoptedOwnersV1(owners coroNativeFleetDomainOwnersV1) bool { sources := owners.sources return owners.p != nil && owners.driver != nil && sources.Timers != nil && - sources.Poll != nil && sources.Manual != nil && sources.Worker != nil && sources.Channel != nil && + sources.Poll != nil && sources.Manual != nil && sources.Channel != nil && sources.Control != nil } @@ -169,8 +172,9 @@ func coroNativeFleetAdoptedOwnersRetiredV1(domain *coroNativeFleetDomainV1) bool } owners := domain.owners sources := owners.sources + workerRetired := sources.Worker == nil || sources.Worker.CanRelease() return *owners.driver == (coro.ExecutorDriver{}) && sources.Timers.CanRelease() && - sources.Poll.CanRelease() && sources.Manual.CanRelease() && sources.Worker.CanRelease() && sources.Channel.CanRelease() && + sources.Poll.CanRelease() && sources.Manual.CanRelease() && workerRetired && sources.Channel.CanRelease() && sources.Control.CanRelease() } @@ -310,7 +314,7 @@ func coroNativeFleetValidateOwnedDomainSourcesV1(domain *coroNativeFleetDomainV1 coro.ChannelOperationConfiguredCapacity(&domain.channel) == coro.ChannelOperationPageCapacity } -func coroNativeFleetBindDomainV1(state *coroNativeFleetStateV1, index uint32) bool { +func coroNativeFleetBindDomainV1(state *coroNativeFleetStateV1, index uint32, workerEnabled bool) bool { if state == nil || index >= coroNativeFleetDomainCapacityV1 { return false } @@ -320,6 +324,10 @@ func coroNativeFleetBindDomainV1(state *coroNativeFleetStateV1, index uint32) bo !domain.doorbell.Open() { return false } + var worker *coro.WorkerOperationSource + if workerEnabled { + worker = &domain.worker + } handle, ok := coro.BindExecutorFleet( &state.fleet, &domain.driver, @@ -328,7 +336,7 @@ func coroNativeFleetBindDomainV1(state *coroNativeFleetStateV1, index uint32) bo Timers: &domain.timers, Poll: &domain.poll, Manual: &domain.manual, - Worker: &domain.worker, + Worker: worker, Channel: &domain.channel, Control: &domain.control, }, @@ -376,12 +384,13 @@ func coroNativeFleetStartDomainsV1( // identities and guessed callback words must keep observing one permanent // fail-stop policy rather than a recyclable zero state. state.domainCount = count + workerEnabled := program == nil || program.sources.Worker != nil for index := uint32(0); index < count; index++ { started := false if index == 0 && program != nil { started = coroNativeFleetAdoptDomainV1(state, index, *program) } else { - started = coroNativeFleetBindDomainV1(state, index) + started = coroNativeFleetBindDomainV1(state, index, workerEnabled) } if !started { for previous := uint32(0); previous < index; previous++ { @@ -889,10 +898,16 @@ func coroNativeFleetPrepareOwnerWaitAtV1( coroNativeFleetDomainActiveV1, ) driver := domain.driverOwnerV1() - if !valid || driver == nil || epoch == 0 || domain.ownerEpoch != epoch || now < 0 || freshNow < now || - !coro.EnterExecutorRunCompatibility(driver) { + if !valid || driver == nil || epoch == 0 || domain.ownerEpoch != epoch || now < 0 || freshNow < now { return coroNativeFleetOwnerWaitPlanV1{}, false } + entered, compatibilityOK := coro.EnterExecutorRunStandbyCompatibility(driver) + if !compatibilityOK { + return coroNativeFleetOwnerWaitPlanV1{}, false + } + if !entered { + return coroNativeFleetOwnerWaitPlanV1{}, true + } prepared, ok := coro.PrepareExecutorStandbyAt(driver, now) if !ok { return coroNativeFleetOwnerWaitPlanV1{}, false diff --git a/runtime/internal/runtime/coro_native_fleet_owner_llgo.go b/runtime/internal/runtime/coro_native_fleet_owner_llgo.go index 6fab184dcd..2229484649 100644 --- a/runtime/internal/runtime/coro_native_fleet_owner_llgo.go +++ b/runtime/internal/runtime/coro_native_fleet_owner_llgo.go @@ -39,9 +39,10 @@ const ( coroNativeFleetPhysicalFailedV1 ) -// coroNativeFleetPhysicalOwnerV1 is one process-lifetime peer M. Production -// starts the complete bounded topology before any managed resume, and each -// slot retains its exact route tombstone after join. +// coroNativeFleetPhysicalOwnerV1 is one process-lifetime peer M. The logical +// route topology is bound eagerly, but a peer M is started only when the +// logical execution quota can use its route. Once started, each slot retains +// its exact route tombstone after join. type coroNativeFleetPhysicalOwnerV1 struct { handle coro.ExecutorFleetHandle slot uint32 @@ -54,10 +55,13 @@ type coroNativeFleetPhysicalOwnerV1 struct { // Seal is the release publication observed by every peer through Quiesced, and // Retire leaves a permanent non-reusable process tombstone after all joins. type coroNativeFleetPhysicalOwnersV1 struct { - stop coro.TargetIngress - peers [coroNativeFleetDomainCapacityV1 - 1]coroNativeFleetPhysicalOwnerV1 - count uint32 - lifecycle coroNativeFleetPhysicalLifecycleV1 + stop coro.TargetIngress + peers [coroNativeFleetDomainCapacityV1 - 1]coroNativeFleetPhysicalOwnerV1 + count uint32 // immutable configured peer capacity + started uint32 // contiguous live prefix, protected by guard + policyEpoch uint32 // atomic GOMAXPROCS placement-policy generation + guard uint32 + lifecycle coroNativeFleetPhysicalLifecycleV1 } var coroNativeFleetPhysicalOwnerV1State coroNativeFleetPhysicalOwnersV1 @@ -87,48 +91,103 @@ func coroNativeFleetPhysicalOwnerForHandleV1( } } -func coroNativeFleetPhysicalOwnersStartV1() bool { - state := &coroNativeFleetPhysicalOwnerV1State - count := coroNativeFleetV1State.domainCount - if coroNativeFleetV1State.lifecycle != coroNativeFleetActiveV1 || - count == 0 || count > coroNativeFleetDomainCapacityV1 || - state.lifecycle != coroNativeFleetPhysicalUnusedV1 || state.count != 0 || - !state.stop.CanReleaseResources() || !state.stop.Start() { +func coroNativeFleetPhysicalOwnerDesiredPeersV1(limit, capacity uint32) (uint32, bool) { + if limit == 0 || capacity >= coroNativeFleetDomainCapacityV1 { + return 0, false + } + if limit-1 < capacity { + return limit - 1, true + } + return capacity, true +} + +func coroNativeFleetPhysicalOwnersLockV1(state *coroNativeFleetPhysicalOwnersV1) bool { + if state == nil { return false } - state.count = count - 1 - state.lifecycle = coroNativeFleetPhysicalActiveV1 - created := uint32(0) - for index := uint32(0); index < state.count; index++ { + for !coroNativeAtomicCASV1(&state.guard, 0, 1) { + if corofleet.Yield() != 0 { + return false + } + } + return true +} + +func coroNativeFleetPhysicalOwnersUnlockV1(state *coroNativeFleetPhysicalOwnersV1) { + if state == nil || coroNativeAtomicLoadV1(&state.guard) != 1 { + coroRuntimeAbort("native coroutine fleet physical owner lock corrupt") + return + } + coroNativeAtomicStoreV1(&state.guard, 0) +} + +// coroNativeFleetPhysicalOwnersEnsureLockedV1 grows the contiguous physical +// peer prefix to cover limit. Logical domains outside that prefix remain fully +// bound but never publish runnable demand, so no task or source completion can +// be routed to an ownerless domain. The clean factory acknowledges each new M +// before this function publishes started. +func coroNativeFleetPhysicalOwnersEnsureLockedV1( + state *coroNativeFleetPhysicalOwnersV1, + limit uint32, +) bool { + if state == nil || state.lifecycle != coroNativeFleetPhysicalActiveV1 || + state.count+1 != coroNativeFleetV1State.domainCount || + state.started > state.count { + return false + } + desired, ok := coroNativeFleetPhysicalOwnerDesiredPeersV1(limit, state.count) + if !ok { + return false + } + for state.started < desired { + index := state.started peer := &state.peers[index] - handle, ok := coroNativeFleetHandleV1(index + 1) - if !ok || peer.lifecycle != coroNativeFleetPhysicalUnusedV1 || + handle, handleOK := coroNativeFleetHandleV1(index + 1) + if !handleOK || peer.lifecycle != coroNativeFleetPhysicalUnusedV1 || coroNativeAtomicLoadV1(&peer.slot) != 0 || peer.handle != (coro.ExecutorFleetHandle{}) || peer.shutdown { - break + return false } slot, owner, ownerOK := coroNativeMInitialPeerV1(handle) if !ownerOK || slot != handle.Route || owner.thread != nil { - break + return false } peer.handle = handle coroNativeAtomicStoreV1(&peer.slot, slot) peer.lifecycle = coroNativeFleetPhysicalActiveV1 if !coroNativeMStartPhysicalOwnerV1(owner, slot) { - // pthread_create leaves the result slot unspecified on failure. + // pthread_create leaves the result slot unspecified on failure. A + // partially published physical route cannot be reused safely. owner.thread = nil owner.token = 0 peer.lifecycle = coroNativeFleetPhysicalFailedV1 - break + return false } - created++ + state.started++ + } + return true +} + +func coroNativeFleetPhysicalOwnersStartV1(limit uint32) bool { + state := &coroNativeFleetPhysicalOwnerV1State + count := coroNativeFleetV1State.domainCount + if coroNativeFleetV1State.lifecycle != coroNativeFleetActiveV1 || + count == 0 || count > coroNativeFleetDomainCapacityV1 || + state.lifecycle != coroNativeFleetPhysicalUnusedV1 || state.count != 0 || + state.started != 0 || coroNativeAtomicLoadV1(&state.policyEpoch) != 0 || + coroNativeAtomicLoadV1(&state.guard) != 0 || + !state.stop.CanReleaseResources() || !state.stop.Start() { + return false } - if created == state.count { + state.count = count - 1 + coroNativeAtomicStoreV1(&state.policyEpoch, 1) + state.lifecycle = coroNativeFleetPhysicalActiveV1 + if coroNativeFleetPhysicalOwnersEnsureLockedV1(state, limit) { return true } sealed := state.stop.Seal() rang := sealed - for index := uint32(0); index < created; index++ { + for index := uint32(0); index < state.started; index++ { domain, ok := coroNativeFleetDomainForHandleV1( &coroNativeFleetV1State, state.peers[index].handle, @@ -138,7 +197,7 @@ func coroNativeFleetPhysicalOwnersStartV1() bool { rang = rang && ringOK } joined := rang - for index := uint32(0); index < created; index++ { + for index := uint32(0); index < state.started; index++ { peer := &state.peers[index] owner, ownerOK := coroNativeMOwnerForSlotV1( coroNativeAtomicLoadV1(&peer.slot), @@ -155,21 +214,67 @@ func coroNativeFleetPhysicalOwnersStartV1() bool { return false } +// coroNativeFleetSetExecutionLimitV1 serializes physical peer growth with the +// logical quota update. Shrinks park excess peers instead of destroying them; +// later growth reuses the same stable routes without another topology change. +func coroNativeFleetSetExecutionLimitV1(limit uint32) ( + previous uint32, + wake, ok bool, +) { + state := &coroNativeFleetPhysicalOwnerV1State + if limit == 0 || !coroNativeFleetPhysicalOwnersLockV1(state) { + return 0, false, false + } + if !coroNativeFleetPhysicalOwnersEnsureLockedV1(state, limit) { + coroNativeFleetPhysicalOwnersUnlockV1(state) + return 0, false, false + } + previous, wake, ok = coroNativeFleetV1State.execution.SetLimit(limit) + if ok && previous != limit { + epoch := coroNativeAtomicLoadV1(&state.policyEpoch) + if epoch == 0 || epoch == ^uint32(0) { + ok = false + } else { + coroNativeAtomicStoreV1(&state.policyEpoch, epoch+1) + } + } + coroNativeFleetPhysicalOwnersUnlockV1(state) + return previous, wake, ok +} + func coroNativeFleetPhysicalOwnersStopV1() bool { state := &coroNativeFleetPhysicalOwnerV1State + if !coroNativeFleetPhysicalOwnersLockV1(state) { + return false + } if state.lifecycle != coroNativeFleetPhysicalActiveV1 || - state.count+1 != coroNativeFleetV1State.domainCount || !state.stop.Seal() { + state.count+1 != coroNativeFleetV1State.domainCount || + state.started > state.count || !state.stop.Seal() { + coroNativeFleetPhysicalOwnersUnlockV1(state) return false } state.lifecycle = coroNativeFleetPhysicalStoppingV1 - for index := uint32(0); index < state.count; index++ { + started := state.started + coroNativeFleetPhysicalOwnersUnlockV1(state) + for index := uint32(0); index < started; index++ { peer := &state.peers[index] domain, ok := coroNativeFleetDomainForHandleV1( &coroNativeFleetV1State, peer.handle, coroNativeFleetDomainActiveV1, ) + request := coro.ExecutorRequestInvalid + if ok { + // Seal is the durable shutdown fact, but an owner can currently be + // inside an unbounded user resume and therefore cannot observe the + // doorbell until that resume returns. Publish through the executor + // gate as well: compiler safepoints observe this gate and return the + // physical action to the owner loop, which then consumes Seal and + // begins the ordinary sticky-cancellation drain. + request = coroNativeFleetV1State.fleet.RequestExecutor(domain.handle) + } if !ok || peer.lifecycle != coroNativeFleetPhysicalActiveV1 || + !coro.ExecutorRequestAccepted(request) || !domain.doorbell.Ring() { state.lifecycle = coroNativeFleetPhysicalFailedV1 return false @@ -185,7 +290,7 @@ func coroNativeFleetPhysicalOwnersStopV1() bool { return false } } - for index := uint32(0); index < state.count; index++ { + for index := uint32(0); index < started; index++ { peer := &state.peers[index] owner, ownerOK := coroNativeMOwnerForSlotV1( coroNativeAtomicLoadV1(&peer.slot), @@ -426,6 +531,19 @@ func coroNativeFleetRunPhysicalOwnerPassV1( if moved != 0 || more { return true } + domain, domainOK := coroNativeFleetDomainForHandleV1( + &coroNativeFleetV1State, + handle, + coroNativeFleetDomainActiveV1, + ) + if !domainOK { + return coroNativeFleetPhysicalOwnerFailV1("native fleet peer completion window domain lost") + } + if ready, fastOK := coroNativeTryFastWorkerCompletionV1(domain.driverOwnerV1()); !fastOK { + return coroNativeFleetPhysicalOwnerFailV1("native fleet peer completion window failed") + } else if ready { + return true + } freshNow, freshClockOK := coroNativeFleetPhysicalOwnerClockV1() if !freshClockOK { return coroNativeFleetPhysicalOwnerFailV1("native fleet peer wait clock failed") diff --git a/runtime/internal/runtime/coro_native_fleet_program_llgo.go b/runtime/internal/runtime/coro_native_fleet_program_llgo.go index 91e920b9f9..6482006a61 100644 --- a/runtime/internal/runtime/coro_native_fleet_program_llgo.go +++ b/runtime/internal/runtime/coro_native_fleet_program_llgo.go @@ -26,6 +26,10 @@ import "github.com/goplus/llgo/runtime/internal/coro" // references until every route, ingress, backend, and driver is strongly // retired. func coroNativeFleetStartProgramV1(count uint32) bool { + var worker *coro.WorkerOperationSource + if coroProgramWorkerCapabilityV2() { + worker = &coroProgramWorkerSourceV1State + } owners := coroNativeFleetDomainOwnersV1{ p: &coroProgramPV1State, driver: &coroProgramExecutorDriverV1State, @@ -33,7 +37,7 @@ func coroNativeFleetStartProgramV1(count uint32) bool { Timers: &coroProgramTimerTableV1State, Poll: &coroProgramPollSourceV1State, Manual: &coroProgramManualSourceV2State, - Worker: &coroProgramWorkerSourceV1State, + Worker: worker, Channel: &coroProgramChannelSourceV1State, Control: &coroProgramTaskControlSourceV1State, }, diff --git a/runtime/internal/runtime/coro_native_m_owner_llgo.go b/runtime/internal/runtime/coro_native_m_owner_llgo.go index c1ff8006d0..09499a753e 100644 --- a/runtime/internal/runtime/coro_native_m_owner_llgo.go +++ b/runtime/internal/runtime/coro_native_m_owner_llgo.go @@ -338,7 +338,13 @@ func coroNativeMInitialPeerV1( coroNativeAtomicLoadV1(&directory.active[handle.Route-1]) == slot } -func coroNativeMCurrentOwnerV1( +// coroNativeMActiveOwnerV1 resolves the directory owner which is allowed to +// execute this driver's route. It deliberately proves only logical directory +// ownership: callers which are about to mutate physical-thread state must add +// the pthread identity check in coroNativeMCurrentOwnerV1. Keeping these two +// proofs separate lets an ordinary bounded run slice detect the exceptional +// replacement-return capability without calling pthread_self/equal. +func coroNativeMActiveOwnerV1( driver *coro.ExecutorDriver, ) ( owner *coroNativeMOwnerV1, @@ -352,8 +358,7 @@ func coroNativeMCurrentOwnerV1( } slot = coroNativeAtomicLoadV1(&coroNativeMDirectoryV1State.active[uint32(route)-1]) owner, ownerOK := coroNativeMOwnerForSlotV1(slot) - if !ownerOK || owner.handle != domain.handle || owner.self == nil || - pthread.Equal(owner.self, pthread.Self()) == 0 { + if !ownerOK || owner.handle != domain.handle || owner.self == nil { return nil, nil, 0, 0, false } switch coroNativeMOwnerLifecycleLoadV1(owner) { @@ -401,6 +406,21 @@ func coroNativeMCurrentOwnerV1( return owner, domain, slot, epoch, true } +func coroNativeMCurrentOwnerV1( + driver *coro.ExecutorDriver, +) ( + owner *coroNativeMOwnerV1, + domain *coroNativeFleetDomainV1, + slot, epoch uint32, + ok bool, +) { + owner, domain, slot, epoch, ok = coroNativeMActiveOwnerV1(driver) + if !ok || pthread.Equal(owner.self, pthread.Self()) == 0 { + return nil, nil, 0, 0, false + } + return owner, domain, slot, epoch, true +} + func coroNativeMAllocateSuccessorV1( predecessorSlot uint32, predecessor *coroNativeMOwnerV1, diff --git a/runtime/internal/runtime/coro_os_thread_foreign_llgo.go b/runtime/internal/runtime/coro_os_thread_foreign_llgo.go index b0ea9213ea..a86762401a 100644 --- a/runtime/internal/runtime/coro_os_thread_foreign_llgo.go +++ b/runtime/internal/runtime/coro_os_thread_foreign_llgo.go @@ -173,7 +173,16 @@ func (boundary *coroNativeForeignBoundaryV1) reclaimReplacementV1() bool { return false } returnResult := boundary.parent.handoff.RequestReturn(boundary.baton) + request := coro.ExecutorRequestInvalid + if returnResult == coro.ExecutionDomainHandoffReturnClaimed { + // The return baton is the durable fact. The executor request is its + // preemption transport: a replacement may currently be inside an + // unbounded managed resume and cannot observe a doorbell until it first + // reaches the compiler safepoint gate. + request = coroNativeFleetV1State.fleet.RequestExecutor(boundary.domain.handle) + } ringOK := returnResult == coro.ExecutionDomainHandoffReturnClaimed && + coro.ExecutorRequestAccepted(request) && boundary.domain.doorbell.Ring() for ringOK && !boundary.parent.handoff.Returned(boundary.baton) { if corofleet.Yield() != 0 { diff --git a/runtime/internal/runtime/coro_panic_payload.go b/runtime/internal/runtime/coro_panic_payload.go index b66862a57f..57c598b574 100644 --- a/runtime/internal/runtime/coro_panic_payload.go +++ b/runtime/internal/runtime/coro_panic_payload.go @@ -94,7 +94,7 @@ func __llgo_coro_await_prepare_v3(g, parent, child unsafe.Pointer, mode uint32, switch mode { case 0: if typeWord != nil || dataWord != nil || - !coro.PrepareAwaitCompletion((*coro.G)(g), parent, child) { + !coro.PrepareAwaitCompletionCompiler((*coro.G)(g), parent, child) { coroRuntimeAbort("invalid ordinary coroutine child completion handoff") } case 1: diff --git a/runtime/internal/runtime/coro_program.go b/runtime/internal/runtime/coro_program.go index 5e730ddd7b..adedee5afb 100644 --- a/runtime/internal/runtime/coro_program.go +++ b/runtime/internal/runtime/coro_program.go @@ -122,6 +122,7 @@ var ( coroProgramLifecycleV1State coroProgramLifecycleV1 coroProgramManifestV1State *coro.ProgramManifestV1 coroProgramFactoryV1State unsafe.Pointer + coroProgramCapabilitiesV2State coro.ProgramCapabilitiesV2 coroProgramGV1State coroG coroProgramPV1State coroP coroProgramContinuationV1State coroProgramContinuationV1 @@ -251,11 +252,17 @@ func coroProgramBeginOwnedV1(manifest, expectedFactory unsafe.Pointer) (unsafe.P return nil, false } programManifest := (*coro.ProgramManifestV1)(manifest) - _, v2Code := coro.ValidateRunnableProgramV2(programManifest, expectedFactory) + program, v2Code := coro.ValidateRunnableProgramV2(programManifest, expectedFactory) if v2Code != coro.ProgramValidationOKV2 { coroProgramLifecycleV1State = coroProgramFailedV1 return nil, false } + capabilities, capabilityCode := coro.ResolveProgramCapabilitiesV2(program) + if capabilityCode != coro.ProgramValidationOKV2 || !capabilities.Valid() { + coroProgramLifecycleV1State = coroProgramFailedV1 + return nil, false + } + coroProgramCapabilitiesV2State = capabilities if !coroInitG(&coroProgramGV1State) || !coro.BindRunnableOwner(&coroProgramGV1State) || !coroProgramBindExecutorV1() || @@ -269,6 +276,10 @@ func coroProgramBeginOwnedV1(manifest, expectedFactory unsafe.Pointer) (unsafe.P return unsafe.Pointer(&coroProgramGV1State), true } +func coroProgramWorkerCapabilityV2() bool { + return coroProgramCapabilitiesV2State.Valid() && coroProgramCapabilitiesV2State.Worker() +} + func coroProgramBeginV1(manifest, expectedFactory unsafe.Pointer) (unsafe.Pointer, bool) { if !coroProgramDriveAdmissionV1State.Acquire() { return nil, false diff --git a/runtime/internal/runtime/coro_program_test.go b/runtime/internal/runtime/coro_program_test.go index 7d150039b0..56196e057f 100644 --- a/runtime/internal/runtime/coro_program_test.go +++ b/runtime/internal/runtime/coro_program_test.go @@ -598,6 +598,7 @@ func resetCoroProgramTestStateV1(t *testing.T) { coroProgramLifecycleV1State = coroProgramUnusedV1 coroProgramManifestV1State = nil coroProgramFactoryV1State = nil + coroProgramCapabilitiesV2State = 0 coroProgramGV1State = coroG{} coroProgramPV1State = coroP{} coroProgramContinuationV1State = coroProgramContinuationNoneV1 @@ -619,6 +620,7 @@ func resetCoroProgramTestStateV1(t *testing.T) { coroProgramLifecycleV1State = coroProgramUnusedV1 coroProgramManifestV1State = nil coroProgramFactoryV1State = nil + coroProgramCapabilitiesV2State = 0 coroProgramGV1State = coroG{} coroProgramPV1State = coroP{} coroProgramContinuationV1State = coroProgramContinuationNoneV1 @@ -663,7 +665,7 @@ func TestCoroProgramV1BeginRunAndDestroy(t *testing.T) { if coroProgramLifecycleV1State != coroProgramCompleteV1 || !coro.TerminalG(&coroProgramPV1State, &coroProgramGV1State) { t.Fatalf("completed coroutine program retained scheduler state: lifecycle=%d", coroProgramLifecycleV1State) } - if driver.doneCalls != 2 || driver.resumeCalls != 1 || driver.destroyCalls != 1 || !driver.released { + if driver.doneCalls != 1 || driver.resumeCalls != 1 || driver.destroyCalls != 1 || !driver.released { t.Fatalf("coroutine wrapper calls = done:%d resume:%d destroy:%d released:%t", driver.doneCalls, driver.resumeCalls, driver.destroyCalls, driver.released) } if !coroProgramTestTargetV1State.joined || coroProgramTestTargetV1State.closeCalls != 1 || @@ -804,7 +806,7 @@ func TestCoroProgramRunSliceBudgetOneKeepsPhysicalActionsAtomic(t *testing.T) { } } if dispatches != 2 || resumes != 1 || destroys != 1 || - driver.doneCalls != 2 || driver.resumeCalls != 1 || driver.destroyCalls != 1 || !driver.released { + driver.doneCalls != 1 || driver.resumeCalls != 1 || driver.destroyCalls != 1 || !driver.released { t.Fatalf("budget-one totals = source:%d dispatch:%d resume:%d destroy:%d wrappers={done:%d resume:%d destroy:%d released:%t}", sources, dispatches, resumes, destroys, driver.doneCalls, driver.resumeCalls, driver.destroyCalls, driver.released) } @@ -1325,6 +1327,7 @@ func TestCoroProgramRunSliceV2ConcurrentDuplicateEpochIsExactOnce(t *testing.T) close(start) var winner coroProgramRunResultV2 winners := 0 + unexpected := make([]callbackResult, 0, callers) for index := 0; index < callers; index++ { got := <-results switch got.status { @@ -1339,12 +1342,19 @@ func TestCoroProgramRunSliceV2ConcurrentDuplicateEpochIsExactOnce(t *testing.T) } case uint32(coroProgramDriveIgnoredV2): if got.result != (coroProgramRunResultV2{}) { - t.Fatalf("ignored duplicate retained result = %+v", got.result) + unexpected = append(unexpected, got) } default: - t.Fatalf("duplicate V2 callback status = %d result:%+v", got.status, got.result) + unexpected = append(unexpected, got) } } + if len(unexpected) != 0 { + t.Fatalf("duplicate V2 callback unexpected results = %+v, lifecycle:%d continuation:%d epoch:%d mode:%d consumes:%d admission-release:%t", + unexpected, coroProgramLifecycleV1State, coroProgramContinuationV1State, + coroProgramContinuationEpochV1, coroProgramDriverModeV2State, + coroProgramTestTargetV1State.runConsumeCalls, + coroProgramDriveAdmissionV1State.CanRelease()) + } if winners != 1 || winner.Epoch == 0 || winner.Epoch == initial.Epoch || coroProgramContinuationEpochV1 != winner.Epoch || coroProgramTestTargetV1State.runConsumeCalls != 1 || @@ -1424,7 +1434,7 @@ func TestCoroProgramV2BeginRunAndDestroy(t *testing.T) { if coroProgramLifecycleV1State != coroProgramCompleteV1 || !coro.TerminalG(&coroProgramPV1State, &coroProgramGV1State) { t.Fatalf("completed coroutine program v2 retained scheduler state: lifecycle=%d", coroProgramLifecycleV1State) } - if driver.doneCalls != 2 || driver.resumeCalls != 1 || driver.destroyCalls != 1 || !driver.released { + if driver.doneCalls != 1 || driver.resumeCalls != 1 || driver.destroyCalls != 1 || !driver.released { t.Fatalf("coroutine v2 wrapper calls = done:%d resume:%d destroy:%d released:%t", driver.doneCalls, driver.resumeCalls, driver.destroyCalls, driver.released) } if !coroProgramTestTargetV1State.joined || coroProgramTestTargetV1State.closeCalls != 1 || @@ -1695,7 +1705,7 @@ func TestCoroProgramExplicitPanicHookAndTerminalDispatcher(t *testing.T) { t.Fatalf("terminal adapter panic record = (%+v, %t)", record, published) } if coroProgramLifecycleV1State != coroProgramFailedV1 || - driver.doneCalls != 2 || driver.resumeCalls != 1 || driver.destroyCalls != 1 || !driver.released || + driver.doneCalls != 1 || driver.resumeCalls != 1 || driver.destroyCalls != 1 || !driver.released || coro.TerminalG(&coroProgramPV1State, &coroProgramGV1State) || coro.ReclaimableG(&coroProgramGV1State) || !coroProgramTestTargetV1State.joined || coroProgramTestTargetV1State.closeCalls != 1 || coroProgramExecutorBoundV1State || !coroProgramExecutorRegistryV1State.CanRelease() { @@ -1859,7 +1869,7 @@ func TestCoroProgramNormalMainReturnCancelsReadyChild(t *testing.T) { if status := coroProgramRunV1(gPointer, frame.handle); status != coroProgramDriveCompleteV1 { t.Fatalf("run command-shutdown program = %d", status) } - if coroProgramLifecycleV1State != coroProgramCompleteV1 || driver.doneCalls != 2 || + if coroProgramLifecycleV1State != coroProgramCompleteV1 || driver.doneCalls != 1 || driver.resumeCalls != 1 || driver.destroyCalls != 1 || driver.cancelDestroyCalls != 1 || driver.taskReleaseCalls != 1 || driver.child == nil || driver.childFrame == nil || !coroProgramTestTargetV1State.joined || coroProgramTestTargetV1State.closeCalls != 1 || @@ -1891,8 +1901,8 @@ func TestCoroProgramMainReturnCancelsBoundedChildDestroyContinuation(t *testing. t.Fatalf("run bounded-child command program = %d", status) } if coroProgramLifecycleV1State != coroProgramCompleteV1 || - driver.doneCalls != 3 || driver.resumeCalls != 2 || driver.destroyCalls != 1 || - driver.childDoneCalls != 1 || driver.childResumeCalls != 1 || !driver.childCompleteReady || + driver.doneCalls != 1 || driver.resumeCalls != 2 || driver.destroyCalls != 1 || + driver.childDoneCalls != 0 || driver.childResumeCalls != 1 || !driver.childCompleteReady || driver.cancelDestroyCalls != 1 || driver.taskReleaseCalls != 1 || driver.child == nil || driver.childFrame == nil || !coroProgramTestTargetV1State.joined || coroProgramTestTargetV1State.closeCalls != 1 || @@ -1948,8 +1958,8 @@ func TestCoroProgramCommandBootstrapDirectChildHandoffPrecedesTwoPeers(t *testin } } if coroProgramLifecycleV1State != coroProgramCompleteV1 || - driver.doneCalls != 3 || driver.resumeCalls != 2 || driver.destroyCalls != 1 || !driver.released || - driver.bootstrapChildDoneCalls != 2 || driver.bootstrapChildResumeCalls != 1 || + driver.doneCalls != 1 || driver.resumeCalls != 2 || driver.destroyCalls != 1 || !driver.released || + driver.bootstrapChildDoneCalls != 1 || driver.bootstrapChildResumeCalls != 1 || driver.bootstrapChildDestroyCalls != 1 || driver.bootstrapPeerDestroyCalls != 2 || driver.taskReleaseCalls != 2 || !coroProgramTestTargetV1State.joined || coroProgramTestTargetV1State.closeCalls != 1 || coroProgramExecutorBoundV1State || diff --git a/runtime/internal/runtime/coro_ready_distribution_default.go b/runtime/internal/runtime/coro_ready_distribution_default.go index c67c50de19..b2a23f8e83 100644 --- a/runtime/internal/runtime/coro_ready_distribution_default.go +++ b/runtime/internal/runtime/coro_ready_distribution_default.go @@ -20,6 +20,21 @@ package runtime import "github.com/goplus/llgo/runtime/internal/coro" +func coroTargetBeginRunSliceV1( + *coro.P, + *coro.ExecutorDriver, +) (coroRunTargetCapabilityV1, bool) { + return coroRunTargetCapabilityV1{}, true +} + +func coroTargetReadyDistributionV1(coroRunTargetCapabilityV1) (distribute, stop, ok bool) { + return false, false, true +} + +func coroTargetRefreshRunSliceV1(coroRunTargetCapabilityV1) (distribute, restart, ok bool) { + return false, false, true +} + func coroTargetAfterStableRunActionV1(*coro.P, *coro.ExecutorDriver) bool { return true } @@ -66,7 +81,7 @@ func coroTargetHandleOSThreadSuspendV1( return false } -func coroTargetStopForOSThreadReturnV1(*coro.ExecutorDriver) (bool, bool) { +func coroTargetStopForPhysicalReturnV1(*coro.ExecutorDriver) (bool, bool) { return false, true } diff --git a/runtime/internal/runtime/coro_ready_distribution_fleet_llgo.go b/runtime/internal/runtime/coro_ready_distribution_fleet_llgo.go index 488f91d402..8b412ceec8 100644 --- a/runtime/internal/runtime/coro_ready_distribution_fleet_llgo.go +++ b/runtime/internal/runtime/coro_ready_distribution_fleet_llgo.go @@ -32,15 +32,103 @@ func coroTargetReadyDistributionDomainV1( if p == nil || driver == nil { return nil, false } + domain, _, ok := coroNativeFleetExecutionDomainV1(driver) + return domain, ok && domain.pOwnerV1() == p +} + +// coroTargetReadyDistributionEnabledV1 keeps runnable ownership local when +// managed Go execution is serial. The native fleet intentionally keeps every +// physical route alive so it can continue to service timer, poll, channel and +// cancellation sources, but moving a runnable between those routes cannot add +// execution capacity while GOMAXPROCS is one. Apart from the mailbox and +// doorbell traffic, such a move also loses the warm owner-local scheduler +// state and makes the next route contend for the sole execution lease. +// +// Limit is a live observation rather than startup policy: increasing +// GOMAXPROCS immediately re-enables surplus sharing, while shrinking it stops +// new transfers without disturbing work that was already published. +func coroTargetReadyDistributionEnabledV1(state *coroNativeFleetStateV1) (bool, bool) { + if state == nil { + return false, false + } + limit, ok := state.execution.Limit() + return limit > 1, ok +} + +// coroTargetBeginRunSliceV1 freezes policy which is safe to observe at a +// bounded scheduler-stack entry. GOMAXPROCS changes take effect on the next +// slice (at most 64 reductions for compatibility runners); the execution quota +// itself remains the exact concurrency gate immediately, so caching only the +// placement policy cannot exceed a shrunken limit. A negative LockOSThread +// handoff observation is exact until this slice itself creates a detached +// handoff, and that transition returns from the slice immediately. +func coroTargetBeginRunSliceV1( + source *coro.P, + driver *coro.ExecutorDriver, +) (coroRunTargetCapabilityV1, bool) { + osThreadPossible, osThreadOK := coro.OSThreadSuspendHandoffPossible(driver) + if source == nil || driver == nil || !osThreadOK { + return coroRunTargetCapabilityV1{}, false + } + target := coroRunTargetCapabilityV1{physicalReturn: osThreadPossible} state := &coroNativeFleetV1State - for index := uint32(0); index < state.domainCount; index++ { - domain := &state.domains[index] - if domain.lifecycle == coroNativeFleetDomainActiveV1 && - domain.pOwnerV1() == p && domain.driverOwnerV1() == driver { - return domain, true + if coroNativeFleetPhysicalOwnerV1State.stop.Quiesced() { + return target, true + } + if state.lifecycle != coroNativeFleetActiveV1 { + return coroRunTargetCapabilityV1{}, false + } + if _, ok := coroTargetReadyDistributionDomainV1(source, driver); !ok { + return coroRunTargetCapabilityV1{}, false + } + policy := &coroNativeFleetPhysicalOwnerV1State.policyEpoch + for { + epoch := coroNativeAtomicLoadV1(policy) + distributionEnabled, distributionOK := coroTargetReadyDistributionEnabledV1(state) + if !distributionOK || epoch == 0 { + return coroRunTargetCapabilityV1{}, false + } + if coroNativeAtomicLoadV1(policy) != epoch { + continue } + target.readyDistribution = distributionEnabled + target.policyEpoch = epoch + break + } + replacementPossible, replacementOK := coroTargetReplacementReturnPossibleV1(driver) + if !replacementOK { + return coroRunTargetCapabilityV1{}, false + } + target.physicalReturn = target.physicalReturn || replacementPossible + return target, true +} + +// The stop word is live because program-main return may publish it while a +// distribution-capable peer is inside this very slice. A serial slice skips +// this post-action hook entirely: shutdown also publishes the executor request +// which makes its next bounded source reduction observe stop, while a physical +// return candidate has the separate exact hook. Ready placement is the bounded +// slice capability above; revalidating the complete quota lifetime and limit +// after every action adds no safety because TryAcquire remains authoritative. +func coroTargetReadyDistributionV1(target coroRunTargetCapabilityV1) (distribute, stop, ok bool) { + stop = coroNativeFleetPhysicalOwnerV1State.stop.Quiesced() + return target.readyDistribution && !stop, stop, true +} + +// A successful quota resize executes an exact compiler yield before its Go +// caller can continue. That rare action is the synchronization point for this +// policy epoch; ordinary channel/park/complete actions do not read it. An +// unrelated explicit yield with an unchanged epoch keeps the current slice. +func coroTargetRefreshRunSliceV1(target coroRunTargetCapabilityV1) (distribute, restart, ok bool) { + stop := coroNativeFleetPhysicalOwnerV1State.stop.Quiesced() + epoch := coroKeyedAtomicLoadUint32(&coroNativeFleetPhysicalOwnerV1State.policyEpoch) + enabled, valid := coroTargetReadyDistributionEnabledV1(&coroNativeFleetV1State) + if !valid || target.policyEpoch == 0 || epoch == 0 { + return false, false, false } - return nil, false + return target.readyDistribution && enabled && !stop, + stop || epoch != target.policyEpoch || enabled != target.readyDistribution, + true } // coroTargetAfterStableRunActionV1 is owner-to-owner work distribution, not a @@ -214,6 +302,9 @@ func coroTargetPrepareOSThreadSuspendV1( action.Handle != nil || action.Flags != 0 { return false, false } + if !coro.OSThreadSuspendHandoffCandidate(task) { + return false, true + } return coro.PrepareOSThreadSuspendHandoff( driver, task, action.Kind, ) @@ -235,16 +326,49 @@ func coroNativeAbortOSThreadSuspendV1( coro.AbortOSThreadSuspendHandoff(driver, task) } -// coroTargetStopForOSThreadReturnV1 is the exact stable-reduction gate for a -// compensation M. A detached Yield stops after its first complete peer Action; -// a detached Park stops as soon as source service promotes the locked owner. -// Keeping this observation inside the common runner lets source transactions -// retain their normal batch budget without crossing the return boundary. -func coroTargetStopForOSThreadReturnV1( +func coroTargetReplacementReturnPossibleV1( + driver *coro.ExecutorDriver, +) (bool, bool) { + owner, _, _, _, ok := coroNativeMActiveOwnerV1(driver) + if !ok { + return false, false + } + if !owner.baton.Valid() { + return false, true + } + parent, parentOK := coroNativeMOwnerForSlotV1(owner.parentSlot) + return parentOK && parent.handle == owner.handle, parentOK +} + +// coroTargetStopForPhysicalReturnV1 is the exact stable-reduction gate for a +// compensation M. An ordinary detached Yield/Park uses the P-local handoff; +// an active foreign-call replacement uses its parent M's atomic return baton. +// Both stop only after a complete reducer commit, so no physical action or +// source transaction is split. Ordinary owners skip this hook through the +// slice-entry negative capability above. +func coroTargetStopForPhysicalReturnV1( driver *coro.ExecutorDriver, ) (bool, bool) { - detached, returnable, ok := coro.OSThreadSuspendHandoffStatus(driver) - return detached && returnable, ok + possible, ok := coro.OSThreadSuspendHandoffPossible(driver) + if !ok { + return false, false + } + if possible { + detached, returnable, statusOK := coro.OSThreadSuspendHandoffStatus(driver) + return detached && returnable, statusOK + } + owner, _, _, _, ownerOK := coroNativeMCurrentOwnerV1(driver) + if !ownerOK || !owner.baton.Valid() { + return false, ownerOK + } + parent, parentOK := coroNativeMOwnerForSlotV1(owner.parentSlot) + if !parentOK || parent.handle != owner.handle { + return false, false + } + if !parent.handoff.ReturnRequested(owner.baton) { + return false, true + } + return coro.ExecutorResumeHandoffReturnable(driver), true } // coroTargetHandleOSThreadSuspendV1 temporarily blocks the original M on diff --git a/runtime/internal/runtime/coro_resume_materialize.go b/runtime/internal/runtime/coro_resume_materialize.go index 573845b391..613d9700c6 100644 --- a/runtime/internal/runtime/coro_resume_materialize.go +++ b/runtime/internal/runtime/coro_resume_materialize.go @@ -106,8 +106,9 @@ type coroKeyedRegistrySlotV2 struct { } type coroKeyedRegistryV2 struct { - sequence uint32 - slots [coroKeyedRegistryCapacityV2]coroKeyedRegistrySlotV2 + sequence uint32 + scanLimit uint32 + slots [coroKeyedRegistryCapacityV2]coroKeyedRegistrySlotV2 } var coroProgramKeyedRegistryV2State coroKeyedRegistryV2 diff --git a/runtime/internal/runtime/coro_run_decision.go b/runtime/internal/runtime/coro_run_decision.go index a263de75ac..1802a07ed6 100644 --- a/runtime/internal/runtime/coro_run_decision.go +++ b/runtime/internal/runtime/coro_run_decision.go @@ -89,7 +89,7 @@ func zeroTicketCoroRunDecisionTaskV1( // //export __llgo_coro_run_decision_take_zero_v1 func __llgo_coro_run_decision_take_zero_v1(g unsafe.Pointer) uint32 { - outcome, caseID, taskKind, sourceSlot, generation, ok := coro.TakeRunDecisionWords((*coro.G)(g), 0, 0) + outcome, caseID, taskKind, sourceSlot, generation, ok := coro.TakeRunDecisionWordsCompiler((*coro.G)(g), 0, 0) task, valid := zeroTicketCoroRunDecisionTaskV1(outcome, caseID, taskKind, sourceSlot, generation, ok) if !valid { coroRuntimeAbort("invalid zero-ticket coroutine run decision") @@ -125,7 +125,7 @@ func __llgo_coro_run_decision_take_v1( return } if mode == coroRunDecisionOutputNormalOnlyV1 { - decisionOutcome, selectedCase, cancelKind, sourceSlot, generation, ok := coro.TakeRunDecisionWords((*coro.G)(g), 0, 0) + decisionOutcome, selectedCase, cancelKind, sourceSlot, generation, ok := coro.TakeRunDecisionWordsCompiler((*coro.G)(g), 0, 0) if !normalCoroRunDecisionWordsV1(decisionOutcome, selectedCase, cancelKind, sourceSlot, generation, ok) { coroRuntimeAbort("unsupported non-normal coroutine run decision") } @@ -136,7 +136,7 @@ func __llgo_coro_run_decision_take_v1( *taskKind = 0 *operationSourceSlot = 0 *operationGeneration = 0 - decisionOutcome, selectedCase, cancelKind, sourceSlot, generation, ok := coro.TakeRunDecisionWords( + decisionOutcome, selectedCase, cancelKind, sourceSlot, generation, ok := coro.TakeRunDecisionWordsCompiler( (*coro.G)(g), expectedEpoch, expectedGeneration, ) if !ok { diff --git a/runtime/internal/runtime/coro_run_slice.go b/runtime/internal/runtime/coro_run_slice.go index f1a0722f62..9727876498 100644 --- a/runtime/internal/runtime/coro_run_slice.go +++ b/runtime/internal/runtime/coro_run_slice.go @@ -39,6 +39,14 @@ func coroHandleResume(unsafe.Pointer) //go:linkname coroHandleDestroy C.__llgo_coro_destroy_v1 func coroHandleDestroy(unsafe.Pointer) +func coroHandleDestroyCommitted(g *coro.G, handle unsafe.Pointer) bool { + if g == nil || handle == nil { + return false + } + coroHandleDestroy(handle) + return coro.CommitFrameDestroyCompiler(g, handle) +} + // __llgo_coro_await_inline_v1 resumes one already prepared child on the // current executor stack. A synchronous completion is destroyed and committed // here; a real suspension returns false so generated parents unwind through @@ -62,7 +70,10 @@ func __llgo_coro_await_inline_v1(g, parent, child unsafe.Pointer) bool { case coro.InlineAwaitSuspend: return false case coro.InlineAwaitDestroy: - coroHandleDestroy(child) + if !coroHandleDestroyCommitted(task, child) { + coroRuntimeAbort("invalid coroutine inline child physical destroy") + return false + } if !coro.CommitInlineAwaitDestroy(task, parent, child) { coroRuntimeAbort("invalid coroutine inline child destroy commit") return false @@ -74,6 +85,44 @@ func __llgo_coro_await_inline_v1(g, parent, child unsafe.Pointer) bool { } } +// __llgo_coro_await_inline_begin_v2 owns only the scheduler-state half of an +// eager static child handoff. Keeping llvm.coro.resume/done/destroy in the +// generated caller exposes the exact handle lifetime to LLVM 22's annotated +// frame-elision pass. +// +//export __llgo_coro_await_inline_begin_v2 +func __llgo_coro_await_inline_begin_v2(g, parent, child unsafe.Pointer) bool { + switch coro.BeginInlineAwaitCompiler((*coro.G)(g), parent, child) { + case coro.InlineAwaitDeclined: + return false + case coro.InlineAwaitStarted: + return true + default: + coroRuntimeAbort("invalid coroutine inline child begin") + return false + } +} + +//export __llgo_coro_await_inline_finish_v2 +func __llgo_coro_await_inline_finish_v2(g, parent, child unsafe.Pointer, done bool) bool { + switch coro.FinishInlineAwaitCompiler((*coro.G)(g), parent, child, done) { + case coro.InlineAwaitSuspend: + return false + case coro.InlineAwaitDestroy: + return true + default: + coroRuntimeAbort("invalid coroutine inline child finish") + return false + } +} + +//export __llgo_coro_await_inline_destroy_commit_v2 +func __llgo_coro_await_inline_destroy_commit_v2(g, parent, child unsafe.Pointer) { + if !coro.CommitInlineAwaitDestroyCompiler((*coro.G)(g), parent, child) { + coroRuntimeAbort("invalid coroutine inline child destroy commit") + } +} + type coroProgramLifecycleV1 uint8 const ( @@ -141,6 +190,19 @@ type coroRunPolicyV1 struct { lifecycle *coroProgramLifecycleV1 } +// coroRunTargetCapabilityV1 is sampled once at the stable entry to a bounded +// run slice. readyDistribution certifies immutable fleet-domain ownership and +// policyEpoch identifies its placement-policy observation. Target glue leaves +// policyEpoch zero when it has no dynamically mutable placement policy. +// physicalReturn is an exact negative capability: it is enabled only for an +// attached lock island or while this M is a claimed replacement. Ordinary +// owners therefore pay no post-reduction return-policy hook. +type coroRunTargetCapabilityV1 struct { + readyDistribution bool + physicalReturn bool + policyEpoch uint32 +} + // coroRuntimeContextActivationV1 distinguishes a physical-thread install from // a nested resume which borrows the same already-installed logical G. The // latter occurs when C synchronously calls Go while the parent LLVM resume is @@ -154,56 +216,54 @@ func (policy coroRunPolicyV1) valid() bool { return policy.main == nil && policy.lifecycle == nil || policy.main != nil && policy.lifecycle != nil } -func (policy coroRunPolicyV1) commandState() (running, returnRequested, ok bool) { - if policy.main == nil && policy.lifecycle == nil { - return false, false, true - } - if policy.main == nil || policy.lifecycle == nil { - return false, false, false - } - switch *policy.lifecycle { - case coroProgramRunningV1: - return true, false, true - case coroProgramMainGoexitV1: - return true, false, true - case coroProgramMainReturnRequestedV1: - return false, true, true - default: - return false, false, false - } -} - // coroRunPhysicalActionV1 is the indivisible runtime half of one runner action // reduction. Neither Checked's ActionResume/ActionDestroy nor a freed handle is // observable at a reducer return boundary. -func coroRunPhysicalActionV1(p *coro.P, g *coro.G, action coro.Action) (coro.Action, bool) { +func coroRunPhysicalActionV1( + p *coro.P, + driver *coro.ExecutorDriver, + g *coro.G, + action coro.Action, + runtimeContext unsafe.Pointer, +) (next coro.Action, advanced, committed bool) { switch action.Kind { case coro.ActionCheckResume: - next, ok := coro.Checked(p, g, action, coroHandleDone(action.Handle)) - if !ok || next.Kind != coro.ActionResume || next.Handle != action.Handle { - return coro.Action{}, false - } - activation, entered := coroEnterRuntimeContext(g) - if !entered { - return coro.Action{}, false + next, needsRuntimeContext, ok := coro.BeginIssuedExecutorResumeRuntimeContext(driver, g) + if !ok { + return coro.Action{}, false, false } - coroHandleResume(next.Handle) - if !coroLeaveRuntimeContext(g, activation) { - return coro.Action{}, false + if needsRuntimeContext { + activation, entered := coroEnterRuntimeContextFrom(g, runtimeContext) + if !entered { + return coro.Action{}, false, false + } + coroHandleResume(next.Handle) + if !coroLeaveRuntimeContext(g, activation) { + return coro.Action{}, false, false + } + } else { + coroHandleResume(next.Handle) } - return coro.Resumed(p, g, next) + next, committed, advanced = coro.ResumedExecutorRun(driver, p, g, next) + return next, advanced, committed case coro.ActionCheckDestroy: - next, ok := coro.Checked(p, g, action, coroHandleDone(action.Handle)) + next, ok := coro.CheckedExecutorRun(driver, g, action, coroHandleDone(action.Handle)) if !ok || next.Kind != coro.ActionDestroy || next.Handle != action.Handle { - return coro.Action{}, false + return coro.Action{}, false, false } - coroHandleDestroy(next.Handle) - return coro.DestroyedBounded(p, g, next) + if !coroHandleDestroyCommitted(g, next.Handle) { + return coro.Action{}, false, false + } + next, advanced = coro.DestroyedBounded(p, g, next) + return next, advanced, false case coro.ActionPanicDestroy: - coroHandleDestroy(action.Handle) - return coro.PanicDestroyedBounded(p, g, action) + if !coroHandleDestroyCommitted(g, action.Handle) { + return coro.Action{}, false, false + } + next, advanced = coro.PanicDestroyedBounded(p, g, action) + return next, advanced, false default: - return coro.Action{}, false + return coro.Action{}, false, false } } @@ -214,13 +274,13 @@ func coroRunPhysicalActionV1(p *coro.P, g *coro.G, action coro.Action) (coro.Act // P, rather than a fresh quota transaction around every llvm.coro.resume. // wait=true is an ordinary stable scheduler-stack return: the route keeps all // runnable and source ownership and waits only for another P publication. -func coroPrepareManagedExecutionV1(driver *coro.ExecutorDriver, held bool) (nextHeld, wait, ok bool) { +func coroPrepareManagedExecutionV1(driver *coro.ExecutorDriver) (nextHeld, wait, ok bool) { pending, valid := coro.ExecutorRunManagedResumePending(driver) if !valid { - return held, false, false + return false, false, false } - if held || !pending { - return held, false, true + if !pending { + return false, false, true } acquired, valid := coroTargetAcquireManagedExecutionV1(driver) if !valid { @@ -239,6 +299,10 @@ func coroStepMatchesManagedExecutionV1(step coro.ExecutorRunStep, held bool) boo return !resume || held } +func coroActionStepMatchesManagedExecutionV1(step coro.ExecutorRunActionStep, held bool) bool { + return step.Action.Kind != coro.ActionCheckResume || held +} + // coroStopAfterStableReductionV1 is the common post-reducer target gate for // both the adopted program P and ordinary fleet Ps. It is called only after // the complete reduction at a stable scheduler boundary. The bounded run slice @@ -253,7 +317,7 @@ func coroStopAfterStableReductionV1( if driver == nil || result == nil { return false, false } - stop, ok = coroTargetStopForOSThreadReturnV1(driver) + stop, ok = coroTargetStopForPhysicalReturnV1(driver) if !ok || !stop { return stop, ok } @@ -261,6 +325,160 @@ func coroStopAfterStableReductionV1( return true, true } +// coroReduceExecutorRunActionPreparedV1 consumes the compact hot action ABI. +// The caller has already sampled command-return state for the dispatch gate; +// the physical resume may change it, so the post-resume sample remains the +// commit authority exactly as in the general reducer. +func coroReduceExecutorRunActionPreparedV1( + p *coro.P, + driver *coro.ExecutorDriver, + policy coroRunPolicyV1, + target coroRunTargetCapabilityV1, + g *coro.G, + action coro.Action, + dispatched bool, + returnRequested bool, + runtimeContext unsafe.Pointer, + result *coroRunResultV1, +) (terminal, ok bool) { + if g == nil || action.Handle == nil || runtimeContext == nil { + return false, false + } + if dispatched { + if returnRequested && g != policy.main && action.Kind == coro.ActionCheckResume && + !coro.RequestTaskCancellation(p, g, coro.TaskCancelShutdown) { + return false, false + } + result.dispatches++ + } + next, advanced, committed := coroRunPhysicalActionV1(p, driver, g, action, runtimeContext) + // The physical resume may have changed program lifecycle. Re-read the live + // policy before selecting the scheduler commit placement. + running, returnRequested := false, false + if policy.main != nil { + if policy.lifecycle == nil { + return false, false + } + switch *policy.lifecycle { + case coroProgramRunningV1, coroProgramMainGoexitV1: + running = true + case coroProgramMainReturnRequestedV1: + returnRequested = true + default: + return false, false + } + } else if policy.lifecycle != nil { + return false, false + } + if advanced && !committed && g == policy.main && returnRequested && next.Kind == coro.ActionCheckDestroy { + committed = coro.CommitExecutorRunCommandRootDestroy(driver, g, next) + } else if advanced && !committed && g == policy.main && running && + coro.CommitExecutorRunCommandBootstrapDirectChildHandoff(driver, g, next) { + committed = true + } else if advanced && !committed { + committed = coro.CommitExecutorRunAction(driver, g, next) + } + if !committed { + return false, false + } + if next.Kind == coro.ActionForeignReentryComplete { + result.used++ + if dispatched { + result.used++ + } + result.destroys++ + result.stop = coroRunForeignReentryCompleteV1 + result.g = g + result.action = next + return true, true + } + // A locked ordinary suspension must decide whether to detach before ready + // distribution can move the peer which justifies a Yield handoff. The + // physical resume may itself call LockOSThread, so the slice-entry target + // capability is not sufficient here. The committed G is nevertheless the + // exact candidate: avoid the target adapter for the common zero-depth task, + // and retain its complete validation for every dynamically locked task. + osThreadSuspend := false + if coro.OSThreadSuspendHandoffCandidate(g) { + var suspendOK bool + osThreadSuspend, suspendOK = coroTargetPrepareOSThreadSuspendV1( + p, driver, g, next, + ) + if !suspendOK { + return false, false + } + } + stopAfterStable := false + if !osThreadSuspend && (target.readyDistribution || next.Kind == coro.ActionYield) { + distribute, stop, distributionOK := false, false, false + if next.Kind == coro.ActionYield { + distribute, stop, distributionOK = coroTargetRefreshRunSliceV1(target) + } else { + distribute, stop, distributionOK = coroTargetReadyDistributionV1(target) + } + if !distributionOK || distribute && !coroTargetAfterStableRunActionV1(p, driver) { + return false, false + } + stopAfterStable = stop + } + result.used++ + if dispatched { + result.used++ + } + switch action.Kind { + case coro.ActionCheckResume: + result.resumes++ + case coro.ActionCheckDestroy, coro.ActionPanicDestroy: + result.destroys++ + } + if osThreadSuspend { + result.stop = coroRunOSThreadSuspendV1 + result.g = g + result.action = next + return true, true + } + switch next.Kind { + case coro.ActionCheckResume, coro.ActionCheckDestroy, coro.ActionPanicDestroy: + if g == policy.main && returnRequested && next.Kind == coro.ActionCheckResume { + return false, false + } + case coro.ActionYield, coro.ActionPark: + if g == policy.main && returnRequested { + return false, false + } + case coro.ActionComplete: + isMain := g == policy.main + retireOwner := coro.ActionRetiresPhysicalOwner(next) + if !coroReleaseCompletedTask(g) { + return false, false + } + if isMain { + result.stop, result.g = coroRunMainDoneV1, policy.main + return true, true + } + if retireOwner && !coroTargetRetirePhysicalOwnerV1(p, driver) { + return false, false + } + case coro.ActionPanicComplete: + result.stop, result.g, result.action = coroRunPanicCompleteV1, g, next + return true, true + case coro.ActionCommitDestroy: + result.stop, result.g, result.action = coroRunDestroyCommitV1, g, next + return true, true + default: + return false, false + } + if stopAfterStable { + // The native fleet uses the same already-paid target observation that + // controls ready distribution to expose its durable stop boundary. Return + // only after this complete reduction so the physical owner can publish + // sticky shutdown cancellation before any user continuation is resumed. + result.stop = coroRunAgainV1 + return true, true + } + return false, true +} + // coroReduceExecutorRunStepV1 is the single physical scheduler reducer shared // by the process program and every fleet domain. It consumes exactly one step; // terminal reports a stable slice boundary, while ok=false invalidates the @@ -270,25 +488,44 @@ func coroReduceExecutorRunStepV1( p *coro.P, driver *coro.ExecutorDriver, policy coroRunPolicyV1, + target coroRunTargetCapabilityV1, step coro.ExecutorRunStep, + runtimeContext unsafe.Pointer, result *coroRunResultV1, ) (terminal, ok bool) { - if p == nil || driver == nil || result == nil || !policy.valid() { + if p == nil || driver == nil || runtimeContext == nil || result == nil || !policy.valid() { return false, false } - _, returnRequested, stateOK := policy.commandState() - if !stateOK { - return false, false + returnRequested := false + if policy.main != nil { + switch *policy.lifecycle { + case coroProgramRunningV1, coroProgramMainGoexitV1: + case coroProgramMainReturnRequestedV1: + returnRequested = true + default: + return false, false + } } switch step.Kind { case coro.ExecutorRunStepSource: - distributed, targetOK := coroTargetAfterSourceReductionV1(p, driver, step.Poll) + distributed, targetOK := false, true + distribute, stopAfterStable, distributionOK := coroTargetReadyDistributionV1(target) + if !distributionOK { + return false, false + } + if distribute { + distributed, targetOK = coroTargetAfterSourceReductionV1(p, driver, step.Poll) + } if !targetOK || distributed && !step.Poll.Complete || step.Poll.Complete && !coro.CommitExecutorRunSourceDistribution(driver, distributed) { return false, false } result.used++ result.sources++ + if stopAfterStable { + result.stop = coroRunAgainV1 + return true, true + } return false, true case coro.ExecutorRunStepMaterialize: if !coroMaterializeResumeCleanupStepV1(step.Cleanup) { @@ -296,8 +533,14 @@ func coroReduceExecutorRunStepV1( } result.used++ return false, true + case coro.ExecutorRunStepDirectChannel: + if step.Direct == nil || !coroMaterializeDirectChannelCompletionV1(step.Direct) { + return false, false + } + result.used++ + return false, true case coro.ExecutorRunStepDispatch: - if step.G == nil || step.Action.Handle == nil { + if step.Dispatched || step.G == nil || step.Action.Handle == nil { return false, false } if returnRequested && step.G != policy.main && step.Action.Kind == coro.ActionCheckResume && @@ -308,98 +551,10 @@ func coroReduceExecutorRunStepV1( result.dispatches++ return false, true case coro.ExecutorRunStepAction: - if step.G == nil || step.Action.Handle == nil { - return false, false - } - next, advanced := coroRunPhysicalActionV1(p, step.G, step.Action) - // The physical resume may have changed program lifecycle. Re-read the - // live policy before selecting the scheduler commit placement. - running, returnRequested, stateOK := policy.commandState() - if !stateOK { - return false, false - } - committed := false - if advanced && step.G == policy.main && returnRequested && next.Kind == coro.ActionCheckDestroy { - committed = coro.CommitExecutorRunCommandRootDestroy(driver, step.G, next) - } else if advanced && step.G == policy.main && running && - coro.CommitExecutorRunCommandBootstrapDirectChildHandoff(driver, step.G, next) { - committed = true - } else if advanced { - committed = coro.CommitExecutorRunAction(driver, step.G, next) - } - if !committed { - return false, false - } - if next.Kind == coro.ActionForeignReentryComplete { - result.used++ - result.destroys++ - result.stop = coroRunForeignReentryCompleteV1 - result.g = step.G - result.action = next - return true, true - } - // A locked ordinary suspension must decide whether to detach before - // ready distribution can move the peer which justifies a Yield handoff. - // Non-native targets compile this observation to a no-op. - osThreadSuspend, suspendOK := coroTargetPrepareOSThreadSuspendV1( - p, driver, step.G, next, + return coroReduceExecutorRunActionPreparedV1( + p, driver, policy, target, step.G, step.Action, step.Dispatched, + returnRequested, runtimeContext, result, ) - if !suspendOK { - return false, false - } - // A resume commit is the first stable scheduler-stack boundary after a - // managed `go` statement publishes its initial child. Native fleet targets - // may opportunistically hand that exact ready head to another P here; all - // other targets compile this call to a no-op. Failure after an actual - // publication is fatal because the mailbox has become the child's sole root. - if !osThreadSuspend && !coroTargetAfterStableRunActionV1(p, driver) { - return false, false - } - result.used++ - switch step.Action.Kind { - case coro.ActionCheckResume: - result.resumes++ - case coro.ActionCheckDestroy, coro.ActionPanicDestroy: - result.destroys++ - } - if osThreadSuspend { - result.stop = coroRunOSThreadSuspendV1 - result.g = step.G - result.action = next - return true, true - } - switch next.Kind { - case coro.ActionCheckResume, coro.ActionCheckDestroy, coro.ActionPanicDestroy: - if step.G == policy.main && returnRequested && next.Kind == coro.ActionCheckResume { - return false, false - } - case coro.ActionYield, coro.ActionPark: - if step.G == policy.main && returnRequested { - return false, false - } - case coro.ActionComplete: - isMain := step.G == policy.main - retireOwner := coro.ActionRetiresPhysicalOwner(next) - if !coroReleaseCompletedTask(step.G) { - return false, false - } - if isMain { - result.stop, result.g = coroRunMainDoneV1, policy.main - return true, true - } - if retireOwner && !coroTargetRetirePhysicalOwnerV1(p, driver) { - return false, false - } - case coro.ActionPanicComplete: - result.stop, result.g, result.action = coroRunPanicCompleteV1, step.G, next - return true, true - case coro.ActionCommitDestroy: - result.stop, result.g, result.action = coroRunDestroyCommitV1, step.G, next - return true, true - default: - return false, false - } - return false, true case coro.ExecutorRunStepDestroyCommit: if step.G == nil || step.Action.Kind != coro.ActionCommitDestroy || step.Action.Handle != nil { return false, false @@ -422,27 +577,71 @@ func coroRunSliceAtV1(p *coro.P, driver *coro.ExecutorDriver, now int64, budget if p == nil || driver == nil || now < 0 || budget == 0 { return coroRunResultV1{} } + runtimeContext := coroCaptureRuntimeContextV1() + if runtimeContext == nil { + return coroRunResultV1{} + } + run, runOK := coro.BeginExecutorRunSlice(driver) + if !runOK { + return coroRunResultV1{} + } + target, targetOK := coroTargetBeginRunSliceV1(p, driver) + if !targetOK { + return coroRunResultV1{} + } result := coroRunResultV1{} held := false for result.used < budget { - var wait, permitOK bool - held, wait, permitOK = coroPrepareManagedExecutionV1(driver, held) - if !permitOK { - _ = coroFinishManagedExecutionV1(driver, held) - return coroRunResultV1{} + // A slice-held lease already proves that this M owns one managed- + // execution slot. Probe/acquire only until that lease exists; calling a + // helper merely to return the same true bit on every action was visible in + // the handoff profile. + if !held { + var wait, permitOK bool + held, wait, permitOK = coroPrepareManagedExecutionV1(driver) + if !permitOK { + _ = coroFinishManagedExecutionV1(driver, held) + return coroRunResultV1{} + } + if wait { + result.stop = coroRunExecutionWaitV1 + return result + } } - if wait { - result.stop = coroRunExecutionWaitV1 - return result + combineDispatch := held && budget-result.used >= 2 + var actionStep coro.ExecutorRunActionStep + var actionSelected, nextOK bool + if combineDispatch { + actionStep, actionSelected, nextOK = run.NextActionCombined() + } else { + actionStep, actionSelected, nextOK = run.NextAction() } - step, nextOK := coro.NextExecutorRunStepAt(driver, now) - if !nextOK || !coroStepMatchesManagedExecutionV1(step, held) { + if !nextOK || actionSelected && !coroActionStepMatchesManagedExecutionV1(actionStep, held) { _ = coroFinishManagedExecutionV1(driver, held) return coroRunResultV1{} } - terminal, reduced := coroReduceExecutorRunStepV1( - p, driver, coroRunPolicyV1{}, step, &result, - ) + var terminal, reduced bool + if actionSelected { + terminal, reduced = coroReduceExecutorRunActionPreparedV1( + p, driver, coroRunPolicyV1{}, target, + actionStep.G, actionStep.Action, actionStep.Dispatched, + false, runtimeContext, &result, + ) + } else { + var step coro.ExecutorRunStep + if combineDispatch { + step, nextOK = run.NextAtCombined(now) + } else { + step, nextOK = run.NextAt(now) + } + if !nextOK || !coroStepMatchesManagedExecutionV1(step, held) { + _ = coroFinishManagedExecutionV1(driver, held) + return coroRunResultV1{} + } + terminal, reduced = coroReduceExecutorRunStepV1( + p, driver, coroRunPolicyV1{}, target, step, runtimeContext, &result, + ) + } if !reduced { _ = coroFinishManagedExecutionV1(driver, held) return coroRunResultV1{} @@ -453,18 +652,18 @@ func coroRunSliceAtV1(p *coro.P, driver *coro.ExecutorDriver, now int64, budget } return result } - stopForReturn, returnOK := coroStopAfterStableReductionV1( - driver, &result, - ) - if !returnOK { - _ = coroFinishManagedExecutionV1(driver, held) - return coroRunResultV1{} - } - if stopForReturn { - if !coroFinishManagedExecutionV1(driver, held) { + if target.physicalReturn { + stopForReturn, returnOK := coroStopAfterStableReductionV1(driver, &result) + if !returnOK { + _ = coroFinishManagedExecutionV1(driver, held) return coroRunResultV1{} } - return result + if stopForReturn { + if !coroFinishManagedExecutionV1(driver, held) { + return coroRunResultV1{} + } + return result + } } } if !coroFinishManagedExecutionV1(driver, held) { diff --git a/runtime/internal/runtime/coro_sched.go b/runtime/internal/runtime/coro_sched.go index b4a42d8227..188860594f 100644 --- a/runtime/internal/runtime/coro_sched.go +++ b/runtime/internal/runtime/coro_sched.go @@ -50,31 +50,72 @@ func coroRunSlice(p *coroP, main *coroG, driver *coro.ExecutorDriver, budget uin if !coroTargetBeforeProgramRunSliceV1(p, driver) { return coroRunResultV1{} } + runtimeContext := coroCaptureRuntimeContextV1() + if runtimeContext == nil { + return coroRunResultV1{} + } + run, runOK := coro.BeginExecutorRunSlice(driver) + if !runOK { + return coroRunResultV1{} + } + target, targetOK := coroTargetBeginRunSliceV1(p, driver) + if !targetOK { + return coroRunResultV1{} + } result := coroRunResultV1{} held := false for result.used < budget { - var wait, permitOK bool - held, wait, permitOK = coroPrepareManagedExecutionV1(driver, held) - if !permitOK { - _ = coroFinishManagedExecutionV1(driver, held) - return coroRunResultV1{} + if !held { + var wait, permitOK bool + held, wait, permitOK = coroPrepareManagedExecutionV1(driver) + if !permitOK { + _ = coroFinishManagedExecutionV1(driver, held) + return coroRunResultV1{} + } + if wait { + result.stop = coroRunExecutionWaitV1 + return result + } } - if wait { - result.stop = coroRunExecutionWaitV1 - return result + combineDispatch := held && budget-result.used >= 2 + var actionStep coro.ExecutorRunActionStep + var actionSelected, ok bool + if combineDispatch { + actionStep, actionSelected, ok = run.NextActionCombined() + } else { + actionStep, actionSelected, ok = run.NextAction() } - step, ok := coroProgramNextRunStepV1(driver) - if !ok || !coroStepMatchesManagedExecutionV1(step, held) { + if !ok || actionSelected && !coroActionStepMatchesManagedExecutionV1(actionStep, held) { _ = coroFinishManagedExecutionV1(driver, held) return coroRunResultV1{} } - terminal, reduced := coroReduceExecutorRunStepV1( - p, - driver, - coroRunPolicyV1{main: main, lifecycle: &coroProgramLifecycleV1State}, - step, - &result, - ) + policy := coroRunPolicyV1{main: main, lifecycle: &coroProgramLifecycleV1State} + var terminal, reduced bool + if actionSelected { + returnRequested := false + switch coroProgramLifecycleV1State { + case coroProgramRunningV1, coroProgramMainGoexitV1: + case coroProgramMainReturnRequestedV1: + returnRequested = true + default: + _ = coroFinishManagedExecutionV1(driver, held) + return coroRunResultV1{} + } + terminal, reduced = coroReduceExecutorRunActionPreparedV1( + p, driver, policy, target, + actionStep.G, actionStep.Action, actionStep.Dispatched, + returnRequested, runtimeContext, &result, + ) + } else { + step, nextOK := coroProgramNextRunStepV1(driver, &run, combineDispatch) + if !nextOK || !coroStepMatchesManagedExecutionV1(step, held) { + _ = coroFinishManagedExecutionV1(driver, held) + return coroRunResultV1{} + } + terminal, reduced = coroReduceExecutorRunStepV1( + p, driver, policy, target, step, runtimeContext, &result, + ) + } if !reduced { _ = coroFinishManagedExecutionV1(driver, held) return coroRunResultV1{} @@ -85,18 +126,18 @@ func coroRunSlice(p *coroP, main *coroG, driver *coro.ExecutorDriver, budget uin } return result } - stopForReturn, returnOK := coroStopAfterStableReductionV1( - driver, &result, - ) - if !returnOK { - _ = coroFinishManagedExecutionV1(driver, held) - return coroRunResultV1{} - } - if stopForReturn { - if !coroFinishManagedExecutionV1(driver, held) { + if target.physicalReturn { + stopForReturn, returnOK := coroStopAfterStableReductionV1(driver, &result) + if !returnOK { + _ = coroFinishManagedExecutionV1(driver, held) return coroRunResultV1{} } - return result + if stopForReturn { + if !coroFinishManagedExecutionV1(driver, held) { + return coroRunResultV1{} + } + return result + } } } if !coroFinishManagedExecutionV1(driver, held) { @@ -143,9 +184,14 @@ func coroFinishRunSliceCompatibility( } return result case coroRunIdleV1: - if !coro.EnterExecutorRunCompatibility(driver) { + entered, compatibilityOK := coro.EnterExecutorRunStandbyCompatibility(driver) + if !compatibilityOK { return coroRunResultV1{} } + if !entered { + result.stop = coroRunAgainV1 + return result + } more, drained := coroTargetDrainProgramTransfersV1(p, driver) if !drained { return coroRunResultV1{} @@ -281,7 +327,9 @@ func coroCancelReady(p *coroP) bool { for { switch action.Kind { case coro.ActionCancelDestroy: - coroHandleDestroy(action.Handle) + if !coroHandleDestroyCommitted(g, action.Handle) { + return false + } action, ok = coro.CancelDestroyed(p, g, action) if !ok { return false diff --git a/runtime/internal/runtime/coro_spawn.go b/runtime/internal/runtime/coro_spawn.go index 74c4f52502..3cd1d98aa8 100644 --- a/runtime/internal/runtime/coro_spawn.go +++ b/runtime/internal/runtime/coro_spawn.go @@ -30,7 +30,12 @@ const coroSpawnProductionEnabledV1 = true func coroSpawnBeginV1(parentPointer unsafe.Pointer) (unsafe.Pointer, bool) { parent := (*coroG)(parentPointer) - if !coro.CanBeginSpawn(parent) || !coroalloc.Ready() { + // BeginSpawn is the authoritative parent/P validation. The production + // caller is already executing on the parent resume, so repeating the + // read-only CanBeginSpawn preflight before every successful allocation only + // audits the same scheduler episode twice. Allocation failure is rolled back + // locally and an invalid boundary still fails atomically in BeginSpawn. + if !coroalloc.Ready() { return nil, false } taskSize := coro.TaskStorageSize() @@ -39,7 +44,6 @@ func coroSpawnBeginV1(parentPointer unsafe.Pointer) (unsafe.Pointer, bool) { if raw == nil { return nil, false } - coro.Zero(raw, allocationSize) child, _, actualSize, allocationOK := coroTaskAllocationAt(raw) if !allocationOK || actualSize != allocationSize || !coro.BeginSpawn(parent, child, raw, taskSize) { coro.Zero(raw, allocationSize) @@ -76,27 +80,27 @@ func coroReleaseCompletedTask(g *coroG) bool { // frame completion after source-specific park cleanup. The cancellation // record remains sticky until the G is physically dead; acknowledge it here // before applying the normal reclaimability/storage transfer contract. - if !coro.ReclaimableG(g) && - !coro.AcknowledgeTaskCancellation(g, coro.TaskCancelAbort) && - !coro.AcknowledgeTaskCancellation(g, coro.TaskCancelShutdown) { - return false - } - owned, ok := coro.TaskStorageOwned(g) + local, raw, taskSize, owned, ok := coro.ReleaseCompletedTask(g) if !ok { - return false + if !coro.AcknowledgeTaskCancellation(g, coro.TaskCancelAbort) && + !coro.AcknowledgeTaskCancellation(g, coro.TaskCancelShutdown) { + return false + } + local, raw, taskSize, owned, ok = coro.ReleaseCompletedTask(g) + if !ok { + return false + } } - if !coroReleaseRuntimeContext(g) { + if !coroReleaseRuntimeContext(g, local) { return false } if !owned { - return true + return raw == nil && taskSize == 0 } - raw, taskSize, ok := coro.ReleaseTaskStorage(g) _, _, allocationSize, allocationOK := coroTaskAllocationAt(raw) - if !ok || !allocationOK || raw != unsafe.Pointer(g) || taskSize != coro.TaskStorageSize() { + if !allocationOK || raw != unsafe.Pointer(g) || taskSize != coro.TaskStorageSize() { return false } - coro.Zero(raw, allocationSize) return coroalloc.FreeTask(raw, allocationSize) } diff --git a/runtime/internal/runtime/coro_target_native_fleet_llgo.go b/runtime/internal/runtime/coro_target_native_fleet_llgo.go index 514823d93a..fc6704df6d 100644 --- a/runtime/internal/runtime/coro_target_native_fleet_llgo.go +++ b/runtime/internal/runtime/coro_target_native_fleet_llgo.go @@ -116,6 +116,14 @@ func coroTargetExecutorStartV1(handle coro.ExecutorHandle) bool { coroRuntimeAbort("native coroutine execution quota start failed") return false } + for index := uint32(0); index < coroNativeFleetV1State.domainCount; index++ { + driver := coroNativeFleetV1State.domains[index].driverOwnerV1() + if !coro.BindExecutorServicePressure(driver, &coroNativeFleetV1State.execution) { + state.lifecycle = coroNativeFleetTargetFailedV1 + coroRuntimeAbort("native coroutine execution quota pressure bind failed") + return false + } + } program, programOK := coroNativeFleetHandleV1(0) if !programOK || program.Executor != handle || program.Route != 1 { state.lifecycle = coroNativeFleetTargetFailedV1 @@ -140,7 +148,7 @@ func coroTargetExecutorStartV1(handle coro.ExecutorHandle) bool { coroRuntimeAbort("native coroutine fleet worker start failed") return false } - if !coroNativeFleetPhysicalOwnersStartV1() { + if !coroNativeFleetPhysicalOwnersStartV1(limit) { _ = coroNativeWorkerPoolStopFleetV1() _ = coroNativeMStopCleanFactoryV1() state.lifecycle = coroNativeFleetTargetFailedV1 @@ -212,6 +220,29 @@ func coroTargetRequestChannelOperationV1(id coro.OperationID) bool { return accepted && (!coroNativeFleetRequestNeedsRingV1(domain, result) || domain.doorbell.Ring()) } +func coroTargetPublishDirectChannelCompletionV1( + owner *coro.ExecutorDriver, + route coro.RouteID, + completion *coro.DirectChannelCompletion, +) bool { + domain, ok := coroNativeFleetActiveDomainForRouteV1(route) + if !ok || owner == nil || domain.driverOwnerV1() != owner || !domain.ingress.Enter() { + return false + } + if domain.lifecycle != coroNativeFleetDomainActiveV1 || domain.driverOwnerV1() != owner || + domain.handle.Route != uint32(route) || + !coro.PublishExecutorDirectChannelCompletion(owner, completion) { + _, _ = domain.ingress.Leave() + return false + } + result := coroNativeFleetV1State.fleet.RequestExecutor(domain.handle) + accepted := result == coro.ExecutorRequestPublished || + result == coro.ExecutorRequestCoalesced || result == coro.ExecutorRequestIdleWake + ringOK := !coroNativeFleetRequestNeedsRingV1(domain, result) || domain.doorbell.Ring() + _, leaveOK := domain.ingress.Leave() + return accepted && ringOK && leaveOK +} + // coroTargetRequestControlledTimerV2 requests the exact owner after // time.Timer atomically publishes a new logical generation. The timer catalog // owns no producer callback: its next owner scan observes the retained control diff --git a/runtime/internal/runtime/coro_task_context.go b/runtime/internal/runtime/coro_task_context.go index 303a7180d0..9012478749 100644 --- a/runtime/internal/runtime/coro_task_context.go +++ b/runtime/internal/runtime/coro_task_context.go @@ -36,9 +36,24 @@ func coroRuntimeContextParent(task, parent *coro.G) (*g, bool) { return nil, false } parentG = &parentContext.g - if parentG.m == nil || parentG.m.curg != parentG || parentG.m.p == nil || - parentG.m.p.m != parentG.m || readgstatus(parentG) != _Grunning || - readpstatus(parentG.m.p) != _Prunning { + // Spawn already carries and validates the exact scheduler-owned parent + // task. Child initialization consumes only its immutable goid, so it does + // not require the logical runtime G to be installed in pthread TLS. A + // context-free parent remains detached/runnable during the physical + // resume; legacy or genuinely context-dependent callers remain attached + // and running. Reject every mixed graph rather than inferring ownership + // from ambient runtime state. + switch readgstatus(parentG) { + case _Grunnable: + if parentG.m != nil { + return nil, false + } + case _Grunning: + if parentG.m == nil || parentG.m.curg != parentG || parentG.m.p == nil || + parentG.m.p.m != parentG.m || readpstatus(parentG.m.p) != _Prunning { + return nil, false + } + default: return nil, false } } @@ -127,17 +142,26 @@ func validCoroRuntimeTaskContext(task *coro.G, ctx *coroRuntimeContext) bool { return ctx.g.isMain } -// coroEnterRuntimeContext installs task's runtime G only for the physical +// coroCaptureRuntimeContextV1 snapshots the executor context once at a stable +// scheduler boundary. A bounded run slice always restores this exact context +// after every physical resume, so its later actions do not need another +// pthread TLS lookup merely to rediscover the same owner. +func coroCaptureRuntimeContextV1() unsafe.Pointer { + return unsafe.Pointer(getg()) +} + +// coroEnterRuntimeContextFrom installs task's runtime G only for the physical // llvm.coro.resume interval. A synchronous C-to-Go reentry resumes a child // frame of the same logical G while its parent resume remains active below C; -// that exact nested case borrows the existing install. -func coroEnterRuntimeContext(task *coro.G) (coroRuntimeContextActivationV1, bool) { +// that exact nested case borrows the existing install. current is captured at +// the surrounding stable scheduler boundary and is restored by leave. +func coroEnterRuntimeContextFrom(task *coro.G, currentRaw unsafe.Pointer) (coroRuntimeContextActivationV1, bool) { ctx := (*coroRuntimeContext)(coro.TaskLocal(task)) if !validCoroRuntimeTaskContext(task, ctx) { return coroRuntimeContextActivationV1{}, false } gp := &ctx.g - current := getg() + current := (*g)(currentRaw) if current == gp { if gp.m == nil || gp.m.curg != gp || gp.m.p == nil || gp.m.p.m != gp.m || readgstatus(gp) != _Grunning || readpstatus(gp.m.p) != _Prunning { @@ -160,13 +184,23 @@ func coroEnterRuntimeContext(task *coro.G) (coroRuntimeContextActivationV1, bool return coroRuntimeContextActivationV1{previous: unsafe.Pointer(current)}, true } +func coroEnterRuntimeContext(task *coro.G) (coroRuntimeContextActivationV1, bool) { + return coroEnterRuntimeContextFrom(task, coroCaptureRuntimeContextV1()) +} + func coroLeaveRuntimeContext(task *coro.G, activation coroRuntimeContextActivationV1) bool { ctx := (*coroRuntimeContext)(coro.TaskLocal(task)) if !validCoroRuntimeTaskContext(task, ctx) { return false } gp := &ctx.g - if getg() != gp || gp.m == nil || gp.m.curg != gp || gp.m.p == nil || + // Enter is the only operation which installs this logical G, and every + // nested C-to-Go entry must borrow and restore the same installation before + // llvm.coro.resume returns. No other runtime path writes the getg slot in + // this interval. Re-reading pthread TLS here therefore proves no additional + // state; the exact G/M/P graph and status below are the authoritative + // post-resume certificate, while setg(previous) remains the checked restore. + if gp.m == nil || gp.m.curg != gp || gp.m.p == nil || gp.m.p.m != gp.m || readgstatus(gp) != _Grunning || readpstatus(gp.m.p) != _Prunning { return false } @@ -188,8 +222,7 @@ func coroLeaveRuntimeContext(task *coro.G, activation coroRuntimeContextActivati // coroReleaseRuntimeContext tears down the runtime sidecar after the scheduler // has made the G terminal but before its scanned task allocation is cleared. -func coroReleaseRuntimeContext(task *coro.G) bool { - raw := coro.TaskLocal(task) +func coroReleaseRuntimeContext(task *coro.G, raw unsafe.Pointer) bool { ctx := (*coroRuntimeContext)(raw) if !validCoroRuntimeTaskContext(task, ctx) { return false @@ -199,10 +232,6 @@ func coroReleaseRuntimeContext(task *coro.G) bool { return false } embedded := ctx.g.coroEmbedded - released, ok := coro.ReleaseTaskLocal(task) - if !ok || released != raw { - return false - } if gp.localContext != nil { releaseLocalBlocks(gp.localContext) gp.localContext = nil diff --git a/runtime/internal/runtime/coro_task_context_test_adapter.go b/runtime/internal/runtime/coro_task_context_test_adapter.go index 2cf0d96020..30eeb08645 100644 --- a/runtime/internal/runtime/coro_task_context_test_adapter.go +++ b/runtime/internal/runtime/coro_task_context_test_adapter.go @@ -18,7 +18,19 @@ package runtime -import "github.com/goplus/llgo/runtime/internal/coro" +import ( + "unsafe" + + "github.com/goplus/llgo/runtime/internal/coro" +) + +// The isolated scheduler/fleet adapter tests do not link the typed hchan +// implementation. A direct-channel step would therefore be an invalid fixture +// input; keep the reducer's production dependency explicit and fail closed if +// a test accidentally manufactures one. +func coroMaterializeDirectChannelCompletionV1(*coro.DirectChannelCompletion) bool { + return false +} // Named-source adapters exercise the target-neutral scheduler without loading // runtime2.go and platform getg implementations into the host Go runtime. @@ -31,14 +43,24 @@ func coroBindTaskAllocationRuntimeContext(task, parent *coro.G) bool { return task != nil } +var coroTestRuntimeContextV1 byte + +func coroCaptureRuntimeContextV1() unsafe.Pointer { + return unsafe.Pointer(&coroTestRuntimeContextV1) +} + +func coroEnterRuntimeContextFrom(task *coro.G, current unsafe.Pointer) (coroRuntimeContextActivationV1, bool) { + return coroRuntimeContextActivationV1{}, task != nil && current != nil +} + func coroEnterRuntimeContext(task *coro.G) (coroRuntimeContextActivationV1, bool) { - return coroRuntimeContextActivationV1{}, task != nil + return coroEnterRuntimeContextFrom(task, coroCaptureRuntimeContextV1()) } func coroLeaveRuntimeContext(task *coro.G, activation coroRuntimeContextActivationV1) bool { return task != nil } -func coroReleaseRuntimeContext(task *coro.G) bool { - return task != nil +func coroReleaseRuntimeContext(task *coro.G, local unsafe.Pointer) bool { + return task != nil && local != nil } diff --git a/runtime/internal/runtime/coro_worker_completion_window_llgo.go b/runtime/internal/runtime/coro_worker_completion_window_llgo.go new file mode 100644 index 0000000000..201a050cfa --- /dev/null +++ b/runtime/internal/runtime/coro_worker_completion_window_llgo.go @@ -0,0 +1,47 @@ +//go:build llgo && llgo_coro && llgo_coro_native_pipe && (darwin || linux) && !baremetal && !coro_runtime_adapter_test + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +import ( + "github.com/goplus/llgo/runtime/internal/coro" +) + +// Active polling is a bounded opportunity, not a new waiting primitive. It +// lets a worker running on another physical CPU finish a cache-hot syscall +// before this executor publishes IdleArmed, avoiding the pipe/poll round trip. +// A worker which can really block misses this window and immediately falls +// back to the retained doorbell protocol. Keep this as an iteration budget so +// the policy does not require a clock read or target timer capability. +const coroNativeWorkerCompletionSpinBudgetV1 uint32 = 32768 + +func coroNativeTryFastWorkerCompletionV1(driver *coro.ExecutorDriver) (ready, ok bool) { + probe, awaiting, ready, ok := coro.PrepareExecutorWorkerCompletionProbe(driver) + if !ok || ready || !awaiting { + return ready, ok + } + if !probe.Valid() { + return false, false + } + for attempt := uint32(0); attempt < coroNativeWorkerCompletionSpinBudgetV1; attempt++ { + if probe.Ready() { + return true, true + } + } + return false, true +} diff --git a/runtime/internal/runtime/coro_worker_native_llgo.go b/runtime/internal/runtime/coro_worker_native_llgo.go index 76b13984e1..a33128eee3 100644 --- a/runtime/internal/runtime/coro_worker_native_llgo.go +++ b/runtime/internal/runtime/coro_worker_native_llgo.go @@ -35,33 +35,6 @@ const ( coroNativeWorkerCapacityV1 = coroNativeWorkerPageCountV1 * coro.WorkerOperationPageCapacity ) -type coroNativeWorkerJobV1 struct { - id coro.OperationID - function uintptr - traceTarget uintptr - argc uint32 - args [coroworker.MaxArgs]uintptr -} - -func (job coroNativeWorkerJobV1) valid() bool { - return job.id.Valid() && job.id.Source() == coro.OperationSourceWorker && - job.function != 0 && job.traceTarget != 0 && job.argc <= coroworker.MaxArgs -} - -func coroNativeWorkerJobFromTransportV1(raw coroworker.Job) (coroNativeWorkerJobV1, bool) { - job := coroNativeWorkerJobV1{ - id: coro.OperationID{ - SourceSlot: raw.SourceSlot, - Generation: raw.Generation, - }, - function: raw.Function, - traceTarget: raw.TraceTarget, - argc: raw.Argc, - args: raw.Args, - } - return job, job.valid() -} - type coroNativeWorkerDeliveryV1 uint8 const ( @@ -190,6 +163,9 @@ func coroNativeWorkerPoolStartV1(handle coro.ExecutorHandle) bool { // already-bound fleet routes. It deliberately retains no executor handle: the // exact destination is encoded by every submitted OperationID. func coroNativeWorkerPoolStartFleetV1() bool { + if !coroProgramWorkerCapabilityV2() { + return coroNativeWorkerPoolCanReleaseV1() + } return coroNativeWorkerPoolStartDeliveryV1( coroNativeWorkerDeliveryFleetV1, coro.ExecutorHandle{}, @@ -216,9 +192,8 @@ func coroNativeWorkerPoolReserveV1( if !coroNativeWorkerSubmissionOwnerV1(handle, route) { return 0, false } - var reservation coroworker.QueueReservation - reserved := coroworker.QueueReserve(&reservation) - return reservation, reserved + reservation := coroworker.QueueReserve() + return reservation, reservation != 0 } func coroNativeWorkerPoolCancelReservationV1( @@ -241,24 +216,22 @@ func coroNativeWorkerPoolSubmitReservedV1( id coro.OperationID, function, traceTarget uintptr, argc uint32, - args *[coroworker.MaxArgs]uintptr, + a0, a1, a2, a3, a4, a5, a6, a7, a8 uintptr, ) bool { - if args == nil { - return false - } - job := coroworker.Job{ - SourceSlot: id.SourceSlot, - Generation: id.Generation, - Function: function, - TraceTarget: traceTarget, - Argc: argc, - Args: *args, - } - if _, valid := coroNativeWorkerJobFromTransportV1(job); !valid { + if !id.Valid() || id.Source() != coro.OperationSourceWorker || + function == 0 || traceTarget == 0 || argc > coroworker.MaxArgs { return false } return id.Route() == route && coroNativeWorkerSubmissionOwnerV1(handle, route) && - coroworker.QueueSubmitReserved(reservation, &job) + coroworker.QueueSubmitReserved( + reservation, + id.SourceSlot, + id.Generation, + function, + traceTarget, + argc, + a0, a1, a2, a3, a4, a5, a6, a7, a8, + ) } // coroNativeWorkerPoolStopDeliveryV1 seals submission, wakes all idle workers, @@ -306,6 +279,9 @@ func coroNativeWorkerPoolStopV1(handle coro.ExecutorHandle) bool { // ingress is still active. The join covers all queued route completions; only // after it returns may the coordinator begin route close. func coroNativeWorkerPoolStopFleetV1() bool { + if coroNativeWorkerPoolV1State == (coroNativeWorkerPoolV1{}) { + return coroNativeWorkerPoolCanReleaseV1() + } return coroNativeWorkerPoolStopDeliveryV1( coroNativeWorkerDeliveryFleetV1, coro.ExecutorHandle{}, @@ -331,25 +307,22 @@ func coroCancelNativeWorkerSubmissionV1( return coroNativeWorkerPoolCancelReservationV1(handle, route, reservation) } -// coroCommitNativeWorkerSubmissionV1 closes the no-return handoff from -// the core Worker park owner into the pre-reserved native queue. A failure to -// enqueue after MarkSubmitted would leave a retained frame with no future -// physical fact and therefore aborts instead of returning to the caller. -func coroCommitNativeWorkerSubmissionV1( - driver *coro.ExecutorDriver, - g *coro.G, +// coroPublishNativeWorkerSubmissionV1 closes the no-return handoff from the +// already committed core Worker generation into the pre-reserved native queue. +// A failure to enqueue would leave a retained frame with no future physical +// fact and therefore aborts instead of returning to the caller. +func coroPublishNativeWorkerSubmissionV1( handle coro.ExecutorHandle, route coro.RouteID, reservation coroworker.QueueReservation, id coro.OperationID, function, traceTarget uintptr, argc uint32, - args *[coroworker.MaxArgs]uintptr, + a0, a1, a2, a3, a4, a5, a6, a7, a8 uintptr, ) bool { - if driver == nil || g == nil || args == nil || function == 0 || traceTarget == 0 || + if function == 0 || traceTarget == 0 || argc > coroworker.MaxArgs || !id.Valid() || id.Source() != coro.OperationSourceWorker || id.Route() != route || - !coroNativeWorkerSubmissionOwnerV1(handle, route) || - !coro.CommitCurrentExecutorWorkerSubmission(driver, g, id) { + !coroNativeWorkerSubmissionOwnerV1(handle, route) { return false } if !coroNativeWorkerPoolSubmitReservedV1( @@ -360,7 +333,7 @@ func coroCommitNativeWorkerSubmissionV1( function, traceTarget, argc, - args, + a0, a1, a2, a3, a4, a5, a6, a7, a8, ) { coroRuntimeAbort("native coroutine worker committed submission failed") for { diff --git a/runtime/internal/runtime/coro_worker_owner_llgo.go b/runtime/internal/runtime/coro_worker_owner_llgo.go index 676348376d..78d555d2bb 100644 --- a/runtime/internal/runtime/coro_worker_owner_llgo.go +++ b/runtime/internal/runtime/coro_worker_owner_llgo.go @@ -96,12 +96,13 @@ func __llgo_coro_worker_park_v1( } state.magic = coroWorkerParkMagicV1 - ticket, operation, ok := coro.PrepareCurrentExecutorWorkerPark( + ticket, operation, ok := coro.PrepareCurrentExecutorWorkerParkCompiler( driver, task, handle, (*coro.HeaderV1)(header), &state.wait, + &state.packet, 1, 1, ) @@ -117,13 +118,9 @@ func __llgo_coro_worker_park_v1( } state.ticket = ticket state.operation = operation - if !coro.BindSingleWaitSetResumePacket(&state.wait, &state.packet, operation) { - coroWorkerAbortV1("cannot bind coroutine worker resume packet") - return - } - args := [coroworker.MaxArgs]uintptr{a0, a1, a2, a3, a4, a5, a6, a7, a8} - if !coroCommitNativeWorkerSubmissionV1( - driver, task, executor, route, reservation, operation, function, traceTarget, argc, &args, + if !coroPublishNativeWorkerSubmissionV1( + executor, route, reservation, operation, function, traceTarget, argc, + a0, a1, a2, a3, a4, a5, a6, a7, a8, ) { coroWorkerAbortV1("cannot commit coroutine worker submission") } diff --git a/runtime/internal/runtime/g_global.go b/runtime/internal/runtime/g_global.go index eca8e5db37..e4463fe2f7 100644 --- a/runtime/internal/runtime/g_global.go +++ b/runtime/internal/runtime/g_global.go @@ -32,6 +32,10 @@ func getg() *g { return currentG } +func getgIfPresent() *g { + return currentG +} + func setg(gp *g) { currentG = gp } diff --git a/runtime/internal/runtime/g_pthread.go b/runtime/internal/runtime/g_pthread.go index 4cfcc34804..bc4d58474a 100644 --- a/runtime/internal/runtime/g_pthread.go +++ b/runtime/internal/runtime/g_pthread.go @@ -47,7 +47,9 @@ func coroRuntimeContextBootstrap() bool { } var key pthread.Key if ret := key.Create(pthread.KeyDestructor(destroyG)); ret != 0 { - c.Fprintf(c.Stderr, c.Str("runtime: pthread_key_create failed (errno=%d)\n"), ret) + // The caller owns the terminal diagnostic. Formatting through stdio here + // would turn an unrecoverable pre-scheduler failure into an asynchronous + // worker transaction even though there is no scheduler which can resume it. return false } gKey = key @@ -62,16 +64,23 @@ func getg() *g { gp := initRuntimeContextUntracked(allocRuntimeContext(), nil, _Grunning) if ret := setgRaw(gp); ret != 0 { destroyG(c.Pointer(unsafe.Pointer(gp))) - c.Fprintf(c.Stderr, c.Str("runtime: pthread_setspecific failed (errno=%d)\n"), ret) coroRuntimeAbort("failed to install runtime g") return nil } return gp } +// getgIfPresent is the observation-only counterpart of getg. Runtime +// locality probes run inside an already installed managed resume and must not +// allocate or install a fallback context merely because no logical G is +// active. Keeping that cold initialization path out of this function also +// lets coroutine effect analysis preserve pthread TLS lookup as noblock. +func getgIfPresent() *g { + return (*g)(gKey.Get()) +} + func setg(gp *g) { if ret := setgRaw(gp); ret != 0 { - c.Fprintf(c.Stderr, c.Str("runtime: pthread_setspecific failed (errno=%d)\n"), ret) coroRuntimeAbort("failed to install runtime g") } } diff --git a/runtime/internal/runtime/tinygogc/rooted.go b/runtime/internal/runtime/tinygogc/rooted.go index b7e5b65ce6..7cebf0668f 100644 --- a/runtime/internal/runtime/tinygogc/rooted.go +++ b/runtime/internal/runtime/tinygogc/rooted.go @@ -18,7 +18,11 @@ package tinygogc -import "unsafe" +import ( + "unsafe" + + c "github.com/goplus/llgo/runtime/internal/clite" +) const rootedAllocationMagic uintptr = 0x726f6f74 @@ -73,6 +77,11 @@ func FreeRooted(ptr unsafe.Pointer, size uintptr) bool { unlock(&gcMutex) return false } + // The caller's raw pointer may remain conservatively visible on its native + // stack after logical release. Clear the payload before unlinking the + // explicit root so such a stale pointer cannot retain the frame's former Go + // object graph during a later collection. + c.Memset(ptr, 0, size) unlinkRootedAllocation(root) unlock(&gcMutex) return true diff --git a/runtime/internal/runtime/z_cgo.go b/runtime/internal/runtime/z_cgo.go index b69f1701a6..d1d8b3f6a3 100644 --- a/runtime/internal/runtime/z_cgo.go +++ b/runtime/internal/runtime/z_cgo.go @@ -22,6 +22,23 @@ import ( c "github.com/goplus/llgo/runtime/internal/clite" ) +// These names are the implementation side of cmd/cgo's generated +// runtime.cgoAlwaysFalse, runtime.cgoUse, and runtime.cgoKeepAlive linknames. +// The guarded calls are deliberately visible to Go escape/liveness analysis, +// but cgoAlwaysFalse makes them unreachable at run time. Keep real definitions +// so whole-program coroutine analysis and raw/plain cgo adapters resolve the +// same ordinary Go symbols instead of treating generated declarations as +// unknown managed externals. +var cgoAlwaysFalse bool + +func cgoUse(any) { + coroRuntimeAbort("runtime.cgoUse was called") +} + +func cgoKeepAlive(any) { + coroRuntimeAbort("runtime.cgoKeepAlive was called") +} + func CString(s string) *int8 { p := c.Malloc(uintptr(len(s)) + 1) return CStrCopy(p, *(*String)(unsafe.Pointer(&s))) diff --git a/runtime/internal/runtime/z_chan.go b/runtime/internal/runtime/z_chan.go index eef419c376..9f854d3179 100644 --- a/runtime/internal/runtime/z_chan.go +++ b/runtime/internal/runtime/z_chan.go @@ -20,6 +20,7 @@ import ( "unsafe" c "github.com/goplus/llgo/runtime/internal/clite" + "github.com/goplus/llgo/runtime/internal/coro" "github.com/goplus/llgo/runtime/internal/runtime/math" ) @@ -77,7 +78,8 @@ type chanWaiter struct { // multi-channel select. Such a waiter never owns pthread mutex/cond state; // z_chan_coro.go commits it through the exact ChannelOperationSource // transaction before any typed payload or completion status is published. - coro *coroChanOperationV1 + coro *coroChanOperationV1 + direct *coro.DirectChannelCompletion } type selectState struct { @@ -228,13 +230,54 @@ func panicSendOnClosedChan() { } func zeroChanRecv(v unsafe.Pointer, eltSize int) { - if v != nil && eltSize > 0 { + if v == nil || eltSize <= 0 { + return + } + // Channel element size is dynamic at this runtime boundary, but the small + // sizes below dominate ordinary scalar, string, slice and interface + // channels. Array types deliberately have byte alignment: this remains + // valid for types such as [8]byte whose address need not be uint64-aligned, + // while LLVM can still lower each constant-size assignment inline. + switch eltSize { + case 1: + *(*[1]byte)(v) = [1]byte{} + case 2: + *(*[2]byte)(v) = [2]byte{} + case 4: + *(*[4]byte)(v) = [4]byte{} + case 8: + *(*[8]byte)(v) = [8]byte{} + case 16: + *(*[16]byte)(v) = [16]byte{} + case 24: + *(*[24]byte)(v) = [24]byte{} + case 32: + *(*[32]byte)(v) = [32]byte{} + default: c.Memset(v, 0, uintptr(eltSize)) } } func copyChanElem(dst, src unsafe.Pointer, eltSize int) { - if dst != nil && src != nil && eltSize > 0 { + if dst == nil || src == nil || eltSize <= 0 { + return + } + switch eltSize { + case 1: + *(*[1]byte)(dst) = *(*[1]byte)(src) + case 2: + *(*[2]byte)(dst) = *(*[2]byte)(src) + case 4: + *(*[4]byte)(dst) = *(*[4]byte)(src) + case 8: + *(*[8]byte)(dst) = *(*[8]byte)(src) + case 16: + *(*[16]byte)(dst) = *(*[16]byte)(src) + case 24: + *(*[24]byte)(dst) = *(*[24]byte)(src) + case 32: + *(*[32]byte)(dst) = *(*[32]byte)(src) + default: c.Memcpy(dst, src, uintptr(eltSize)) } } @@ -322,7 +365,7 @@ func (w *chanWaiter) finish(status waitStatus) { } func claimWaiter(w *chanWaiter) bool { - if w.coro != nil { + if w.coro != nil || w.direct != nil { return false } if w.sel != nil { @@ -339,9 +382,20 @@ func claimWaiter(w *chanWaiter) bool { return true } -func completeRecvWaiter(w *chanWaiter, src unsafe.Pointer, eltSize int, status waitStatus) coroChanMatchResult { +func completeRecvWaiterWithContext( + w *chanWaiter, + src unsafe.Pointer, + eltSize int, + status waitStatus, + context *coroChanExternalCommitContextV1, +) coroChanMatchResult { + if w.direct != nil { + return commitDirectCoroRecvWaiterLockedV1(w, src, eltSize, status, context) + } if w.coro != nil { - return commitCoroRecvWaiterLocked(w, src, eltSize, status) + return commitCoroRecvWaiterLockedWithContext( + w, src, eltSize, status, coroChanExternalContextValueV1(context), + ) } if !claimWaiter(w) { return coroChanMatchDiscarded @@ -355,9 +409,23 @@ func completeRecvWaiter(w *chanWaiter, src unsafe.Pointer, eltSize int, status w return coroChanMatchCommitted } -func completeSendWaiter(w *chanWaiter, status waitStatus) coroChanMatchResult { +func completeRecvWaiter(w *chanWaiter, src unsafe.Pointer, eltSize int, status waitStatus) coroChanMatchResult { + context := currentCoroChannelExternalContextV1() + return completeRecvWaiterWithContext(w, src, eltSize, status, &context) +} + +func completeSendWaiterWithContext( + w *chanWaiter, + status waitStatus, + context *coroChanExternalCommitContextV1, +) coroChanMatchResult { + if w.direct != nil { + return commitDirectCoroSendWaiterLockedV1(w, nil, w.size, status, context) + } if w.coro != nil { - return commitCoroSendWaiterLocked(w, nil, w.size, status) + return commitCoroSendWaiterLockedWithContext( + w, nil, w.size, status, coroChanExternalContextValueV1(context), + ) } if !claimWaiter(w) { return coroChanMatchDiscarded @@ -366,9 +434,24 @@ func completeSendWaiter(w *chanWaiter, status waitStatus) coroChanMatchResult { return coroChanMatchCommitted } -func recvFromSendWaiter(dst unsafe.Pointer, w *chanWaiter, eltSize int) coroChanMatchResult { +func completeSendWaiter(w *chanWaiter, status waitStatus) coroChanMatchResult { + context := currentCoroChannelExternalContextV1() + return completeSendWaiterWithContext(w, status, &context) +} + +func recvFromSendWaiterWithContext( + dst unsafe.Pointer, + w *chanWaiter, + eltSize int, + context *coroChanExternalCommitContextV1, +) coroChanMatchResult { + if w.direct != nil { + return commitDirectCoroSendWaiterLockedV1(w, dst, eltSize, waitSendOK, context) + } if w.coro != nil { - return commitCoroSendWaiterLocked(w, dst, eltSize, waitSendOK) + return commitCoroSendWaiterLockedWithContext( + w, dst, eltSize, waitSendOK, coroChanExternalContextValueV1(context), + ) } if !claimWaiter(w) { return coroChanMatchDiscarded @@ -378,13 +461,24 @@ func recvFromSendWaiter(dst unsafe.Pointer, w *chanWaiter, eltSize int) coroChan return coroChanMatchCommitted } -func dequeueRecvAndComplete(p *Chan, src unsafe.Pointer, eltSize int, status waitStatus) bool { +func recvFromSendWaiter(dst unsafe.Pointer, w *chanWaiter, eltSize int) coroChanMatchResult { + context := currentCoroChannelExternalContextV1() + return recvFromSendWaiterWithContext(dst, w, eltSize, &context) +} + +func dequeueRecvAndCompleteWithContext( + p *Chan, + src unsafe.Pointer, + eltSize int, + status waitStatus, + context *coroChanExternalCommitContextV1, +) bool { for { w := p.recvq.dequeue() if w == nil { return false } - switch result := completeRecvWaiter(w, src, eltSize, status); result { + switch result := completeRecvWaiterWithContext(w, src, eltSize, status, context); result { case coroChanMatchCommitted: return true case coroChanMatchDiscarded: @@ -399,13 +493,23 @@ func dequeueRecvAndComplete(p *Chan, src unsafe.Pointer, eltSize int, status wai } } -func dequeueSendAndRecv(p *Chan, dst unsafe.Pointer, eltSize int) bool { +func dequeueRecvAndComplete(p *Chan, src unsafe.Pointer, eltSize int, status waitStatus) bool { + context := currentCoroChannelExternalContextV1() + return dequeueRecvAndCompleteWithContext(p, src, eltSize, status, &context) +} + +func dequeueSendAndRecvWithContext( + p *Chan, + dst unsafe.Pointer, + eltSize int, + context *coroChanExternalCommitContextV1, +) bool { for { w := p.sendq.dequeue() if w == nil { return false } - switch result := recvFromSendWaiter(dst, w, eltSize); result { + switch result := recvFromSendWaiterWithContext(dst, w, eltSize, context); result { case coroChanMatchCommitted: return true case coroChanMatchDiscarded: @@ -420,12 +524,25 @@ func dequeueSendAndRecv(p *Chan, dst unsafe.Pointer, eltSize int) bool { } } -func chanTrySendLocked(p *Chan, v unsafe.Pointer, eltSize int) (tryOK bool, closed bool) { +func dequeueSendAndRecv(p *Chan, dst unsafe.Pointer, eltSize int) bool { + return dequeueSendAndRecvWithContext(p, dst, eltSize, nil) +} + +func chanTrySendLockedWithContext( + p *Chan, + v unsafe.Pointer, + eltSize int, + context *coroChanExternalCommitContextV1, +) (tryOK bool, closed bool) { elemSize := p.elemsize if p.closed { return false, true } - if dequeueRecvAndComplete(p, v, elemSize, waitRecvOK) { + // The overwhelmingly common buffered fast path has no parked receiver. + // Avoid entering the claim/queue helper chain unless the O(1) queue header + // says there is work. A non-nil head may still be a stale select case; the + // existing helper retains the exact retry/discard semantics for that case. + if p.recvq.first != nil && dequeueRecvAndCompleteWithContext(p, v, elemSize, waitRecvOK, context) { return true, false } if p.qcount < p.dataqsiz { @@ -440,6 +557,10 @@ func chanTrySendLocked(p *Chan, v unsafe.Pointer, eltSize int) (tryOK bool, clos return false, false } +func chanTrySendLocked(p *Chan, v unsafe.Pointer, eltSize int) (tryOK bool, closed bool) { + return chanTrySendLockedWithContext(p, v, eltSize, nil) +} + func ChanTrySend(p *Chan, v unsafe.Pointer, eltSize int) bool { if p == nil { return false @@ -457,12 +578,17 @@ func ChanTrySend(p *Chan, v unsafe.Pointer, eltSize int) bool { // compiler-owned stackless channel lowering. A closed channel deliberately // returns false: the exact park transaction rechecks it and returns a typed // send-closed status without unwinding across an LLVM coroutine suspension. -func CoroChanTrySend(p *Chan, v unsafe.Pointer, eltSize int) bool { +func CoroChanTrySend(g unsafe.Pointer, p *Chan, v unsafe.Pointer, eltSize int) bool { if p == nil { return false } p.mutex.Lock() - ok, closed := chanTrySendLocked(p, v, eltSize) + var context *coroChanExternalCommitContextV1 + if p.recvq.first != nil { + resolved := coroChannelExternalContextForTaskV1((*coro.G)(g)) + context = &resolved + } + ok, closed := chanTrySendLockedWithContext(p, v, eltSize, context) p.mutex.Unlock() return ok && !closed } @@ -493,10 +619,15 @@ func ChanSend(p *Chan, v unsafe.Pointer, eltSize int) bool { return true } -func chanTryRecvLocked(p *Chan, v unsafe.Pointer, eltSize int) (recvOK bool, tryOK bool) { +func chanTryRecvLockedWithContext( + p *Chan, + v unsafe.Pointer, + eltSize int, + context *coroChanExternalCommitContextV1, +) (recvOK bool, tryOK bool) { elemSize := p.elemsize if p.dataqsiz == 0 { - if dequeueSendAndRecv(p, v, elemSize) { + if dequeueSendAndRecvWithContext(p, v, elemSize, context) { return true, true } } else if p.qcount > 0 { @@ -507,7 +638,13 @@ func chanTryRecvLocked(p *Chan, v unsafe.Pointer, eltSize int) (recvOK bool, try p.recvx = 0 } p.qcount-- - dequeueSendToBuffer(p) + // A local buffered send/receive pair normally has no parked sender. + // Refill only when the queue header proves that the reconciliation path + // can make progress; stale/select entries are still handled by the + // existing helper once present. + if p.sendq.first != nil { + dequeueSendToBuffer(p, context) + } return true, true } if p.closed { @@ -517,6 +654,10 @@ func chanTryRecvLocked(p *Chan, v unsafe.Pointer, eltSize int) (recvOK bool, try return false, false } +func chanTryRecvLocked(p *Chan, v unsafe.Pointer, eltSize int) (recvOK bool, tryOK bool) { + return chanTryRecvLockedWithContext(p, v, eltSize, nil) +} + func ChanTryRecv(p *Chan, v unsafe.Pointer, eltSize int) (recvOK bool, tryOK bool) { if p == nil { return false, false @@ -530,8 +671,19 @@ func ChanTryRecv(p *Chan, v unsafe.Pointer, eltSize int) (recvOK bool, tryOK boo // CoroChanTryRecv is the nonblocking first attempt used by compiler-owned // stackless channel lowering. Unlike ChanRecv it never retains the caller's // activation; a false tryOK is completed by the exact park transaction. -func CoroChanTryRecv(p *Chan, v unsafe.Pointer, eltSize int) (recvOK bool, tryOK bool) { - return ChanTryRecv(p, v, eltSize) +func CoroChanTryRecv(g unsafe.Pointer, p *Chan, v unsafe.Pointer, eltSize int) (recvOK bool, tryOK bool) { + if p == nil { + return false, false + } + p.mutex.Lock() + var context *coroChanExternalCommitContextV1 + if p.sendq.first != nil { + resolved := coroChannelExternalContextForTaskV1((*coro.G)(g)) + context = &resolved + } + recvOK, tryOK = chanTryRecvLockedWithContext(p, v, eltSize, context) + p.mutex.Unlock() + return } func ChanRecv(p *Chan, v unsafe.Pointer, eltSize int) (recvOK bool) { @@ -558,12 +710,10 @@ const ( coroChanCloseClosed ) -// CoroChanTryClose performs the complete owner-local close transaction without -// raising a Go panic. The compiler maps the two ordinary language errors to -// its explicit-status terminal path, so neither can unwind through a live LLVM -// coroutine frame. Closing wakes receivers, senders, and select candidates via -// the same channel operation source used by send/receive cancellation. -func CoroChanTryClose(p *Chan) uint32 { +func coroChanTryCloseWithContext( + p *Chan, + context *coroChanExternalCommitContextV1, +) uint32 { if p == nil { return coroChanCloseNil } @@ -576,12 +726,12 @@ func CoroChanTryClose(p *Chan) uint32 { // Claim contention can temporarily leave buffered data behind a queued // receiver. Preserve Go's close ordering: publish those values before the // remaining receivers observe the closed zero value. - if !reconcileBufferedChanLocked(p, false) { + if !reconcileBufferedChanLocked(p, false, context) { p.mutex.Unlock() coroRuntimeAbort("invalid coroutine buffered channel close reconciliation") return coroChanCloseClosed } - if !drainClosedChanWaitersLocked(p) { + if !drainClosedChanWaitersLocked(p, context) { p.mutex.Unlock() coroRuntimeAbort("invalid coroutine channel close completion") return coroChanCloseClosed @@ -590,6 +740,26 @@ func CoroChanTryClose(p *Chan) uint32 { return coroChanCloseOK } +// CoroChanTryClose performs the complete owner-local close transaction without +// raising a Go panic. The compiler maps the two ordinary language errors to +// its explicit-status terminal path, so neither can unwind through a live LLVM +// coroutine frame. Closing wakes receivers, senders, and select candidates via +// the same channel operation source used by send/receive cancellation. +func CoroChanTryClose(p *Chan) uint32 { + context := currentCoroChannelExternalContextV1() + return coroChanTryCloseWithContext(p, &context) +} + +// CoroChanTryCloseTask is the compiler-owned close boundary. The hidden task +// parameter closes the owner-local completion transaction without consulting +// ambient runtime state, allowing whole-program analysis to prove that a +// physical caller is runtime-context independent. Ordinary ChanClose retains +// CoroChanTryClose and its ambient compatibility path. +func CoroChanTryCloseTask(g unsafe.Pointer, p *Chan) uint32 { + context := coroChannelExternalContextForTaskV1((*coro.G)(g)) + return coroChanTryCloseWithContext(p, &context) +} + func ChanClose(p *Chan) { switch CoroChanTryClose(p) { case coroChanCloseOK: @@ -603,7 +773,10 @@ func ChanClose(p *Chan) { } } -func dequeueClosedRecvStepLocked(p *Chan) coroChanQueueStepV1 { +func dequeueClosedRecvStepLocked( + p *Chan, + context *coroChanExternalCommitContextV1, +) coroChanQueueStepV1 { if p == nil || !p.closed { return coroChanQueueInvalidV1 } @@ -611,7 +784,9 @@ func dequeueClosedRecvStepLocked(p *Chan) coroChanQueueStepV1 { if w == nil { return coroChanQueueIdleV1 } - switch result := completeRecvWaiter(w, nil, p.elemsize, waitRecvClosed); result { + switch result := completeRecvWaiterWithContext( + w, nil, p.elemsize, waitRecvClosed, context, + ); result { case coroChanMatchCommitted: return coroChanQueueCommittedV1 case coroChanMatchDiscarded: @@ -624,7 +799,10 @@ func dequeueClosedRecvStepLocked(p *Chan) coroChanQueueStepV1 { } } -func dequeueClosedSendStepLocked(p *Chan) coroChanQueueStepV1 { +func dequeueClosedSendStepLocked( + p *Chan, + context *coroChanExternalCommitContextV1, +) coroChanQueueStepV1 { if p == nil || !p.closed { return coroChanQueueInvalidV1 } @@ -632,7 +810,7 @@ func dequeueClosedSendStepLocked(p *Chan) coroChanQueueStepV1 { if w == nil { return coroChanQueueIdleV1 } - switch result := completeSendWaiter(w, waitSendClosed); result { + switch result := completeSendWaiterWithContext(w, waitSendClosed, context); result { case coroChanMatchCommitted: return coroChanQueueCommittedV1 case coroChanMatchDiscarded: @@ -650,9 +828,12 @@ func dequeueClosedSendStepLocked(p *Chan) coroChanQueueStepV1 { // eventually resume and remove that exact node. Its resume tail calls this // helper again, so ordinary waiters behind it cannot remain stranded on an // already-closed channel. -func drainClosedChanWaitersLocked(p *Chan) bool { +func drainClosedChanWaitersLocked( + p *Chan, + context *coroChanExternalCommitContextV1, +) bool { for { - switch dequeueClosedRecvStepLocked(p) { + switch dequeueClosedRecvStepLocked(p, context) { case coroChanQueueCommittedV1, coroChanQueueDiscardedV1: continue case coroChanQueueBlockedV1: @@ -666,7 +847,7 @@ func drainClosedChanWaitersLocked(p *Chan) bool { send: for { - switch dequeueClosedSendStepLocked(p) { + switch dequeueClosedSendStepLocked(p, context) { case coroChanQueueCommittedV1, coroChanQueueDiscardedV1: continue case coroChanQueueIdleV1, coroChanQueueBlockedV1: diff --git a/runtime/internal/runtime/z_chan_coro.go b/runtime/internal/runtime/z_chan_coro.go index c724978f61..35281a2b89 100644 --- a/runtime/internal/runtime/z_chan_coro.go +++ b/runtime/internal/runtime/z_chan_coro.go @@ -38,6 +38,7 @@ type coroChanOperationV1 struct { waiter *chanWaiter source *coro.ChannelOperationSource magic uint32 + direct bool } type coroChanCleanupPhaseV1 uint8 @@ -71,17 +72,13 @@ type coroChanCleanupCursorV1 struct { // // The type is exported solely so the compiler can request its target layout // from the frozen runtime package. Its fields remain runtime-private and no Go -// aggregate crosses a C or compiler hook ABI. +// aggregate crosses a C or compiler hook ABI. Storage is zero-filled with its +// containing coroutine frame before the first prepare; every successful +// resume restores the complete zero value before the compiler can reuse it. type CoroChanParkV1 struct { - wait coro.WaitSetRecord - claim coro.SelectClaim - packet coro.ResumePacket - cleanup coro.ResumeCleanupPlan - reconcile coroChanCleanupCursorV1 - ticket coro.ParkTicket - operation coroChanOperationV1 - waiter chanWaiter - magic uint32 + coro.DirectChannelParkStorageV1 + waiter chanWaiter + magic uint32 } // CoroChanSelectCaseV1 is one compiler-spilled physical channel candidate. @@ -153,25 +150,6 @@ func validCoroChanCleanupCursorV1(cursor *coroChanCleanupCursorV1) bool { (!cursor.deliver || cursor.status.done()) } -func validCoroChanParkV1(state *CoroChanParkV1) bool { - if state == nil || state.magic != coroChanParkMagicV1 || - state.waiter.status > waitSendClosed || state.waiter.size < 0 || - state.reconcile != (coroChanCleanupCursorV1{}) { - return false - } - if state.operation == (coroChanOperationV1{}) && state.waiter == (chanWaiter{}) { - return state.ticket != (coro.ParkTicket{}) - } - if state.waiter.coro != &state.operation || state.operation.waiter != &state.waiter || - state.operation.claim != &state.claim { - return false - } - if state.waiter.ch == nil { - return state.operation.id == (coro.OperationID{}) && state.operation.magic == 0 - } - return validCoroChanOperationV1(&state.operation, &state.waiter) -} - func coroChanSelectCaseAt(base unsafe.Pointer, index uintptr) *CoroChanSelectCaseV1 { return (*CoroChanSelectCaseV1)(unsafe.Add(base, index*unsafe.Sizeof(CoroChanSelectCaseV1{}))) } @@ -288,70 +266,512 @@ func classifyCoroChanPairBegin(result coro.ChannelExternalCommitPairBeginResult) } } -func publishCoroChannelOwnerLocalV1(operation *coroChanOperationV1) (published, ok bool) { - if operation == nil || !operation.id.Valid() { +// coroChanCompletionEndpointV1 is the only source identity needed after an +// irreversible hchan commit. Keeping it outside compiler-spilled waiter state +// lets an exact open/unbuffered direct rendezvous retire that state before the +// executor can observe the completion. +type coroChanCompletionEndpointV1 struct { + source *coro.ChannelOperationSource + id coro.OperationID +} + +func coroChanCompletionEndpointForV1( + operation *coroChanOperationV1, +) (coroChanCompletionEndpointV1, bool) { + if operation == nil || operation.source == nil || !operation.id.Valid() { + return coroChanCompletionEndpointV1{}, false + } + return coroChanCompletionEndpointV1{source: operation.source, id: operation.id}, true +} + +// coroChanDirectResultFinalizableV1 recognizes the exact typed cleanup which +// is a no-op: a compiler-owned direct waiter has already been detached from an +// open unbuffered hchan and its payload/status effect is complete. Buffered +// channels, close propagation, select candidates, cancellation, and any node +// still linked in a queue retain the bounded runtime cleanup cursor. +func coroChanDirectResultFinalizableV1( + operation *coroChanOperationV1, + status waitStatus, +) bool { + if operation == nil || !operation.direct || operation.waiter == nil || + !validCoroChanOperationV1(operation, operation.waiter) || !status.done() { + return false + } + return coroChanDirectCommitShapeV1(operation, status, true) +} + +// coroChanDirectCommitShapeV1 checks only the hchan-owned half of the exact +// direct waiter certificate. Callers which already proved +// validCoroChanOperationV1 under this hchan lock can retain that proof across +// the no-suspend transaction instead of revalidating the same operation, +// waiter, source route, and back-pointers at every phase boundary. +func coroChanDirectCommitShapeV1( + operation *coroChanOperationV1, + status waitStatus, + completed bool, +) bool { + waiter := operation.waiter + ch := waiter.ch + wantStatus := waitPending + if completed { + wantStatus = status + } + return operation.direct && status.done() && waiter.status == wantStatus && + !waiter.queued && waiter.prev == nil && waiter.next == nil && + ch != nil && ch.dataqsiz == 0 && ch.qcount == 0 && !ch.closed +} + +// coroChanDirectCommitCandidateV1 is the pre-effect counterpart of +// coroChanDirectResultFinalizableV1. The exact waiter has been detached under +// its hchan lock but its typed payload and terminal status have not yet been +// written. Only this open, unbuffered, one-operation shape may prepare the +// owner-local source fast lane. +func coroChanDirectCommitCandidateV1( + operation *coroChanOperationV1, + status waitStatus, +) bool { + if operation == nil || !operation.direct || operation.waiter == nil || + !validCoroChanOperationV1(operation, operation.waiter) || !status.done() { + return false + } + return coroChanDirectCommitShapeV1(operation, status, false) +} + +func finalizeCoroChanDirectResultV1(operation *coroChanOperationV1, recycled bool) bool { + if operation == nil || operation.waiter == nil { + return false + } + id, waiter := operation.id, operation.waiter + if recycled { + ch := waiter.ch + if id != (coro.OperationID{}) || !operation.direct || + operation.magic != coroChanOperationMagicV1 || operation.claim == nil || + operation.source == nil || waiter.coro != operation || !waiter.status.done() || + waiter.queued || waiter.prev != nil || waiter.next != nil || ch == nil || + ch.dataqsiz != 0 || ch.qcount != 0 || ch.closed { + return false + } + *waiter = chanWaiter{} + *operation = coroChanOperationV1{} + return true + } + if !id.Valid() { + return false + } + *waiter = chanWaiter{} + *operation = coroChanOperationV1{id: id} + return true +} + +func publishCoroChannelOwnerLocalV1( + current *coro.G, + driver *coro.ExecutorDriver, + endpoint coroChanCompletionEndpointV1, +) (published, ok bool) { + if endpoint.source == nil || !endpoint.id.Valid() { return false, false } - if current, _ := coroCurrentTaskV1(); current != nil { - return coro.TryPublishOwnerLocalChannelCompletion( + if current != nil && driver != nil { + return coro.TryPublishOwnerLocalChannelCompletionCurrent( current, - operation.source, - operation.id, + driver, + endpoint.source, + endpoint.id, ) } return false, true } -func requestCoroChannelExecutorV1(operation *coroChanOperationV1) bool { - local, ok := publishCoroChannelOwnerLocalV1(operation) - if !ok || local { - return ok +type coroChanExternalCommitContextV1 struct { + current *coro.G + driver *coro.ExecutorDriver + route coro.RouteID + ownerLocalDirect bool + directResult bool +} + +// coroChanExternalContextValueV1 turns an optional optimization capability +// into an explicit value without consulting ambient runtime state. A zero +// value retains the fully routed correctness path; wrappers which actually +// own a current-task capability sample it before entering the shared channel +// transaction. +func coroChanExternalContextValueV1( + context *coroChanExternalCommitContextV1, +) coroChanExternalCommitContextV1 { + if context != nil { + return *context + } + return coroChanExternalCommitContextV1{} +} + +func currentCoroChannelExternalContextV1() coroChanExternalCommitContextV1 { + current, driver, route := coroCurrentTaskV1() + return coroChanExternalCommitContextV1{ + current: current, + driver: driver, + route: route, + } +} + +// coroChannelExternalContextForTaskV1 consumes the compiler-carried logical +// task directly. Channel lowering already owns this value as its first hidden +// coroutine parameter, so recovering it through getg/TLS on every rendezvous +// would both repeat work and unnecessarily disable owner-local completion on +// targets without a native TLS-backed current-task adapter. +func coroChannelExternalContextForTaskV1(task *coro.G) coroChanExternalCommitContextV1 { + // Direct one-case completion consumes the compiler-carried task and derives + // its exact P/driver capability in the same core transaction which commits + // the peer. Do not decompose that capability here merely to make the core + // reconstruct and revalidate it a second time. A queued select endpoint asks + // resolveCoroChannelExternalContextV1 for the richer source capability only + // on that less common branch. + return coroChanExternalCommitContextV1{current: task} +} + +func resolveCoroChannelExternalContextV1( + context coroChanExternalCommitContextV1, +) coroChanExternalCommitContextV1 { + if context.current == nil || context.driver != nil { + return context + } + driver, route, current := coro.CurrentExecutorDriverForCompilerTask(context.current) + if !current { + // Compatibility adapters do not retain the bounded runner's private + // issued marker. They still admit the complete arbitrary-caller proof. + var handle coro.ExecutorHandle + driver, handle, route, current = coro.CurrentExecutorDriver(context.current) + if !current || handle.Slot == 0 || handle.Generation == 0 { + return coroChanExternalCommitContextV1{} + } + } + context.driver = driver + context.route = route + return context +} + +func finishDirectCoroChannelCompletionV1( + waiter *chanWaiter, + status waitStatus, + context *coroChanExternalCommitContextV1, +) bool { + if waiter == nil || waiter.direct == nil || waiter.coro != nil || !status.done() { + return false + } + current, route := (*coro.G)(nil), coro.RouteID(0) + if context != nil { + current, route = context.current, context.route + } + owner, route, result := coro.FinishDirectChannelCompletionFromCompilerTask( + current, waiter.direct, uint8(status), route, + ) + switch result { + case coro.DirectChannelCompletionFinishInline: + *waiter = chanWaiter{} + return true + case coro.DirectChannelCompletionFinishOwnerPublished: + return true + case coro.DirectChannelCompletionFinishNeedsTarget: + return coroTargetPublishDirectChannelCompletionV1(owner, route, waiter.direct) + default: + return false + } +} + +func prepareDirectCoroRecvWaiterLockedV1( + waiter *chanWaiter, + src unsafe.Pointer, + eltSize int, + status waitStatus, +) coroChanMatchResult { + if waiter == nil || waiter.direct == nil || waiter.coro != nil || waiter.send || + waiter.size != eltSize || !status.done() || status == waitSendClosed { + return coroChanMatchInvalid + } + switch coro.BeginDirectChannelCompletion(waiter.direct) { + case coro.DirectChannelCompletionBeginCanceled: + return coroChanMatchDiscarded + case coro.DirectChannelCompletionBeginAcquired: + default: + return coroChanMatchInvalid + } + if status.recvOK() { + copyChanElem(waiter.elem, src, eltSize) + } else { + zeroChanRecv(waiter.elem, eltSize) + } + waiter.status = status + return coroChanMatchCommitted +} + +func commitDirectCoroRecvWaiterLockedV1( + waiter *chanWaiter, + src unsafe.Pointer, + eltSize int, + status waitStatus, + context *coroChanExternalCommitContextV1, +) coroChanMatchResult { + if result := prepareDirectCoroRecvWaiterLockedV1( + waiter, src, eltSize, status, + ); result != coroChanMatchCommitted { + return result + } + if !finishDirectCoroChannelCompletionV1(waiter, status, context) { + return coroChanMatchInvalid + } + return coroChanMatchCommitted +} + +func prepareDirectCoroSendWaiterLockedV1( + waiter *chanWaiter, + dst unsafe.Pointer, + eltSize int, + status waitStatus, +) coroChanMatchResult { + if waiter == nil || waiter.direct == nil || waiter.coro != nil || !waiter.send || + waiter.size != eltSize || (status != waitSendOK && status != waitSendClosed) { + return coroChanMatchInvalid + } + switch coro.BeginDirectChannelCompletion(waiter.direct) { + case coro.DirectChannelCompletionBeginCanceled: + return coroChanMatchDiscarded + case coro.DirectChannelCompletionBeginAcquired: + default: + return coroChanMatchInvalid + } + if status == waitSendOK { + copyChanElem(dst, waiter.elem, eltSize) + } + waiter.status = status + return coroChanMatchCommitted +} + +func commitDirectCoroSendWaiterLockedV1( + waiter *chanWaiter, + dst unsafe.Pointer, + eltSize int, + status waitStatus, + context *coroChanExternalCommitContextV1, +) coroChanMatchResult { + if result := prepareDirectCoroSendWaiterLockedV1( + waiter, dst, eltSize, status, + ); result != coroChanMatchCommitted { + return result + } + if !finishDirectCoroChannelCompletionV1(waiter, status, context) { + return coroChanMatchInvalid + } + return coroChanMatchCommitted +} + +// prepareCoroChannelExternalV1 samples the managed-owner capability before the +// no-return effect boundary. The common path reuses the same sample after the +// typed transfer; the exact direct shape may additionally reserve an empty +// same-P source slot for mailbox-free publication. +func prepareCoroChannelExternalV1( + transaction *coro.ChannelExternalCommit, + operation *coroChanOperationV1, + status waitStatus, +) coroChanExternalCommitContextV1 { + context := currentCoroChannelExternalContextV1() + return prepareCoroChannelExternalWithContextV1(transaction, operation, status, context) +} + +func prepareCoroChannelExternalWithContextV1( + transaction *coro.ChannelExternalCommit, + operation *coroChanOperationV1, + status waitStatus, + context coroChanExternalCommitContextV1, +) coroChanExternalCommitContextV1 { + context.ownerLocalDirect = false + context.directResult = coroChanDirectCommitCandidateV1(operation, status) + if context.directResult && + context.current != nil && context.driver != nil { + context.ownerLocalDirect = transaction.PrepareOwnerLocalDirect( + context.current, + context.driver, + ) + } + return context +} + +// beginCoroChannelExternalV1 is the queued-waiter direct ingress. The exact +// current-G capability is sampled once; on the common same-P shape the source +// transaction consumes it while acquiring admission, avoiding a second full +// endpoint audit after the select claim has already excluded its owner. +func beginCoroChannelExternalV1( + waiter *chanWaiter, + status waitStatus, + transaction *coro.ChannelExternalCommit, +) (coroChanMatchResult, coroChanExternalCommitContextV1) { + context := currentCoroChannelExternalContextV1() + return beginCoroChannelExternalWithContextV1(waiter, status, transaction, context) +} + +func beginCoroChannelExternalWithContextV1( + waiter *chanWaiter, + status waitStatus, + transaction *coro.ChannelExternalCommit, + context coroChanExternalCommitContextV1, +) (coroChanMatchResult, coroChanExternalCommitContextV1) { + return beginCoroChannelExternalValidatedWithContextV1( + waiter, status, transaction, context, + coroChanDirectCommitCandidateV1(waiter.coro, status), + ) +} + +func beginCoroChannelExternalValidatedWithContextV1( + waiter *chanWaiter, + status waitStatus, + transaction *coro.ChannelExternalCommit, + context coroChanExternalCommitContextV1, + directResult bool, +) (coroChanMatchResult, coroChanExternalCommitContextV1) { + context = resolveCoroChannelExternalContextV1(context) + context.ownerLocalDirect = false + context.directResult = directResult + direct := directResult && + context.current != nil && context.driver != nil + for { + var result coro.ChannelExternalCommitBeginResult + if direct { + result, context.ownerLocalDirect = coro.BeginChannelOwnerLocalDirectCommit( + transaction, + waiter.coro.source, + waiter.coro.id, + waiter.coro.claim, + context.current, + context.driver, + ) + } else { + result = coro.BeginChannelExternalCommit( + transaction, + waiter.coro.source, + waiter.coro.id, + waiter.coro.claim, + ) + } + classified := classifyCoroChanSingleBegin(result) + if classified != coroChanMatchRetry { + return classified, context + } + } +} + +// commitCoroChannelExternalV1 samples the current managed owner once for the +// complete post-effect tail. Previously CommitAtRoute and owner-local +// publication each repeated the TLS/runtime-context/driver proof, making a +// successful same-P rendezvous pay that full lookup three times. The physical +// transaction still publishes before any scheduler-local mutation, and a nil +// current retains the ordinary routed producer path. +func commitCoroChannelExternalV1( + transaction *coro.ChannelExternalCommit, + operation *coroChanOperationV1, + status waitStatus, + context coroChanExternalCommitContextV1, +) bool { + endpoint, endpointOK := coroChanCompletionEndpointForV1(operation) + if transaction == nil || !status.done() || !endpointOK { + return false + } + directResult := context.directResult + if context.ownerLocalDirect { + if !directResult { + return false + } + switch transaction.CommitOwnerLocalDirectWithResult(uint8(status)) { + case coro.ChannelOwnerLocalCommitted: + return finalizeCoroChanDirectResultV1(operation, false) + case coro.ChannelOwnerLocalCompletedInline: + return finalizeCoroChanDirectResultV1(operation, true) + case coro.ChannelOwnerLocalCommitFallback: + case coro.ChannelOwnerLocalCommitInvalid: + return false + default: + return false + } + } + var committed bool + if directResult { + committed = transaction.CommitAtRouteWithResult(context.route, uint8(status)) + } else { + committed = transaction.CommitAtRoute(context.route) + } + if !committed { + return false + } + local, localOK := publishCoroChannelOwnerLocalV1(context.current, context.driver, endpoint) + if !localOK { + return false + } + if local { + return !directResult || finalizeCoroChanDirectResultV1(operation, false) } - return coroTargetRequestChannelOperationV1(operation.id) + return coroTargetRequestChannelOperationV1(endpoint.id) } -func requestCoroChannelPairExecutorsV1(first, second *coroChanOperationV1) bool { - if first == nil || second == nil || !first.id.Valid() || !second.id.Valid() { +func commitCoroChannelExternalPairV1( + transaction *coro.ChannelExternalCommitPair, + first, second *coroChanOperationV1, +) bool { + firstEndpoint, firstEndpointOK := coroChanCompletionEndpointForV1(first) + secondEndpoint, secondEndpointOK := coroChanCompletionEndpointForV1(second) + if transaction == nil || !firstEndpointOK || !secondEndpointOK { return false } - firstLocal, firstOK := publishCoroChannelOwnerLocalV1(first) - secondLocal, secondOK := publishCoroChannelOwnerLocalV1(second) - if !firstOK || !secondOK { + firstDirect := coroChanDirectResultFinalizableV1(first, waitSendOK) + secondDirect := coroChanDirectResultFinalizableV1(second, waitRecvOK) + firstSmall, secondSmall := uint8(coro.ResumeSmallInvalid), uint8(coro.ResumeSmallInvalid) + if firstDirect { + firstSmall = uint8(waitSendOK) + } + if secondDirect { + secondSmall = uint8(waitRecvOK) + } + current, driver, route := coroCurrentTaskV1() + if !transaction.CommitAtRouteWithResults(route, firstSmall, secondSmall) || + firstEndpoint.source == nil || secondEndpoint.source == nil { + return false + } + firstLocal, firstOK := publishCoroChannelOwnerLocalV1(current, driver, firstEndpoint) + secondLocal, secondOK := publishCoroChannelOwnerLocalV1(current, driver, secondEndpoint) + if !firstOK || !secondOK || + firstLocal && firstDirect && !finalizeCoroChanDirectResultV1(first, false) || + secondLocal && secondDirect && !finalizeCoroChanDirectResultV1(second, false) { return false } - if !firstLocal && !coroTargetRequestChannelOperationV1(first.id) { + if !firstLocal && !coroTargetRequestChannelOperationV1(firstEndpoint.id) { return false } - if secondLocal || !firstLocal && first.id.Route() == second.id.Route() { + if secondLocal || !firstLocal && firstEndpoint.id.Route() == secondEndpoint.id.Route() { return true } - return coroTargetRequestChannelOperationV1(second.id) + return coroTargetRequestChannelOperationV1(secondEndpoint.id) } func commitCoroRecvWaiterLocked(w *chanWaiter, src unsafe.Pointer, eltSize int, status waitStatus) coroChanMatchResult { + return commitCoroRecvWaiterLockedWithContext( + w, src, eltSize, status, currentCoroChannelExternalContextV1(), + ) +} + +func commitCoroRecvWaiterLockedWithContext( + w *chanWaiter, + src unsafe.Pointer, + eltSize int, + status waitStatus, + context coroChanExternalCommitContextV1, +) coroChanMatchResult { if !validCoroChanOperationV1(w.coro, w) || w.send || w.size != eltSize || !status.done() || status == waitSendClosed { return coroChanMatchInvalid } + directResult := coroChanDirectCommitShapeV1(w.coro, status, false) var transaction coro.ChannelExternalCommit - for { - result := coro.BeginChannelExternalCommit( - &transaction, - w.coro.source, - w.coro.id, - w.coro.claim, - ) - classified := classifyCoroChanSingleBegin(result) - if classified == coroChanMatchRetry { - // Acquiring/Committing is a no-suspend claim critical section. Its - // holder never waits for hchan, so retrying under the already-held - // channel gate cannot form a lock cycle or strand a rendezvous. - continue - } - if classified != coroChanMatchCommitted { - return classified - } - break + classified, context := beginCoroChannelExternalValidatedWithContextV1( + w, status, &transaction, context, directResult, + ) + if classified != coroChanMatchCommitted { + return classified } if !transaction.BeginEffect() { return coroChanMatchInvalid @@ -362,33 +782,36 @@ func commitCoroRecvWaiterLocked(w *chanWaiter, src unsafe.Pointer, eltSize int, zeroChanRecv(w.elem, eltSize) } w.status = status - if !transaction.CommitAtRoute(coroCurrentTaskRouteV1()) || !requestCoroChannelExecutorV1(w.coro) { + if !commitCoroChannelExternalV1(&transaction, w.coro, status, context) { return coroChanMatchInvalid } return coroChanMatchCommitted } func commitCoroSendWaiterLocked(w *chanWaiter, dst unsafe.Pointer, eltSize int, status waitStatus) coroChanMatchResult { + return commitCoroSendWaiterLockedWithContext( + w, dst, eltSize, status, currentCoroChannelExternalContextV1(), + ) +} + +func commitCoroSendWaiterLockedWithContext( + w *chanWaiter, + dst unsafe.Pointer, + eltSize int, + status waitStatus, + context coroChanExternalCommitContextV1, +) coroChanMatchResult { if !validCoroChanOperationV1(w.coro, w) || !w.send || w.size != eltSize || (status != waitSendOK && status != waitSendClosed) { return coroChanMatchInvalid } + directResult := coroChanDirectCommitShapeV1(w.coro, status, false) var transaction coro.ChannelExternalCommit - for { - result := coro.BeginChannelExternalCommit( - &transaction, - w.coro.source, - w.coro.id, - w.coro.claim, - ) - classified := classifyCoroChanSingleBegin(result) - if classified == coroChanMatchRetry { - continue - } - if classified != coroChanMatchCommitted { - return classified - } - break + classified, context := beginCoroChannelExternalValidatedWithContextV1( + w, status, &transaction, context, directResult, + ) + if classified != coroChanMatchCommitted { + return classified } if !transaction.BeginEffect() { return coroChanMatchInvalid @@ -397,7 +820,7 @@ func commitCoroSendWaiterLocked(w *chanWaiter, dst unsafe.Pointer, eltSize int, copyChanElem(dst, w.elem, eltSize) } w.status = status - if !transaction.CommitAtRoute(coroCurrentTaskRouteV1()) || !requestCoroChannelExecutorV1(w.coro) { + if !commitCoroChannelExternalV1(&transaction, w.coro, status, context) { return coroChanMatchInvalid } return coroChanMatchCommitted @@ -437,7 +860,7 @@ func commitCoroPairLocked(send, recv *chanWaiter, eltSize int) coroChanMatchResu copyChanElem(recv.elem, send.elem, eltSize) send.status = waitSendOK recv.status = waitRecvOK - if !transaction.CommitAtRoute(coroCurrentTaskRouteV1()) || !requestCoroChannelPairExecutorsV1(send.coro, recv.coro) { + if !commitCoroChannelExternalPairV1(&transaction, send.coro, recv.coro) { return coroChanMatchInvalid } return coroChanMatchCommitted @@ -466,12 +889,15 @@ func finishCurrentCoroChannelCommit( transaction *coro.ChannelExternalCommit, status waitStatus, ) bool { - if !validCoroChanOperationV1(waiter.coro, waiter) || transaction == nil || !status.done() || - !transaction.BeginEffect() { + if !validCoroChanOperationV1(waiter.coro, waiter) || transaction == nil || !status.done() { + return false + } + context := prepareCoroChannelExternalV1(transaction, waiter.coro, status) + if !transaction.BeginEffect() { return false } waiter.status = status - return transaction.CommitAtRoute(coroCurrentTaskRouteV1()) && requestCoroChannelExecutorV1(waiter.coro) + return commitCoroChannelExternalV1(transaction, waiter.coro, status, context) } func coroChanTrySendLocked(ch *Chan, waiter *chanWaiter) (ready bool, ok bool) { @@ -520,21 +946,25 @@ func coroChanTrySendLocked(ch *Chan, waiter *chanWaiter) (ready bool, ok bool) { } continue } + context := prepareCoroChannelExternalV1(&transaction, waiter.coro, waitSendOK) if !transaction.BeginEffect() { return false, false } copyChanElem(peer.elem, waiter.elem, ch.elemsize) waiter.status = waitSendOK peer.finish(waitRecvOK) - if !transaction.CommitAtRoute(coroCurrentTaskRouteV1()) || !requestCoroChannelExecutorV1(waiter.coro) { + if !commitCoroChannelExternalV1(&transaction, waiter.coro, waitSendOK, context) { return false, false } return true, true } if ch.qcount < ch.dataqsiz { var transaction coro.ChannelExternalCommit - if beginCurrentCoroChannelCommit(waiter, &transaction) != coroChanMatchCommitted || - !transaction.BeginEffect() { + if beginCurrentCoroChannelCommit(waiter, &transaction) != coroChanMatchCommitted { + return false, false + } + context := prepareCoroChannelExternalV1(&transaction, waiter.coro, waitSendOK) + if !transaction.BeginEffect() { return false, false } copyChanElem(chanBuf(ch, ch.sendx), waiter.elem, ch.elemsize) @@ -544,7 +974,7 @@ func coroChanTrySendLocked(ch *Chan, waiter *chanWaiter) (ready bool, ok bool) { } ch.qcount++ waiter.status = waitSendOK - if !transaction.CommitAtRoute(coroCurrentTaskRouteV1()) || !requestCoroChannelExecutorV1(waiter.coro) { + if !commitCoroChannelExternalV1(&transaction, waiter.coro, waitSendOK, context) { return false, false } return true, true @@ -591,21 +1021,25 @@ func coroChanTryRecvLocked(ch *Chan, waiter *chanWaiter) (ready bool, ok bool) { } continue } + context := prepareCoroChannelExternalV1(&transaction, waiter.coro, waitRecvOK) if !transaction.BeginEffect() { return false, false } copyChanElem(waiter.elem, peer.elem, ch.elemsize) waiter.status = waitRecvOK peer.finish(waitSendOK) - if !transaction.CommitAtRoute(coroCurrentTaskRouteV1()) || !requestCoroChannelExecutorV1(waiter.coro) { + if !commitCoroChannelExternalV1(&transaction, waiter.coro, waitRecvOK, context) { return false, false } return true, true } } else if ch.qcount > 0 { var transaction coro.ChannelExternalCommit - if beginCurrentCoroChannelCommit(waiter, &transaction) != coroChanMatchCommitted || - !transaction.BeginEffect() { + if beginCurrentCoroChannelCommit(waiter, &transaction) != coroChanMatchCommitted { + return false, false + } + context := prepareCoroChannelExternalV1(&transaction, waiter.coro, waitRecvOK) + if !transaction.BeginEffect() { return false, false } copyChanElem(waiter.elem, chanBuf(ch, ch.recvx), ch.elemsize) @@ -616,24 +1050,27 @@ func coroChanTryRecvLocked(ch *Chan, waiter *chanWaiter) (ready bool, ok bool) { } ch.qcount-- waiter.status = waitRecvOK - if !transaction.CommitAtRoute(coroCurrentTaskRouteV1()) || !requestCoroChannelExecutorV1(waiter.coro) { + if !commitCoroChannelExternalV1(&transaction, waiter.coro, waitRecvOK, context) { return false, false } // Refill is a separate committed sender endpoint under the same hchan // lock. Failure leaves the now-available buffer slot visible to a later // sender without changing the completed receive. - dequeueSendToBuffer(ch) + dequeueSendToBuffer(ch, &context) return true, true } if ch.closed { var transaction coro.ChannelExternalCommit - if beginCurrentCoroChannelCommit(waiter, &transaction) != coroChanMatchCommitted || - !transaction.BeginEffect() { + if beginCurrentCoroChannelCommit(waiter, &transaction) != coroChanMatchCommitted { + return false, false + } + context := prepareCoroChannelExternalV1(&transaction, waiter.coro, waitRecvClosed) + if !transaction.BeginEffect() { return false, false } zeroChanRecv(waiter.elem, ch.elemsize) waiter.status = waitRecvClosed - if !transaction.CommitAtRoute(coroCurrentTaskRouteV1()) || !requestCoroChannelExecutorV1(waiter.coro) { + if !commitCoroChannelExternalV1(&transaction, waiter.coro, waitRecvClosed, context) { return false, false } return true, true @@ -653,7 +1090,10 @@ const ( // dequeueSendToBufferStepLocked examines and removes at most one queued // sender. The caller owns ch.mutex. -func dequeueSendToBufferStepLocked(ch *Chan) coroChanQueueStepV1 { +func dequeueSendToBufferStepLocked( + ch *Chan, + context *coroChanExternalCommitContextV1, +) coroChanQueueStepV1 { if ch == nil || ch.closed || ch.qcount >= ch.dataqsiz { return coroChanQueueIdleV1 } @@ -661,8 +1101,30 @@ func dequeueSendToBufferStepLocked(ch *Chan) coroChanQueueStepV1 { if w == nil { return coroChanQueueIdleV1 } + if w.direct != nil { + result := commitDirectCoroSendWaiterLockedV1( + w, chanBuf(ch, ch.sendx), ch.elemsize, waitSendOK, context, + ) + switch result { + case coroChanMatchCommitted: + ch.sendx++ + if ch.sendx == ch.dataqsiz { + ch.sendx = 0 + } + ch.qcount++ + return coroChanQueueCommittedV1 + case coroChanMatchDiscarded: + return coroChanQueueDiscardedV1 + default: + return coroChanQueueInvalidV1 + } + } if w.coro != nil { - switch result := commitCoroSendWaiterLocked(w, chanBuf(ch, ch.sendx), ch.elemsize, waitSendOK); result { + result := commitCoroSendWaiterLockedWithContext( + w, chanBuf(ch, ch.sendx), ch.elemsize, waitSendOK, + coroChanExternalContextValueV1(context), + ) + switch result { case coroChanMatchCommitted: ch.sendx++ if ch.sendx == ch.dataqsiz { @@ -692,9 +1154,12 @@ func dequeueSendToBufferStepLocked(ch *Chan) coroChanQueueStepV1 { return coroChanQueueCommittedV1 } -func dequeueSendToBufferLocked(ch *Chan) (progress, ok bool) { +func dequeueSendToBufferLocked( + ch *Chan, + context *coroChanExternalCommitContextV1, +) (progress, ok bool) { for { - switch dequeueSendToBufferStepLocked(ch) { + switch dequeueSendToBufferStepLocked(ch, context) { case coroChanQueueCommittedV1: return true, true case coroChanQueueDiscardedV1: @@ -707,8 +1172,8 @@ func dequeueSendToBufferLocked(ch *Chan) (progress, ok bool) { } } -func dequeueSendToBuffer(ch *Chan) { - if _, ok := dequeueSendToBufferLocked(ch); !ok { +func dequeueSendToBuffer(ch *Chan, context *coroChanExternalCommitContextV1) { + if _, ok := dequeueSendToBufferLocked(ch, context); !ok { coroRuntimeAbort("invalid coroutine channel buffer refill") } } @@ -716,7 +1181,10 @@ func dequeueSendToBuffer(ch *Chan) { // dequeueBufferToRecvStepLocked examines and removes at most one queued // receiver. The buffer position advances only after the exact receiver // transaction has committed its typed copy. -func dequeueBufferToRecvStepLocked(ch *Chan) coroChanQueueStepV1 { +func dequeueBufferToRecvStepLocked( + ch *Chan, + context *coroChanExternalCommitContextV1, +) coroChanQueueStepV1 { if ch == nil || ch.qcount == 0 { return coroChanQueueIdleV1 } @@ -724,7 +1192,9 @@ func dequeueBufferToRecvStepLocked(ch *Chan) coroChanQueueStepV1 { if w == nil { return coroChanQueueIdleV1 } - switch result := completeRecvWaiter(w, chanBuf(ch, ch.recvx), ch.elemsize, waitRecvOK); result { + switch result := completeRecvWaiterWithContext( + w, chanBuf(ch, ch.recvx), ch.elemsize, waitRecvOK, context, + ); result { case coroChanMatchCommitted: zeroChanRecv(chanBuf(ch, ch.recvx), ch.elemsize) ch.recvx++ @@ -746,9 +1216,12 @@ func dequeueBufferToRecvStepLocked(ch *Chan) coroChanQueueStepV1 { // dequeueBufferToRecvLocked restores the ordinary buffered-channel invariant // after claim contention temporarily leaves both queued receivers and buffered // data. -func dequeueBufferToRecvLocked(ch *Chan) (progress, ok bool) { +func dequeueBufferToRecvLocked( + ch *Chan, + context *coroChanExternalCommitContextV1, +) (progress, ok bool) { for { - switch dequeueBufferToRecvStepLocked(ch) { + switch dequeueBufferToRecvStepLocked(ch, context) { case coroChanQueueCommittedV1: return true, true case coroChanQueueDiscardedV1: @@ -766,21 +1239,25 @@ func dequeueBufferToRecvLocked(ch *Chan) (progress, ok bool) { // slots from queued senders. Claim contention stops this bounded pass; the // winning/canceled coroutine's resume tail invokes it again after removing the // contended node. -func reconcileBufferedChanLocked(ch *Chan, refill bool) bool { +func reconcileBufferedChanLocked( + ch *Chan, + refill bool, + context *coroChanExternalCommitContextV1, +) bool { if ch == nil || ch.dataqsiz == 0 { return true } for { progress := false if ch.qcount > 0 { - consumed, ok := dequeueBufferToRecvLocked(ch) + consumed, ok := dequeueBufferToRecvLocked(ch, context) if !ok { return false } progress = consumed } if refill && !ch.closed && ch.qcount < ch.dataqsiz { - filled, ok := dequeueSendToBufferLocked(ch) + filled, ok := dequeueSendToBufferLocked(ch, context) if !ok { return false } @@ -851,23 +1328,52 @@ func ensureCoroChannelOperationCapacityV1( return false } for !coro.CanReserveChannelOperations(p, source, needed) { - if coro.ChannelOperationConfiguredCapacity(source) >= coro.ChannelOperationMaximumCapacity { - return false - } - page := new(coro.ChannelOperationPage) - if page == nil { + if !growCoroChannelOperationCapacityV1(p, source) { return false } - attached := coro.AttachChannelOperationPage(source, p, page, nil) - if !attached { - block := new(coro.OperationPageDirectoryBlock) - attached = block != nil && coro.AttachChannelOperationPage(source, p, page, block) + } + return true +} + +func growCoroChannelOperationCapacityV1( + p *coro.P, + source *coro.ChannelOperationSource, +) bool { + if p == nil || source == nil || + coro.ChannelOperationConfiguredCapacity(source) >= coro.ChannelOperationMaximumCapacity { + return false + } + page := new(coro.ChannelOperationPage) + if page == nil { + return false + } + attached := coro.AttachChannelOperationPage(source, p, page, nil) + if !attached { + block := new(coro.OperationPageDirectoryBlock) + attached = block != nil && coro.AttachChannelOperationPage(source, p, page, block) + } + return attached +} + +// prepareCoroChannelDirectReservationV1 combines the direct park's capacity +// check with selection of its exact reusable slot. The opaque capability is +// consumed before any suspension, eliminating the second catalog scan which +// the general select-capacity API necessarily performs. +func prepareCoroChannelDirectReservationV1( + p *coro.P, + source *coro.ChannelOperationSource, +) (coro.ChannelDirectReservation, bool) { + if p == nil || source == nil { + return coro.ChannelDirectReservation{}, false + } + for { + if reservation, ok := source.PreflightDirectReservation(p); ok { + return reservation, true } - if !attached { - return false + if !growCoroChannelOperationCapacityV1(p, source) { + return coro.ChannelDirectReservation{}, false } } - return true } func prepareCoroChanSelectV1( @@ -914,9 +1420,8 @@ func prepareCoroChanSelectV1( return } sortCoroChanSelectOrder(candidates, ops) - driver, _, route, current := coro.CurrentExecutorChannelDriver(task) - p, park, source, ownerOK := coro.CurrentExecutorChannelParkOwner(driver, task) - if !current || !ownerOK || !ensureCoroChannelOperationCapacityV1(p, source, physical) { + _, _, route, p, park, source, current := coro.CurrentExecutorChannelParkContext(task) + if !current || !ensureCoroChannelOperationCapacityV1(p, source, physical) { coroRuntimeAbort("coroutine channel select source capacity exhausted") return } @@ -1111,92 +1616,272 @@ func prepareCoroChanParkV1( return } state := (*CoroChanParkV1)(storage) - *state = CoroChanParkV1{} + // The frame allocator and the prior resume own whole-state clearing. Doing + // it again here made every actual park clear this large record twice. ch := (*Chan)(channel) size := int(eltSize) state.magic = coroChanParkMagicV1 - state.operation = coroChanOperationV1{claim: &state.claim, waiter: &state.waiter} - state.waiter = chanWaiter{ch: ch, elem: elem, size: size, send: send, coro: &state.operation} - if ch == nil { - ticket, ok := coro.PrepareEmptyChannelPark( - (*coro.G)(g), handle, (*coro.HeaderV1)(header), &state.wait, fastrand(), - ) - if !ok { - coroRuntimeAbort("cannot prepare nil coroutine channel park") - return - } - state.ticket = ticket - if !coro.BindSingleWaitSetResumePacket(&state.wait, &state.packet, coro.OperationID{}) { - coroRuntimeAbort("cannot bind nil coroutine channel resume") - } - return - } - if size != ch.elemsize { + if ch != nil && size != ch.elemsize { coroRuntimeAbort("coroutine channel element size mismatch") return } task := (*coro.G)(g) - driver, _, route, current := coro.CurrentExecutorChannelDriver(task) - p, _, source, ownerOK := coro.CurrentExecutorChannelParkOwner(driver, task) - if !current || !ownerOK || !ensureCoroChannelOperationCapacityV1(p, source, 1) { - coroRuntimeAbort("cannot resolve coroutine channel park owner") - return + state.waiter = chanWaiter{ + ch: ch, elem: elem, size: size, send: send, direct: &state.Completion, } - ticket, id, ok := coro.PrepareSingleChannelPark( + driver, route := coro.PrepareCurrentDirectChannelPark( task, handle, (*coro.HeaderV1)(header), - source, - &state.wait, - &state.claim, - 1, - fastrand(), + &state.DirectChannelParkStorageV1, ) - if !ok || id.Route() != route { - coroRuntimeAbort("cannot prepare coroutine channel park") + if driver == nil { + coroRuntimeAbort("cannot prepare compact coroutine channel park") return } - state.ticket = ticket - state.operation.id = id - state.operation.source = source - state.operation.magic = coroChanOperationMagicV1 - if !coro.BindWaitSetResumeCleanup( - &state.wait, - &state.packet, - &state.cleanup, - coro.ResumeCleanupBinding{ - Kind: coro.ResumeCleanupChannelDirect, - Context: unsafe.Pointer(state), - Entries: unsafe.Pointer(&state.operation.id), - Claim: &state.claim, - Count: 1, - RuntimeCount: 1, - Stride: unsafe.Sizeof(coro.OperationID{}), - }, - ) { - coroRuntimeAbort("cannot bind coroutine channel cleanup") + // A nil channel has no physical endpoint. Its compact record remains bound + // until task cancellation publishes it to the same owner inbox. + if ch == nil { return } + // Preparation has already authenticated the compiler-carried task and + // frozen its exact executor route for this no-suspend runtime call. Reuse + // that certificate instead of repeating the G/P/driver lookup before the + // hchan transaction. + context := coroChanExternalCommitContextV1{ + current: task, + driver: driver, + route: route, + } ch.mutex.Lock() - var ready bool + ready := false + status := waitPending if send { - ready, ok = coroChanTrySendLocked(ch, &state.waiter) + tryOK, closed := chanTrySendLockedWithContext(ch, elem, size, &context) + ready = tryOK || closed + if closed { + status = waitSendClosed + } else if tryOK { + status = waitSendOK + } } else { - ready, ok = coroChanTryRecvLocked(ch, &state.waiter) + recvOK, tryOK := chanTryRecvLockedWithContext(ch, elem, size, &context) + ready = tryOK + if tryOK && recvOK { + status = waitRecvOK + } else if tryOK { + status = waitRecvClosed + } } - if !ok { - ch.mutex.Unlock() - coroRuntimeAbort("cannot commit coroutine channel park") - return + if ready { + if coro.BeginDirectChannelCompletion(&state.Completion) != + coro.DirectChannelCompletionBeginAcquired { + ch.mutex.Unlock() + coroRuntimeAbort("cannot claim compact coroutine channel result") + return + } + state.waiter.status = status + if !finishDirectCoroChannelCompletionV1(&state.waiter, status, &context) { + ch.mutex.Unlock() + coroRuntimeAbort("cannot publish compact coroutine channel result") + return + } + } else if send { + ch.sendq.enqueue(&state.waiter) + } else { + ch.recvq.enqueue(&state.waiter) + } + ch.mutex.Unlock() +} + +func prepareCoroChanParkStateV2( + task *coro.G, + handle unsafe.Pointer, + frameHeader *coro.HeaderV1, + state *CoroChanParkV1, + ch *Chan, + elem unsafe.Pointer, + size int, + send bool, + stateID, line uint32, +) (*coro.ExecutorDriver, coro.RouteID) { + // magic is the compiler-spill lifecycle capability. Fresh coroutine frames + // are zero-filled, and the direct resume prologue clears magic only after + // every embedded wait/completion/waiter record has been retired. Reject a + // second prepare before mutating the frame header; the issued core path may + // then initialize only live fields instead of re-zeroing the whole spill. + if state == nil || state.magic != 0 || state.Ticket.Valid() { + return nil, 0 + } + frameHeader.SuspendReason = uint16(coro.SuspendPark) + frameHeader.Lifecycle = uint16(coro.FrameSuspended) + frameHeader.StateID = stateID + frameHeader.Line = line + // magic == 0 certifies that the prior resume retired every embedded record. + // Initialize only the five words consumed by the hchan queue instead of + // materializing a mostly-zero chanWaiter aggregate on every handoff. + state.waiter.ch = ch + state.waiter.elem = elem + state.waiter.size = size + state.waiter.send = send + state.waiter.direct = &state.Completion + driver, route := coro.PrepareCurrentDirectChannelPark( + task, + handle, + frameHeader, + &state.DirectChannelParkStorageV1, + ) + if driver == nil { + return nil, 0 + } + // Publish the compiler-spill lifecycle capability only after the core has + // accepted the park. A nil result is terminal for every caller and never + // exposes partially initialized state as live. + state.magic = coroChanParkMagicV1 + return driver, route +} + +// tryOrParkCoroChanV2 is the single-lock compiler transaction for an ordinary +// one-case channel operation. A ready endpoint returns its typed status without +// touching the coroutine header or park storage. Only the not-ready edge +// publishes SuspendPark, builds the compact wait graph, and exposes the hchan +// waiter while the same channel critical section is still held. This removes +// the former Try-unlock-Park-lock sequence without giving the hchan ownership +// of a coroutine handle or scheduler queue. +func tryOrParkCoroChanV2( + g, handle, header, channel, elem, storage unsafe.Pointer, + eltSize uintptr, + stateID, line uint32, + send bool, +) uint32 { + if g == nil || handle == nil || header == nil || elem == nil || storage == nil || + eltSize > uintptr(^uint(0)>>1) { + coroRuntimeAbort("invalid coroutine channel try-or-park ABI") + return coroChanResumeInvalid + } + task := (*coro.G)(g) + frameHeader := (*coro.HeaderV1)(header) + if frameHeader.G != g || frameHeader.SuspendReason != uint16(coro.SuspendNone) || + frameHeader.Lifecycle != uint16(coro.FrameActive) { + coroRuntimeAbort("invalid active coroutine channel frame") + return coroChanResumeInvalid + } + state := (*CoroChanParkV1)(storage) + ch := (*Chan)(channel) + size := int(eltSize) + if ch != nil && size != ch.elemsize { + coroRuntimeAbort("coroutine channel element size mismatch") + return coroChanResumeInvalid + } + + // A nil channel has no physical critical section. It publishes only the + // cancellation-owned compact park and always reaches llvm.coro.suspend. + if ch == nil { + if driver, _ := prepareCoroChanParkStateV2( + task, handle, frameHeader, state, ch, elem, size, send, stateID, line, + ); driver == nil { + coroRuntimeAbort("cannot prepare nil coroutine channel park") + return coroChanResumeInvalid + } + return coroChanResumeInvalid + } + + ch.mutex.Lock() + var resolved coroChanExternalCommitContextV1 + context := (*coroChanExternalCommitContextV1)(nil) + if send && ch.recvq.first != nil || !send && ch.sendq.first != nil { + resolved = coroChannelExternalContextForTaskV1(task) + context = &resolved } - if !ready { - if send { + if send { + ready, closed := chanTrySendLockedWithContext(ch, elem, size, context) + if ready && !closed { + ch.mutex.Unlock() + return coroChanResumeSendOK + } + if !closed { + driver, _ := prepareCoroChanParkStateV2( + task, handle, frameHeader, state, ch, elem, size, send, stateID, line, + ) + if driver == nil { + ch.mutex.Unlock() + coroRuntimeAbort("cannot prepare coroutine channel send park") + return coroChanResumeInvalid + } ch.sendq.enqueue(&state.waiter) - } else { - ch.recvq.enqueue(&state.waiter) + ch.mutex.Unlock() + return coroChanResumeInvalid } + // Preserve the existing explicit-status fault route: a send on a closed + // channel materializes a typed completion and reports the fault only from + // the post-suspend resume gate, never by unwinding across llvm.coro.resume. + driver, route := prepareCoroChanParkStateV2( + task, handle, frameHeader, state, ch, elem, size, send, stateID, line, + ) + if driver == nil { + ch.mutex.Unlock() + coroRuntimeAbort("cannot prepare closed coroutine channel send") + return coroChanResumeInvalid + } + closedContext := coroChanExternalCommitContextV1{ + current: task, driver: driver, route: route, + } + if coro.BeginDirectChannelCompletion(&state.Completion) != + coro.DirectChannelCompletionBeginAcquired { + ch.mutex.Unlock() + coroRuntimeAbort("cannot claim closed coroutine channel send") + return coroChanResumeInvalid + } + state.waiter.status = waitSendClosed + if !finishDirectCoroChannelCompletionV1(&state.waiter, waitSendClosed, &closedContext) { + ch.mutex.Unlock() + coroRuntimeAbort("cannot publish closed coroutine channel send") + return coroChanResumeInvalid + } + ch.mutex.Unlock() + return coroChanResumeInvalid + } + + recvOK, ready := chanTryRecvLockedWithContext(ch, elem, size, context) + if ready { + ch.mutex.Unlock() + if recvOK { + return coroChanResumeRecvOK + } + return coroChanResumeRecvClosed + } + if driver, _ := prepareCoroChanParkStateV2( + task, handle, frameHeader, state, ch, elem, size, send, stateID, line, + ); driver == nil { + ch.mutex.Unlock() + coroRuntimeAbort("cannot prepare coroutine channel receive park") + return coroChanResumeInvalid } + ch.recvq.enqueue(&state.waiter) ch.mutex.Unlock() + return coroChanResumeInvalid +} + +//export __llgo_coro_chan_send_try_park_v2 +func __llgo_coro_chan_send_try_park_v2( + g, handle, header, channel, elem, storage unsafe.Pointer, + eltSize uintptr, + stateID, line uint32, +) uint32 { + return tryOrParkCoroChanV2( + g, handle, header, channel, elem, storage, eltSize, stateID, line, true, + ) +} + +//export __llgo_coro_chan_recv_try_park_v2 +func __llgo_coro_chan_recv_try_park_v2( + g, handle, header, channel, elem, storage unsafe.Pointer, + eltSize uintptr, + stateID, line uint32, +) uint32 { + return tryOrParkCoroChanV2( + g, handle, header, channel, elem, storage, eltSize, stateID, line, false, + ) } //export __llgo_coro_chan_send_park_v1 @@ -1215,45 +1900,52 @@ func __llgo_coro_chan_recv_park_v1( prepareCoroChanParkV1(g, handle, header, channel, elem, storage, eltSize, false) } -//export __llgo_coro_chan_resume_v1 -func __llgo_coro_chan_resume_v1(g, storage unsafe.Pointer) uint32 { +// coroChanResumeCompatibilityV1 remains available to the runtime's manually +// driven adapter tests. Current compiler lowering uses the issued V2 ABI below +// and does not retain this function as an external binary entry. +func coroChanResumeCompatibilityV1(g, storage unsafe.Pointer) uint32 { state := (*CoroChanParkV1)(storage) - if g == nil || !validCoroChanParkV1(state) { + if g == nil || state == nil || state.magic != coroChanParkMagicV1 || + state.Ticket == (coro.ParkTicket{}) { coroRuntimeAbort("invalid coroutine channel resume ABI") return coroChanResumeInvalid } - outcome, caseID, task, result, small, ok := coro.TakeResumePacket( + outcome, task, small, ok := coro.TakeDirectChannelResume( (*coro.G)(g), - state.ticket, - &state.packet, - nil, + &state.DirectChannelParkStorageV1, ) if !ok { coroRuntimeAbort("invalid coroutine channel resume packet") return coroChanResumeInvalid } - if result == coro.ResumeResultNone { - if outcome != coro.ParkOutcomeCanceled || caseID != 0 || small != coro.ResumeSmallInvalid { - coroRuntimeAbort("invalid nil-channel run decision") + if outcome == coro.ParkOutcomeCanceled { + if small != coro.ResumeSmallInvalid { + coroRuntimeAbort("invalid canceled channel run decision") return coroChanResumeInvalid } - *state = CoroChanParkV1{} + // Promotion/materialization and TakeDirectChannelResume have already + // cleared wait, waiter, and completion ownership. Only these two scalar + // compiler receipts remain live; clearing the entire spill record here + // rewrote roughly two hundred already-zero bytes on every handoff. + state.Ticket = coro.ParkTicket{} + state.magic = 0 switch task { case coro.TaskCancelAbort: return coroChanResumeTaskAbort case coro.TaskCancelShutdown: return coroChanResumeShutdown default: - coroRuntimeAbort("nil-channel park resumed without task cancellation") + coroRuntimeAbort("channel park resumed without task cancellation") return coroChanResumeInvalid } } - if result != coro.ResumeResultChannel || outcome != coro.ParkOutcomeCompleted || - caseID != 1 || task != coro.TaskCancelNone || small == coro.ResumeSmallInvalid { + if outcome != coro.ParkOutcomeCompleted || task != coro.TaskCancelNone || + small == coro.ResumeSmallInvalid { coroRuntimeAbort("invalid materialized coroutine channel decision") return coroChanResumeInvalid } - *state = CoroChanParkV1{} + state.Ticket = coro.ParkTicket{} + state.magic = 0 switch waitStatus(small) { case waitSendOK: return coroChanResumeSendOK @@ -1269,6 +1961,58 @@ func __llgo_coro_chan_resume_v1(g, storage unsafe.Pointer) uint32 { } } +// __llgo_coro_chan_resume_v2 consumes only a bounded runner's issued physical +// resume. The compiler already uses the matching V2 try-or-park transaction; +// keeping its resume half exact removes the arbitrary-runner fallback from +// every generated one-case channel continuation. +// +//export __llgo_coro_chan_resume_v2 +func __llgo_coro_chan_resume_v2(g, storage unsafe.Pointer) uint32 { + state := (*CoroChanParkV1)(storage) + if g == nil || state == nil || state.magic != coroChanParkMagicV1 || + state.Ticket == (coro.ParkTicket{}) { + coroRuntimeAbort("invalid coroutine channel resume ABI") + return coroChanResumeInvalid + } + word := coro.TakeIssuedDirectChannelResumeWordV1( + (*coro.G)(g), + &state.DirectChannelParkStorageV1, + ) + class := word & coro.DirectChannelResumeWordClassMaskV1 + payload := uint8(word & coro.DirectChannelResumeWordPayloadMaskV1) + if class != coro.DirectChannelResumeWordCompletedV1 && + class != coro.DirectChannelResumeWordCanceledV1 { + coroRuntimeAbort("invalid issued coroutine channel resume packet") + return coroChanResumeInvalid + } + state.Ticket = coro.ParkTicket{} + state.magic = 0 + if class == coro.DirectChannelResumeWordCanceledV1 { + switch coro.TaskCancelKind(payload) { + case coro.TaskCancelAbort: + return coroChanResumeTaskAbort + case coro.TaskCancelShutdown: + return coroChanResumeShutdown + default: + coroRuntimeAbort("channel park resumed without task cancellation") + return coroChanResumeInvalid + } + } + switch waitStatus(payload) { + case waitSendOK: + return coroChanResumeSendOK + case waitRecvOK: + return coroChanResumeRecvOK + case waitRecvClosed: + return coroChanResumeRecvClosed + case waitSendClosed: + return coroChanResumeSendClosed + default: + coroRuntimeAbort("invalid coroutine channel completion status") + return coroChanResumeInvalid + } +} + func finishCoroChanCleanupCursorV1( cursor *coroChanCleanupCursorV1, ) (small uint8, complete bool, ok bool) { @@ -1303,7 +2047,7 @@ func advanceCoroChanCleanupCursorV1( cursor.phase = coroChanCleanupBufferSendV1 break } - switch dequeueBufferToRecvStepLocked(ch) { + switch dequeueBufferToRecvStepLocked(ch, nil) { case coroChanQueueCommittedV1, coroChanQueueDiscardedV1: case coroChanQueueIdleV1, coroChanQueueBlockedV1: cursor.phase = coroChanCleanupBufferSendV1 @@ -1313,7 +2057,7 @@ func advanceCoroChanCleanupCursorV1( } case coroChanCleanupBufferSendV1: if ch.dataqsiz != 0 && !ch.closed && ch.qcount < ch.dataqsiz { - switch dequeueSendToBufferStepLocked(ch) { + switch dequeueSendToBufferStepLocked(ch, nil) { case coroChanQueueCommittedV1: cursor.phase = coroChanCleanupBufferRecvV1 case coroChanQueueDiscardedV1: @@ -1337,7 +2081,7 @@ func advanceCoroChanCleanupCursorV1( ch.mutex.Unlock() return 0, false, false } - switch dequeueClosedRecvStepLocked(ch) { + switch dequeueClosedRecvStepLocked(ch, nil) { case coroChanQueueCommittedV1, coroChanQueueDiscardedV1: case coroChanQueueIdleV1: cursor.phase = coroChanCleanupClosedSendV1 @@ -1352,7 +2096,7 @@ func advanceCoroChanCleanupCursorV1( ch.mutex.Unlock() return 0, false, false } - switch dequeueClosedSendStepLocked(ch) { + switch dequeueClosedSendStepLocked(ch, nil) { case coroChanQueueCommittedV1, coroChanQueueDiscardedV1: case coroChanQueueIdleV1, coroChanQueueBlockedV1: finish = true @@ -1413,10 +2157,24 @@ func materializeCoroChanOperationV1( } else { ch.recvq.remove(waiter) } + // A completed rendezvous on an open unbuffered channel has no buffer or + // closed-channel reconciliation work. The old bounded cursor would perform + // two phase-only reductions before reaching the same result. Finish this + // exact zero-peer shape while the channel snapshot is still protected; all + // buffered, closed, and potentially cascading cases retain the resumable + // one-peer-per-reduction cleanup path below. + directRendezvous := deliver && ch.dataqsiz == 0 && !ch.closed ch.mutex.Unlock() id := operation.id *waiter = chanWaiter{} *operation = coroChanOperationV1{id: id} + if directRendezvous { + small := uint8(coro.ResumeSmallInvalid) + if deliver { + small = uint8(status) + } + return small, true, true + } *cursor = coroChanCleanupCursorV1{ ch: ch, status: status, @@ -1435,21 +2193,6 @@ func coroMaterializeChannelResumeCleanupStepV1(step coro.ResumeCleanupStep) bool ok bool ) switch step.Kind { - case coro.ResumeCleanupChannelDirect: - if step.Index != 0 { - return false - } - state := (*CoroChanParkV1)(step.Context) - if state == nil || state.magic != coroChanParkMagicV1 { - return false - } - small, complete, ok = materializeCoroChanOperationV1( - &state.operation, - &state.waiter, - &state.reconcile, - deliver, - allowCommittedCancel, - ) case coro.ResumeCleanupChannelSelect: state := (*CoroChanSelectV1)(step.Context) if state == nil || state.magic != coroChanSelectMagicV1 || step.Index >= uint32(state.count) || @@ -1475,3 +2218,43 @@ func coroMaterializeChannelResumeCleanupStepV1(step coro.ResumeCleanupStep) bool } return coro.CommitResumeCleanupStep(step, small) } + +func coroMaterializeDirectChannelCompletionV1( + completion *coro.DirectChannelCompletion, +) bool { + context, small, matched, ok := coro.DirectChannelCompletionSnapshot(completion) + if !ok || context == nil { + return false + } + state := (*CoroChanParkV1)(context) + if state == nil || state.magic != coroChanParkMagicV1 || state.waiter.direct != completion || + state.waiter.coro != nil { + return false + } + waiter, ch := &state.waiter, state.waiter.ch + if ch == nil { + if matched || waiter.queued || waiter.status != waitPending { + return false + } + *waiter = chanWaiter{} + return coro.CommitDirectChannelCompletion(completion, small) + } + ch.mutex.Lock() + if waiter.send { + ch.sendq.remove(waiter) + } else { + ch.recvq.remove(waiter) + } + if matched { + if small == coro.ResumeSmallInvalid || !waitStatus(small).done() || waiter.status != waitStatus(small) { + ch.mutex.Unlock() + return false + } + } else if waiter.status != waitPending { + ch.mutex.Unlock() + return false + } + *waiter = chanWaiter{} + ch.mutex.Unlock() + return coro.CommitDirectChannelCompletion(completion, small) +} diff --git a/runtime/poll_worker_source_test.go b/runtime/poll_worker_source_test.go index 1d1322e02f..86d88b0f7b 100644 --- a/runtime/poll_worker_source_test.go +++ b/runtime/poll_worker_source_test.go @@ -226,8 +226,10 @@ func TestRuntimeCoroWorkerCapacityUsesPagedLogicalSourceAndBoundedNativePool(t * "coroNativeWorkerCapacityV1 = coroNativeWorkerPageCountV1 * coro.WorkerOperationPageCapacity", "coroNativeWorkerQueueSizeV1 = coroworker.QueueCapacity", "bounded C11 sequence ring", - "coroworker.QueueReserve(&reservation)", - "coroworker.QueueSubmitReserved(reservation, &job)", + "reservation := coroworker.QueueReserve()", + "coroworker.QueueSubmitReserved(", + "id.SourceSlot,", + "id.Generation,", "coroNativeWorkerDeliveryFleetV1", "func coroNativeWorkerPoolStartFleetV1() bool", } { @@ -278,7 +280,7 @@ func TestRuntimeCoroWorkerCapacityUsesPagedLogicalSourceAndBoundedNativePool(t * t.Fatalf("native worker reservation contains managed blocking edge %q:\n%s", forbidden, reserve) } } - if !strings.Contains(reserve, "coroworker.QueueReserve(&reservation)") { + if !strings.Contains(reserve, "reservation := coroworker.QueueReserve()") { t.Fatalf("native worker reservation bypasses the C11 capacity preflight:\n%s", reserve) } for _, forbidden := range []string{"psync", "state.mutex", "state.work", "pthread.Mutex", "pthread.Cond"} { @@ -289,7 +291,7 @@ func TestRuntimeCoroWorkerCapacityUsesPagedLogicalSourceAndBoundedNativePool(t * for _, required := range []string{ "func coroReserveNativeWorkerSubmissionV1(", "coroNativeWorkerSubmissionOwnerV1(handle, route)", - "coro.CommitCurrentExecutorWorkerSubmission(driver, g, id)", + "func coroPublishNativeWorkerSubmissionV1(", "id.Route() != route", } { if !strings.Contains(native, required) { @@ -300,8 +302,8 @@ func TestRuntimeCoroWorkerCapacityUsesPagedLogicalSourceAndBoundedNativePool(t * owner := readRuntimePollFile(t, runtimeCoroWorkerOwnerSource) for _, required := range []string{ "coro.CurrentExecutorWorkerDriver(task)", - "coro.PrepareCurrentExecutorWorkerPark(", - "coro.BindSingleWaitSetResumePacket(", + "coro.PrepareCurrentExecutorWorkerParkCompiler(", + "&state.packet,", "coro.TakeResumePacket(", "coroReserveNativeWorkerSubmissionV1(executor, route)", "packet coro.ResumePacket", @@ -315,6 +317,8 @@ func TestRuntimeCoroWorkerCapacityUsesPagedLogicalSourceAndBoundedNativePool(t * "coroProgramReserveNativeWorkerSubmissionV1", "coroProgramCancelNativeWorkerSubmissionV1", "coroProgramCommitNativeWorkerSubmissionV1", + "coro.BindSingleWaitSetResumePacket(", + "coro.CommitCurrentExecutorWorkerSubmission(", "coro.TakeRunDecision(", "coro.FinishCurrentExecutorWorkerPark(", } { @@ -337,9 +341,9 @@ func TestRuntimeCoroWorkerCapacityUsesPagedLogicalSourceAndBoundedNativePool(t * for _, required := range []string{ "//go:linkname QueueInit C.__llgo_coro_worker_queue_init_v1", "//go:linkname QueueCanRelease C.__llgo_coro_worker_queue_can_release_v1", - "//go:linkname QueueReserve C.__llgo_coro_worker_queue_reserve_v1", - "//go:linkname QueueCancelReservation C.__llgo_coro_worker_queue_cancel_reservation_v1", - "//go:linkname QueueSubmitReserved C.__llgo_coro_worker_queue_submit_reserved_v1", + "//go:linkname QueueReserve C.__llgo_coro_worker_queue_reserve_v2", + "//go:linkname QueueCancelReservation C.__llgo_coro_worker_queue_cancel_reservation_v2", + "//go:linkname QueueSubmitReserved C.__llgo_coro_worker_queue_submit_reserved_v4", "//go:linkname QueueStop C.__llgo_coro_worker_queue_stop_v1", "lock-free by QueueInit", "semaphore_signal never wait for worker", @@ -362,8 +366,13 @@ func TestRuntimeCoroWorkerCapacityUsesPagedLogicalSourceAndBoundedNativePool(t * "_Atomic size_t sequence;", "_Atomic uint32_t producer_state;", "_Atomic size_t enqueue_position;", + "_Atomic bool handoff_poller;", "atomic_is_lock_free(&queue->enqueue_position)", "atomic_store_explicit(&slot->sequence, reservation + 1, memory_order_release);", + "LLGO_CORO_WORKER_HANDOFF_POLLS_V1", + "handoff_slot->sequence", + "llgo_coro_worker_cpu_relax_v1();", + "&queue->handoff_poller, false", "llgo_coro_worker_job_canceled_v1", "sem_post(wake)", "semaphore_signal(*wake)", @@ -372,6 +381,8 @@ func TestRuntimeCoroWorkerCapacityUsesPagedLogicalSourceAndBoundedNativePool(t * "__llgo_coro_native_worker_complete_v1(", "siglongjmp(*state->landing, 1)", "state->fault_pc = llgo_coro_worker_fault_pc_v1(context)", + "sigsetjmp(landing, 0)", + "llgo_coro_worker_unblock_fault_signals_v1(fault_signal)", } { if !strings.Contains(cSource, required) { t.Errorf("%s lacks C11 worker transport marker %q", runtimeCoroWorkerCSource, required) @@ -382,6 +393,18 @@ func TestRuntimeCoroWorkerCapacityUsesPagedLogicalSourceAndBoundedNativePool(t * t.Errorf("%s retains pthread queue synchronization %q", runtimeCoroWorkerCSource, forbidden) } } + for _, forbidden := range []string{ + "QueueReserve(&", + "args *[coroworker.MaxArgs]uintptr", + "args := [coroworker.MaxArgs]uintptr", + "__llgo_coro_worker_queue_submit_reserved_v2", + "__llgo_coro_worker_queue_submit_reserved_v3", + } { + if strings.Contains(native, forbidden) || strings.Contains(owner, forbidden) || + strings.Contains(declaration, forbidden) || strings.Contains(cSource, forbidden) { + t.Errorf("worker submission retains escaping Go aggregate boundary %q", forbidden) + } + } header := readRuntimePollFile(t, runtimeCoroWorkerHeaderSource) for _, required := range []string{ "LLGO_CORO_WORKER_THREAD_COUNT_V1 = 4", diff --git a/runtime/sema_coro_source_test.go b/runtime/sema_coro_source_test.go index 2d0387fcc1..ace2349a56 100644 --- a/runtime/sema_coro_source_test.go +++ b/runtime/sema_coro_source_test.go @@ -244,13 +244,16 @@ func TestCoroSemaphoreOwnerV2FailStopABIAndKeyedSource(t *testing.T) { func TestCoroKeyedResumeIsPNeutralAndRegistryIsPreemptibleLockFree(t *testing.T) { const ( parkPath = "internal/runtime/coro_keyed_park.go" + manualPath = "internal/coro/manual_park_owner.go" materializePath = "internal/runtime/coro_resume_materialize.go" ) park := readRuntimePollFile(t, parkPath) + manual := readRuntimePollFile(t, manualPath) materialize := readRuntimePollFile(t, materializePath) for _, marker := range []string{ - "coro.BindWaitSetResumeCleanup(", - "Kind: coro.ResumeCleanupKeyedPark", + "coro.PrepareCurrentExecutorManualCleanupParkReserved(", + "coro.BeginOwnerLocalManualCompletionCurrent(", + "coro.FinishOwnerLocalManualCompletionCurrent(", "coro.TakeResumePacket(", "validMaterializedCoroKeyedParkV2(state)", } { @@ -258,6 +261,21 @@ func TestCoroKeyedResumeIsPNeutralAndRegistryIsPreemptibleLockFree(t *testing.T) t.Errorf("%s lacks keyed P-neutral marker %q", parkPath, marker) } } + for _, marker := range []string{ + "func PrepareCurrentExecutorManualCleanupParkReserved(", + "installWaitSetResumeCleanup(", + "Kind: ResumeCleanupKeyedPark", + "Context: context", + "Entries: unsafe.Pointer(entry)", + "RuntimeCount: 1", + } { + if !strings.Contains(manual, marker) { + t.Errorf("%s lacks fused keyed cleanup marker %q", manualPath, marker) + } + } + if strings.Contains(park, "coro.BindWaitSetResumeCleanup(") { + t.Errorf("%s repeats the fused keyed cleanup binding", parkPath) + } resumeStart := strings.Index(park, "func __llgo_coro_keyed_resume_v2(") if resumeStart < 0 { t.Fatalf("%s lacks keyed resume", parkPath) diff --git a/runtime/syscall_worker_source_test.go b/runtime/syscall_worker_source_test.go index 2d04610b3a..68c7d06cdb 100644 --- a/runtime/syscall_worker_source_test.go +++ b/runtime/syscall_worker_source_test.go @@ -697,6 +697,10 @@ static uintptr_t success_with_errno(uintptr_t ignored) { return 7; } +static uintptr_t fault_memory(uintptr_t address) { + return *(volatile uintptr_t *)address; +} + static int call_and_check( uintptr_t function, uintptr_t want_r1, @@ -752,6 +756,31 @@ int main(void) { if (status != 0) { return status; } + for (int attempt = 0; attempt < 2; ++attempt) { + const uintptr_t args[9] = {0}; + struct llgo_coro_worker_result_v1 result = {0}; + uintptr_t function = (uintptr_t)(void *)&fault_memory; + if (!__llgo_coro_worker_call_v1(function, function, 1, args, &result)) { + return 50 + attempt * 10; + } + if (result.fault != 1 || result.fault_target != function) { + return 51 + attempt * 10; + } +#if (defined(__APPLE__) || defined(__linux__)) && \ + (defined(__aarch64__) || defined(__x86_64__)) + if (result.fault_pc == 0) { + return 52 + attempt * 10; + } +#endif + } + status = call_and_check( + (uintptr_t)(void *)&success_with_errno, + 7, + (uintptr_t)EBUSY, + 80); + if (status != 0) { + return status; + } return 0; } ` diff --git a/ssa/abitype.go b/ssa/abitype.go index 8bbb4a7ff2..e56333f9f6 100644 --- a/ssa/abitype.go +++ b/ssa/abitype.go @@ -19,6 +19,7 @@ package ssa import ( "crypto/sha256" "encoding/binary" + "encoding/hex" "go/ast" "go/token" "go/types" @@ -62,6 +63,22 @@ var ( types.NewTuple(types.NewVar(token.NoPos, nil, "", types.Typ[types.Uintptr])), false) ) +type abiMethodEntryKey struct { + concrete string + name string + methodTyp string +} + +func abiTypeHash(name string) uint32 { + h := sha256.Sum256([]byte(name)) + return binary.LittleEndian.Uint32(h[:4]) +} + +func staticItabSymbol(packagePath, interfaceName, concreteName string) string { + digest := sha256.Sum256([]byte(packagePath + "\x00" + interfaceName + "\x00" + concreteName)) + return "__llgo_static_itab." + hex.EncodeToString(digest[:16]) +} + // ABITypeRuntimeFunctions returns the logical runtime functions whose // addresses abiType embeds while materializing the descriptor for t. These are // references, not calls: consumers must demand the selected entries without @@ -103,8 +120,7 @@ func (b Builder) abiCommonFields(t types.Type, name string, hasUncommon bool, gl // PtrBytes uintptr fields = append(fields, prog.IntVal(uint64(ab.PtrBytes(t)), prog.Uintptr()).impl) // Hash uint32 - h := sha256.Sum256([]byte(name)) - hash := binary.LittleEndian.Uint32(h[:4]) + hash := abiTypeHash(name) fields = append(fields, prog.IntVal(uint64(hash), prog.Uint32()).impl) // TFlag uint8 tflag := ab.TFlag(t) @@ -565,9 +581,18 @@ func (b Builder) abiUncommonMethods(t types.Type, methods []*types.Selection) ll ifn = descriptor.impl } } + ftyp := funcType(prog, m.Type()) + methodTypeName, _ := prog.abi.TypeName(ftyp) + if b.Pkg.abiMethodIfns == nil { + b.Pkg.abiMethodIfns = make(map[abiMethodEntryKey]llvm.Value) + } + b.Pkg.abiMethodIfns[abiMethodEntryKey{ + concrete: typeName, + name: fullName, + methodTyp: methodTypeName, + }] = ifn var values []llvm.Value values = append(values, name) - ftyp := funcType(prog, m.Type()) values = append(values, b.abiType(ftyp).impl) values = append(values, ifn) values = append(values, tfn) diff --git a/ssa/closure_abi.go b/ssa/closure_abi.go index a13fd88edc..b6d35c8eef 100644 --- a/ssa/closure_abi.go +++ b/ssa/closure_abi.go @@ -75,10 +75,9 @@ func aarch64UsesSwiftSelf(triple string) bool { } // closureContextABIForTarget is retained for the coroutine lowering API. The -// LLVM version is intentionally ignored: changing LLVM must not change the -// ABI of already compiled libraries. goos only disambiguates legacy generic -// AArch64 triples which did not encode their platform. -func closureContextABIForTarget(triple, goos string, _ int) closureContextABI { +// goos only disambiguates legacy generic AArch64 triples which did not encode +// their platform. The ABI does not vary within LLGo's LLVM 22 baseline. +func closureContextABIForTarget(triple, goos string) closureContextABI { lower := strings.ToLower(triple) arch, _, _ := strings.Cut(lower, "-") if (arch == "arm64" || arch == "arm64_32" || arch == "aarch64" || arch == "aarch64_be") && @@ -92,7 +91,7 @@ func closureContextABIForTarget(triple, goos string, _ int) closureContextABI { } func aarch64PlatformReservesX18(triple, goos string) bool { - return closureContextABIForTarget(triple, goos, 0) == closureEnvSwiftSelf + return closureContextABIForTarget(triple, goos) == closureEnvSwiftSelf } func (p *Target) closureEnvABI() closureEnvABI { diff --git a/ssa/closure_abi_test.go b/ssa/closure_abi_test.go index 7df13b24f9..24549011c5 100644 --- a/ssa/closure_abi_test.go +++ b/ssa/closure_abi_test.go @@ -24,33 +24,30 @@ func closureContextIRAttr(prog Program) string { func TestClosureContextABIForTarget(t *testing.T) { tests := []struct { - name string - triple string - goos string - llvmMajor int - want closureContextABI + name string + triple string + goos string + want closureContextABI }{ - {"amd64", "x86_64-unknown-linux-gnu", "linux", 19, closureContextNest}, - {"386", "i386-unknown-linux-gnu", "linux", 22, closureContextNest}, - {"arm", "thumbv7em-none-eabi", "linux", 19, closureContextSwiftSelf}, - {"riscv32", "riscv32-unknown-none", "linux", 20, closureContextNest}, - {"riscv64", "riscv64-unknown-linux-gnu", "linux", 22, closureContextNest}, - {"arm64 linux llvm19", "aarch64-unknown-linux-gnu", "linux", 19, closureContextNest}, - {"arm64 darwin llvm19", "arm64-apple-darwin", "darwin", 19, closureContextSwiftSelf}, - {"arm64 windows llvm20", "aarch64-pc-windows-msvc", "windows", 20, closureContextSwiftSelf}, - {"arm64 android llvm20", "aarch64-linux-android", "android", 20, closureContextSwiftSelf}, - {"arm64 darwin llvm21", "arm64-apple-darwin", "darwin", 21, closureContextSwiftSelf}, - {"arm64 windows llvm22", "aarch64-pc-windows-msvc", "windows", 22, closureContextSwiftSelf}, - {"wasm", "wasm32-unknown-wasip1", "wasip1", 22, closureContextExplicit}, - {"xtensa", "xtensa-esp32-none-elf", "linux", 22, closureContextExplicit}, - {"avr", "avr-unknown-unknown", "linux", 22, closureContextExplicit}, + {"amd64", "x86_64-unknown-linux-gnu", "linux", closureContextNest}, + {"386", "i386-unknown-linux-gnu", "linux", closureContextNest}, + {"arm", "thumbv7em-none-eabi", "linux", closureContextSwiftSelf}, + {"riscv32", "riscv32-unknown-none", "linux", closureContextNest}, + {"riscv64", "riscv64-unknown-linux-gnu", "linux", closureContextNest}, + {"arm64 linux", "aarch64-unknown-linux-gnu", "linux", closureContextNest}, + {"arm64 darwin", "arm64-apple-darwin", "darwin", closureContextSwiftSelf}, + {"arm64 windows", "aarch64-pc-windows-msvc", "windows", closureContextSwiftSelf}, + {"arm64 android", "aarch64-linux-android", "android", closureContextSwiftSelf}, + {"wasm", "wasm32-unknown-wasip1", "wasip1", closureContextExplicit}, + {"xtensa", "xtensa-esp32-none-elf", "linux", closureContextExplicit}, + {"avr", "avr-unknown-unknown", "linux", closureContextExplicit}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - if got := closureContextABIForTarget(test.triple, test.goos, test.llvmMajor); got != test.want { + if got := closureContextABIForTarget(test.triple, test.goos); got != test.want { t.Fatalf( - "closureContextABIForTarget(%q, %q, %d) = %d, want %d", - test.triple, test.goos, test.llvmMajor, got, test.want, + "closureContextABIForTarget(%q, %q) = %d, want %d", + test.triple, test.goos, got, test.want, ) } }) diff --git a/ssa/coro.go b/ssa/coro.go index 84e778d8dc..e8d9341e6b 100644 --- a/ssa/coro.go +++ b/ssa/coro.go @@ -20,8 +20,6 @@ import ( "encoding/binary" "fmt" "go/types" - "strconv" - "strings" "github.com/xgo-dev/llvm" ) @@ -207,7 +205,8 @@ type CoroProgramStep struct { } // CoroProgramBootstrapOptions describes the entry module's immutable startup -// table. Version must be one or two. Flags is reserved and must be zero. +// table. Version must be one or two. Version one requires zero flags; version +// two accepts the CoroProgramBootstrapFlag* capability bits below. // ABIHash covers the ordered steps and their referenced catalog. Factory may // be Nil in the data-only phase; a non-Nil factory must use the root factory // ABI and belong to this module. @@ -219,6 +218,11 @@ type CoroProgramBootstrapOptions struct { Factory Expr } +// CoroProgramBootstrapFlagWorkerV2 says that the final physical program owns +// at least one reachable bounded-worker transaction. It is program demand, +// not a declaration that the selected target supports workers. +const CoroProgramBootstrapFlagWorkerV2 uint32 = 1 << 0 + // NewCoroFrameDescriptor defines a link-once constant descriptor with layout: // // { version i32, flags i32, hashLo i64, hashHi i64, @@ -631,14 +635,17 @@ func (p Package) NewCoroProgramBootstrap( if p.coroProgramBootstrap != "" { panic(fmt.Sprintf("ssa: coroutine program bootstrap already defined as %q", p.coroProgramBootstrap)) } - if opts.Flags != 0 { - panic("ssa: coroutine program bootstrap flags must be zero") - } var roles []uint32 switch opts.Version { case 1: + if opts.Flags != 0 { + panic("ssa: coroutine program bootstrap version 1 flags must be zero") + } roles = []uint32{CoroProgramStepInit, CoroProgramStepMain} case 2: + if unknown := opts.Flags &^ CoroProgramBootstrapFlagWorkerV2; unknown != 0 { + panic(fmt.Sprintf("ssa: coroutine program bootstrap version 2 has unknown capability flags %#x", unknown)) + } if len(opts.Steps) < coroProgramStepMinimumV2 { panic(fmt.Sprintf( "ssa: coroutine program bootstrap version %d requires at least %d steps, got %d", @@ -797,7 +804,7 @@ func (p Package) NewCoroProgramBootstrap( bootstrap := p.NewVarEx(name, prog.Pointer(bootstrapType)) bootstrap.impl.SetInitializer(prog.ctx.ConstStruct([]llvm.Value{ prog.IntVal(uint64(opts.Version), prog.Uint32()).impl, - prog.IntVal(0, prog.Uint32()).impl, + prog.IntVal(uint64(opts.Flags), prog.Uint32()).impl, prog.IntVal(binary.BigEndian.Uint64(opts.ABIHash[:8]), prog.Uint64()).impl, prog.IntVal(binary.BigEndian.Uint64(opts.ABIHash[8:]), prog.Uint64()).impl, prog.IntVal(uint64(len(stepValues)), prog.Uintptr()).impl, @@ -1215,6 +1222,22 @@ func (c *CoroBuilder) emitSuspendWithCallbacks( switchValue := b.impl.CreateSwitch(result, c.suspendBlk.first, 2) switchValue.AddCase(llvm.ConstInt(prog.tyInt8(), 0, false), resumeBlk.first) switchValue.AddCase(llvm.ConstInt(prog.tyInt8(), 1, false), c.cleanupBlk.first) + if !final { + // A coroutine's normal continuation is entered by a later resume call, + // not by the ramp invocation which first executes this switch. Generic + // block-frequency analysis otherwise assigns the resume arm roughly one + // third of the entry frequency and LLVM 22's CoroAnnotationElide rejects + // every source-style static await as cold (its default threshold is 55%). + // Describe the language-level common path explicitly. The suspend return + // and destroy arms stay possible and retain their exact control flow. + ctx := prog.ctx + switchValue.SetMetadata(ctx.MDKindID("prof"), ctx.MDNode([]llvm.Metadata{ + ctx.MDString("branch_weights"), + llvm.ConstInt(prog.tyInt32(), 1, false).ConstantAsMetadata(), + llvm.ConstInt(prog.tyInt32(), 1000, false).ConstantAsMetadata(), + llvm.ConstInt(prog.tyInt32(), 1, false).ConstantAsMetadata(), + })) + } b.SetBlock(resumeBlk) if !final && dispatch != nil { callbackPoint := captureCoroFrameCallbackPoint(b) @@ -1314,6 +1337,28 @@ func (b Builder) CoroDestroy(handle Expr) { ) } +// MarkCoroElideSafe marks one exact direct coroutine ramp call as having a +// caller-bounded lifetime. LLVM 22 uses this proof to synthesize and select a +// no-allocation ramp while both caller and callee are still presplit. +// +// The caller must prove that every use of the returned handle is contained by +// its own coroutine frame lifetime. This is deliberately not inferred from an +// arbitrary function value or exposed as a source annotation. +func (b Builder) MarkCoroElideSafe(call Expr) bool { + if b == nil || b.Func == nil || b.blk == nil { + panic("ssa: cannot mark coroutine elision without an active function block") + } + if call.IsNil() || call.impl.IsACallInst().IsNil() { + panic("ssa: coroutine elision requires an exact call result") + } + kind := llvm.AttributeKindID("coro_elide_safe") + if kind == 0 { + panic(fmt.Sprintf("ssa: LLVM %s has no coro_elide_safe attribute", llvm.Version)) + } + call.impl.AddCallSiteAttribute(-1, b.Prog.ctx.CreateEnumAttribute(kind, 0)) + return true +} + func (b Builder) requireCoroHandle(operation string, handle Expr) { if b == nil || b.Func == nil || b.blk == nil { panic("ssa: cannot " + operation + " coroutine without an active function block") @@ -1435,15 +1480,7 @@ func coroBlockInstructions(block llvm.BasicBlock) []llvm.Value { } func markPresplitCoroutine(fn Function) { - major := llvmMajorVersion() ctx := fn.Pkg.mod.Context() - if major == 14 { - // LLVM 14's string attribute encodes a legacy state machine. Frontends - // must emit the unprepared "0" state before CoroEarly; "1" is reserved - // for a coroutine already prepared for a direct CoroSplit invocation. - fn.impl.AddFunctionAttr(ctx.CreateStringAttribute("coroutine.presplit", "0")) - return - } kind := llvm.AttributeKindID("presplitcoroutine") if kind == 0 { panic(fmt.Sprintf("ssa: LLVM %s has no presplitcoroutine attribute", llvm.Version)) @@ -1466,18 +1503,12 @@ func (b Builder) coroFrameLayout(allocationAlign uint32) (size, align Expr) { } func (b Builder) coroEnd(handle Expr) { - major := llvmMajorVersion() - args := []llvm.Value{handle.impl, b.Prog.BoolVal(false).impl} - if major >= 18 { - args = append(args, b.Prog.ctx.ConstTokenNone()) - } - ret := b.Prog.Bool().ll - name := "coro.end" - if major >= 22 { - ret = b.Prog.Void().ll - name = "" - } - b.coroIntrinsic("llvm.coro.end", ret, args, name) + args := []llvm.Value{ + handle.impl, + b.Prog.BoolVal(false).impl, + b.Prog.ctx.ConstTokenNone(), + } + b.coroIntrinsic("llvm.coro.end", b.Prog.Void().ll, args, "") } func (b Builder) coroIntrinsic(name string, ret llvm.Type, args []llvm.Value, resultName string) llvm.Value { @@ -1491,12 +1522,3 @@ func (b Builder) coroIntrinsic(name string, ret llvm.Type, args []llvm.Value, re } return value } - -func llvmMajorVersion() int { - text, _, _ := strings.Cut(llvm.Version, ".") - major, err := strconv.Atoi(text) - if err != nil { - panic(fmt.Sprintf("ssa: parse LLVM version %q: %v", llvm.Version, err)) - } - return major -} diff --git a/ssa/coro_atomic_cost.go b/ssa/coro_atomic_cost.go index ce5bde6a98..8b10b0a7c2 100644 --- a/ssa/coro_atomic_cost.go +++ b/ssa/coro_atomic_cost.go @@ -178,6 +178,20 @@ type coroAtomicLLVMFunction struct { // optimization/compiler-owned site insertion, so backend-created structural // work cannot silently escape the certificate boundary. func VerifyCoroAtomicCostModule(module llvm.Module) (CoroAtomicCostReport, error) { + return verifyCoroAtomicCostModule(module, false) +} + +// VerifyOptimizedCoroAtomicCostModule verifies a module which has already +// passed VerifyCoroAtomicCostModule and then an ordinary LLVM optimization +// pipeline. Named metadata deliberately does not retain code; full inlining may +// therefore erase a local or imported outcome symbol while leaving its string +// proof row behind. A missing symbol has no remaining call edge and is ignored. +// Present symbols and every reachable certified call remain strictly checked. +func VerifyOptimizedCoroAtomicCostModule(module llvm.Module) (CoroAtomicCostReport, error) { + return verifyCoroAtomicCostModule(module, true) +} + +func verifyCoroAtomicCostModule(module llvm.Module, allowMissingAfterOptimization bool) (CoroAtomicCostReport, error) { report := CoroAtomicCostReport{Schema: coroAtomicCostReportSchema} if module.IsNil() { return report, fmt.Errorf("ssa: verify atomic-cost metadata in a nil module") @@ -194,6 +208,9 @@ func VerifyCoroAtomicCostModule(module llvm.Module) (CoroAtomicCostReport, error for symbol, record := range records { function := module.NamedFunction(symbol) if function.IsNil() { + if allowMissingAfterOptimization { + continue + } return report, fmt.Errorf("ssa: atomic-cost symbol %q has no LLVM declaration", symbol) } if !record.local { @@ -413,7 +430,10 @@ func projectCoroAtomicLLVMFunction( if bounded { continue } - return projection, fmt.Errorf("ssa: atomic-cost LLVM function %q contains an indirect or inline-assembly call", function.Name()) + return projection, fmt.Errorf( + "ssa: atomic-cost LLVM function %q contains an indirect or inline-assembly call: %s", + function.Name(), instruction.String(), + ) } if _, certified := records[callee]; !certified { return projection, fmt.Errorf("ssa: atomic-cost LLVM function %q calls uncertified helper %q", function.Name(), callee) diff --git a/ssa/coro_atomic_cost_test.go b/ssa/coro_atomic_cost_test.go index c31ff2ebdf..b5f6f353eb 100644 --- a/ssa/coro_atomic_cost_test.go +++ b/ssa/coro_atomic_cost_test.go @@ -111,6 +111,38 @@ func TestVerifyCoroAtomicCostModuleRejectsTruncatedProofMetadata(t *testing.T) { } } +func TestVerifyOptimizedCoroAtomicCostModuleAllowsInlinedAwayDependency(t *testing.T) { + ctx, module := newCoroAtomicCostTestModule(t) + functionType := llvm.FunctionType(ctx.VoidType(), nil, false) + function := llvm.AddFunction(module, "pkg.atomic", functionType) + builder := ctx.NewBuilder() + defer builder.Dispose() + entry := ctx.AddBasicBlock(function, "entry") + builder.SetInsertPointAtEnd(entry) + builder.CreateRetVoid() + addCoroAtomicCostTestMetadata(ctx, module, function.Name()) + module.AddNamedMetadataOperand(CoroAtomicCostMetadataName, ctx.MDNode([]llvm.Metadata{ + llvm.ConstInt(ctx.Int32Type(), coroAtomicCostMetadataV1, false).ConstantAsMetadata(), + ctx.MDString("pkg.inlined-away"), + llvm.ConstInt(ctx.Int64Type(), 2, false).ConstantAsMetadata(), + llvm.ConstInt(ctx.Int32Type(), 1, false).ConstantAsMetadata(), + ctx.MDString(strings.Repeat("b", 64)), + llvm.ConstInt(ctx.Int32Type(), 0, false).ConstantAsMetadata(), + })) + + if _, err := VerifyCoroAtomicCostModule(module); err == nil || + !strings.Contains(err.Error(), "has no LLVM declaration") { + t.Fatalf("strict verifier accepted missing dependency: %v", err) + } + report, err := VerifyOptimizedCoroAtomicCostModule(module) + if err != nil { + t.Fatal(err) + } + if len(report.Functions) != 1 || report.Functions[0].Symbol != function.Name() { + t.Fatalf("stale imported dependency report = %+v", report) + } +} + func TestVerifyCoroAtomicCostModuleRequiresInjectedInlineAsmCapability(t *testing.T) { for _, test := range []struct { name string @@ -155,6 +187,24 @@ func TestVerifyCoroAtomicCostModuleRequiresInjectedInlineAsmCapability(t *testin } } +func TestCoroAtomicDataAnchorInjectsBoundedCapabilityAtCreation(t *testing.T) { + prog := NewProgram(nil) + defer prog.Dispose() + pkg := prog.NewPackage("example.com/atomic", "atomic") + function := pkg.NewFunc("example.com/atomic.body", NoArgsNoRet, InGo) + builder := function.MakeBody(1) + builder.CoroAtomicDataAnchor(".pushsection .llgo_test_anchor\n.byte 0\n.popsection") + builder.Return() + if err := pkg.EmitCoroAtomicCostCertificate( + function.Name(), 1, 1, strings.Repeat("a", 64), + ); err != nil { + t.Fatal(err) + } + if _, err := VerifyCoroAtomicCostModule(pkg.Module()); err != nil { + t.Fatalf("verify compiler-created atomic data anchor: %v\n%s", err, pkg.String()) + } +} + func TestVerifyCoroAtomicCostModuleAcceptsConstantMemoryIntrinsic(t *testing.T) { ctx, module := newCoroAtomicCostTestModule(t) pointerType := llvm.PointerType(ctx.Int8Type(), 0) diff --git a/ssa/coro_dynamic_dispatch_test.go b/ssa/coro_dynamic_dispatch_test.go index c029e67591..75a79a7fd0 100644 --- a/ssa/coro_dynamic_dispatch_test.go +++ b/ssa/coro_dynamic_dispatch_test.go @@ -37,10 +37,7 @@ type coroDynamicDispatchTestFixture struct { dual Expr } -func TestCoroDynamicDispatchV1LLVM19CapturedCoroAndDualEntries(t *testing.T) { - if llvmMajorVersion() != 19 { - t.Skipf("dynamic coroutine descriptor IR proof is focused on LLVM 19, using %s", llvm.Version) - } +func TestCoroDynamicDispatchV1LLVM22CapturedCoroAndDualEntries(t *testing.T) { fixture := newCoroDynamicDispatchTestFixture(t) prog, pkg := fixture.prog, fixture.pkg diff --git a/ssa/coro_keepalive_test.go b/ssa/coro_keepalive_test.go index 82362bb43c..e821dbf828 100644 --- a/ssa/coro_keepalive_test.go +++ b/ssa/coro_keepalive_test.go @@ -32,10 +32,6 @@ import ( // no resumed use other than KeepAlive. The fake use must remain on the resumed // side of the cut and make CoroSplit place the local in the coroutine frame. func TestCoroKeepAliveRetainsLocalAddress(t *testing.T) { - if major := llvmMajorVersion(); major < 19 || major > 22 { - t.Skipf("coroutine keepalive regression requires supported LLVM 19-22, using %s", llvm.Version) - } - Initialize(InitAll) prog := NewProgram(nil) defer prog.Dispose() diff --git a/ssa/coro_test.go b/ssa/coro_test.go index 65d4351a0a..53788251d3 100644 --- a/ssa/coro_test.go +++ b/ssa/coro_test.go @@ -44,11 +44,7 @@ func TestCoroBuilderPresplitShape(t *testing.T) { } ir := mod.String() - if major := llvmMajorVersion(); major == 14 { - if !strings.Contains(ir, `"coroutine.presplit"="0"`) { - t.Fatalf("LLVM 14 coroutine lacks unprepared frontend presplit state:\n%s", ir) - } - } else if !strings.Contains(ir, "presplitcoroutine") { + if !strings.Contains(ir, "presplitcoroutine") { t.Fatalf("coroutine lacks enum presplit attribute:\n%s", ir) } if !strings.Contains(ir, "@llvm.coro.id(i32 32") { @@ -280,13 +276,7 @@ func TestCoroBuilderPerSuspendAfterResumeOverride(t *testing.T) { func TestCoroBuilderCoroSplit(t *testing.T) { fixture := newCoroTestFixture(t, nil, 32) mod := fixture.pkg.Module() - pipeline := "coro-early,cgscc(coro-split),coro-cleanup" - if llvmMajorVersion() == 14 { - // LLVM 14 implicitly treats a pipeline beginning with coro-early as a - // function pipeline, so every pass-manager level must be explicit. - pipeline = "function(coro-early),cgscc(coro-split),function(coro-cleanup)" - } - runCoroPasses(t, fixture, pipeline) + runCoroPasses(t, fixture, "coro-early,cgscc(coro-split),coro-cleanup") post := mod.String() for _, suffix := range []string{".resume", ".destroy"} { @@ -364,11 +354,7 @@ func TestCoroHandleIntrinsicsBeforeAndAfterCoroSplit(t *testing.T) { t.Fatalf("llvm.coro.promise does not use payload ABI alignment %d and from=false:\n%s", wantAlign, pre) } - pipeline := "coro-early,cgscc(coro-split),coro-cleanup" - if llvmMajorVersion() == 14 { - pipeline = "function(coro-early),cgscc(coro-split),function(coro-cleanup)" - } - runCoroPasses(t, fixture, pipeline) + runCoroPasses(t, fixture, "coro-early,cgscc(coro-split),coro-cleanup") post := mod.String() for _, intrinsic := range []string{ "llvm.coro.promise", "llvm.coro.done", "llvm.coro.resume", "llvm.coro.destroy", @@ -384,10 +370,106 @@ func TestCoroHandleIntrinsicsBeforeAndAfterCoroSplit(t *testing.T) { } } -func TestCoroBuilderDefaultPipelineLLVM19(t *testing.T) { - if llvmMajorVersion() != 19 { - t.Skipf("production default smoke is specific to LLVM 19, using %s", llvm.Version) +func TestCoroElideStaticChildFrameContract(t *testing.T) { + Initialize(InitAll) + prog := NewProgram(nil) + pkg := prog.NewPackage("coroelide", "coro/elide") + t.Cleanup(func() { + pkg.Module().Dispose() + prog.Dispose() + }) + + childAlloc := pkg.NewFunc("coro_elide_child_alloc_probe", functionSignature( + []types.Type{types.Typ[types.Uintptr], types.Typ[types.Uintptr]}, + []types.Type{types.Typ[types.UnsafePointer]}, + ), InC) + childFree := pkg.NewFunc("coro_elide_child_free_probe", functionSignature( + []types.Type{types.Typ[types.UnsafePointer], types.Typ[types.Uintptr], types.Typ[types.Uintptr]}, + nil, + ), InC) + childPublish := pkg.NewFunc("coro_elide_child_publish_probe", functionSignature( + []types.Type{types.Typ[types.UnsafePointer], types.Typ[types.UnsafePointer]}, + nil, + ), InC) + bodyProbe := pkg.NewFunc("coro_elide_body_probe", functionSignature(nil, nil), InC) + doneProbe := pkg.NewFunc("coro_elide_done_probe", functionSignature( + []types.Type{types.Typ[types.Bool]}, nil, + ), InC) + + child := pkg.NewFunc("coro_elide_child", coroHandleSignature(), InGo) + childBuilder := child.MakeBody(1) + childCoro := childBuilder.BeginCoro(CoroOptions{ + AllocationAlign: 32, + Frame: CoroFrameOps{ + Alloc: func(b Builder, size, align Expr) Expr { + return b.Call(childAlloc.Expr, size, align) + }, + Free: func(b Builder, frame, size, align Expr) { + b.Call(childFree.Expr, frame, size, align) + }, + }, + BeforeInitialSuspend: func(b Builder, handle, storage Expr) { + b.Call(childPublish.Expr, handle, storage) + }, + }) + childBuilder.Call(bodyProbe.Expr) + childCoro.Finish() + childBuilder.EndBuild() + childBuilder.Dispose() + + parentAlloc := pkg.NewFunc("coro_elide_parent_alloc_probe", functionSignature( + []types.Type{types.Typ[types.Uintptr], types.Typ[types.Uintptr]}, + []types.Type{types.Typ[types.UnsafePointer]}, + ), InC) + parentFree := pkg.NewFunc("coro_elide_parent_free_probe", functionSignature( + []types.Type{types.Typ[types.UnsafePointer], types.Typ[types.Uintptr], types.Typ[types.Uintptr]}, + nil, + ), InC) + parent := pkg.NewFunc("coro_elide_parent", coroHandleSignature(), InGo) + parentBuilder := parent.MakeBody(1) + parentCoro := parentBuilder.BeginCoro(CoroOptions{ + AllocationAlign: 32, + Frame: CoroFrameOps{ + Alloc: func(b Builder, size, align Expr) Expr { + return b.Call(parentAlloc.Expr, size, align) + }, + Free: func(b Builder, frame, size, align Expr) { + b.Call(parentFree.Expr, frame, size, align) + }, + }, + }) + handle := parentBuilder.Call(child.Expr) + if !parentBuilder.MarkCoroElideSafe(handle) { + t.Fatal("LLVM 22 rejected an exact coroutine elision proof") + } + parentBuilder.CoroResume(handle) + done := parentBuilder.CoroDone(handle) + parentBuilder.CoroDestroy(handle) + parentBuilder.Call(doneProbe.Expr, done) + parentCoro.Finish() + parentBuilder.EndBuild() + parentBuilder.Dispose() + + fixture := &coroTestFixture{prog: prog, pkg: pkg, fn: parent, coro: parentCoro} + runCoroPasses(t, fixture, "lto") + post := pkg.Module().String() + parentResume := pkg.Module().NamedFunction("coro_elide_parent.resume").String() + if !strings.Contains(post, "@coro_elide_child.noalloc") { + t.Fatalf("CoroSplit did not synthesize the attributed no-allocation ramp:\n%s", post) + } + if strings.Contains(parentResume, "call ptr @coro_elide_child()") || + strings.Contains(parentResume, "@coro_elide_child_alloc_probe") || + strings.Contains(parentResume, "@coro_elide_child_free_probe") { + t.Fatalf("static child retained its dynamic allocation path in the parent resume:\n%s", parentResume) + } + if !regexp.MustCompile( + `call void @coro_elide_child_publish_probe\(ptr [^,]+, ptr null\)`, + ).MatchString(parentResume) { + t.Fatalf("elided child publication did not expose a null dynamic-storage operand:\n%s", parentResume) } +} + +func TestCoroBuilderDefaultPipelineLLVM22(t *testing.T) { fixture := newCoroTestFixture(t, nil, 0) runCoroPasses(t, fixture, "default") post := fixture.pkg.String() @@ -1660,6 +1742,7 @@ func TestCoroProgramBootstrapV2MixedStartupTable(t *testing.T) { } bootstrap := pkg.NewCoroProgramBootstrap("__llgo_coro_program_bootstrap_v2", CoroProgramBootstrapOptions{ Version: 2, + Flags: CoroProgramBootstrapFlagWorkerV2, ABIHash: [16]byte{ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, @@ -1672,6 +1755,9 @@ func TestCoroProgramBootstrapV2MixedStartupTable(t *testing.T) { if got := initializer.Operand(0).ZExtValue(); got != 2 { t.Fatalf("bootstrap version = %d, want 2", got) } + if got := initializer.Operand(1).ZExtValue(); got != uint64(CoroProgramBootstrapFlagWorkerV2) { + t.Fatalf("bootstrap flags = %#x, want worker capability", got) + } if got := initializer.Operand(4).ZExtValue(); got != uint64(len(steps)) { t.Fatalf("bootstrap step count = %d, want %d", got, len(steps)) } @@ -1740,6 +1826,11 @@ func TestCoroProgramBootstrapV2RejectsShapeAndRoles(t *testing.T) { Kind: CoroProgramStepDirectPlain, Flags: roles[index], Target: plains[index].Expr, } } + badFlags := valid + badFlags.Flags = CoroProgramBootstrapFlagWorkerV2 << 1 + mustPanicContains(t, "unknown capability flags", func() { + pkg.NewCoroProgramBootstrap("v2_bad_capability_flags", badFlags) + }) for _, count := range []int{0, 1, 2, 3, 4} { bad := valid bad.Steps = append([]CoroProgramStep(nil), valid.Steps...) diff --git a/ssa/datastruct.go b/ssa/datastruct.go index 0f63bc0b87..4cb4e8a7fd 100644 --- a/ssa/datastruct.go +++ b/ssa/datastruct.go @@ -1051,27 +1051,28 @@ func (b Builder) Recv(ch Expr, commaOk bool) (ret Expr) { // CoroChanTrySend performs only the nonblocking, non-panicking first attempt // of a compiler-owned stackless channel send. The caller owns elem storage and // must enter the exact channel park transaction when false is returned. -func (b Builder) CoroChanTrySend(ch, elem Expr) Expr { +func (b Builder) CoroChanTrySend(task, ch, elem Expr) Expr { prog := b.Prog eltSize := prog.IntVal(prog.SizeOf(prog.Elem(ch.Type)), prog.Int()) - return b.InlineCall(b.Pkg.rtFunc("CoroChanTrySend"), ch, elem, eltSize) + return b.InlineCall(b.Pkg.rtFunc("CoroChanTrySend"), task, ch, elem, eltSize) } // CoroChanTryRecv performs only the nonblocking first attempt of a // compiler-owned stackless channel receive. It returns (recvOK, tryOK); the // caller must enter the exact channel park transaction when tryOK is false. -func (b Builder) CoroChanTryRecv(ch, elem Expr) Expr { +func (b Builder) CoroChanTryRecv(task, ch, elem Expr) Expr { prog := b.Prog eltSize := prog.IntVal(prog.SizeOf(prog.Elem(ch.Type)), prog.Int()) - return b.InlineCall(b.Pkg.rtFunc("CoroChanTryRecv"), ch, elem, eltSize) + return b.InlineCall(b.Pkg.rtFunc("CoroChanTryRecv"), task, ch, elem, eltSize) } -// CoroChanTryClose performs one complete non-panicking channel-close -// transaction. Its scalar result distinguishes success, nil channel, and an -// already closed channel; physical coroutine lowering owns the two language -// panic outcomes through its explicit-status ABI. -func (b Builder) CoroChanTryClose(ch Expr) Expr { - return b.InlineCall(b.Pkg.rtFunc("CoroChanTryClose"), ch) +// CoroChanTryCloseTask performs one complete non-panicking channel-close +// transaction using the compiler-carried logical task capability. Its scalar +// result distinguishes success, nil channel, and an already closed channel; +// physical coroutine lowering owns the two language panic outcomes through +// its explicit-status ABI. +func (b Builder) CoroChanTryCloseTask(task, ch Expr) Expr { + return b.InlineCall(b.Pkg.rtFunc("CoroChanTryCloseTask"), task, ch) } type SelectState struct { diff --git a/ssa/expr.go b/ssa/expr.go index e30ca72b45..cfe228d45f 100644 --- a/ssa/expr.go +++ b/ssa/expr.go @@ -289,11 +289,27 @@ func (b Builder) CBytes(v Expr) Expr { // InlineAsm generates inline assembly instruction func (b Builder) InlineAsm(instruction string) { + b.inlineAsmCall(instruction) +} + +func (b Builder) inlineAsmCall(instruction string) llvm.Value { dbgInstrf("InlineAsm %s\n", instruction) typ := llvm.FunctionType(b.Prog.tyVoid(), nil, false) asm := llvm.InlineAsm(typ, instruction, "", true, false, llvm.InlineAsmDialectATT, false) - b.impl.CreateCall(typ, asm, nil, "") + return b.impl.CreateCall(typ, asm, nil, "") +} + +// CoroAtomicDataAnchor emits compiler-owned, zero-runtime-work inline +// assembly and attaches the capability consumed by the atomic outcome +// verifier. The capability is injected at construction time and binds the +// complete inline-assembly content; arbitrary source assembly cannot acquire +// it by spelling or later reverse classification. +func (b Builder) CoroAtomicDataAnchor(instruction string) { + call := b.inlineAsmCall(instruction) + MarkCoroAtomicBoundedCompilerCall( + b.Pkg.mod.Context(), call, CoroAtomicCompilerDataAnchorV1, + ) } func (b Builder) InlineAsmFull(instruction, constraints string, retType Type, exprs []Expr) Expr { @@ -324,10 +340,9 @@ func (b Builder) KeepAlive(values ...Expr) { } args[index] = value.impl } - // LLVM 19's C API does not expose llvm.fake.use through + // The LLVM C API does not expose llvm.fake.use through // LLVMLookupIntrinsicID even though the intrinsic is part of the IR and - // verifier. Declare its canonical variadic form directly so the same code - // works on LLVM 19--22. + // verifier. Declare its canonical variadic form directly. fnType := llvm.FunctionType(b.Prog.tyVoid(), nil, true) fn := b.Pkg.mod.NamedFunction("llvm.fake.use") if fn.IsNil() { @@ -339,7 +354,7 @@ func (b Builder) KeepAlive(values ...Expr) { } // RemoveKeepAliveCallsAfterCoroSplit erases the optimizer-only liveness uses -// after LLVM has materialized coroutine frames. LLVM 19 accepts llvm.fake.use +// after LLVM has materialized coroutine frames. LLVM 22 accepts llvm.fake.use // in IR and CoroSplit honors it, but its target pipeline can otherwise emit an // unresolved external call at O0. The declaration may remain: with every call // removed it contributes no object-file reference. diff --git a/ssa/globals.go b/ssa/globals.go index 494ce2146d..e9a92f00f9 100644 --- a/ssa/globals.go +++ b/ssa/globals.go @@ -59,6 +59,24 @@ func (pkg Package) ConstBytes(value []byte) Expr { return Expr{cv, styp} } +// constantAddress materializes immutable package-local storage for an LLVM +// constant. Interface values whose physical representation is indirect may +// point at this storage instead of allocating and copying an identical value +// on every conversion. The address must never escape as writable Go storage; +// callers use it only as the read-only data word of an interface value. +func (pkg Package) constantAddress(value Expr) Expr { + if value.IsNil() || value.impl.IsAConstant().IsNil() { + panic("ssa: constantAddress requires an LLVM constant") + } + global := llvm.AddGlobal(pkg.mod, value.impl.Type(), "") + global.SetInitializer(value.impl) + global.SetLinkage(llvm.PrivateLinkage) + global.SetGlobalConstant(true) + global.SetUnnamedAddr(true) + global.SetAlignment(pkg.Prog.td.ABITypeAlignment(value.impl.Type())) + return Expr{global, pkg.Prog.Pointer(value.Type)} +} + // ConstArray creates an LLVM constant array expression. func (prog Program) ConstArray(t Type, values []Expr) Expr { elem := prog.Index(t) diff --git a/ssa/interface.go b/ssa/interface.go index 2ec7c90893..f3e3ab051b 100644 --- a/ssa/interface.go +++ b/ssa/interface.go @@ -50,6 +50,103 @@ func (b Builder) unsafeInterface(rawIntf *types.Interface, t Expr, data llvm.Val return b.unsafeIface(itab.impl, data) } +type staticItabKey struct { + interfaceType string + concreteType string +} + +// CanBuildStaticItab reports whether a concrete-to-interface conversion has a +// statically known concrete type. A well-typed MakeInterface instruction is +// already the assignability proof; do not repeat types.Implements here because +// frontend physical type patches may use equivalent package-local type copies +// with deliberately different go/types object identity. staticItab validates +// every ABI method key before publishing the constant. Interface-to-interface +// conversions retain the runtime path because their concrete type is dynamic. +func CanBuildStaticItab(target, concrete types.Type) bool { + rawIntf, ok := types.Unalias(target).Underlying().(*types.Interface) + if !ok || rawIntf.Empty() || concrete == nil { + return false + } + if _, dynamic := types.Unalias(concrete).Underlying().(*types.Interface); dynamic { + return false + } + return true +} + +func (b Builder) staticItab(rawIntf *types.Interface, concrete Type, tintf, typ Expr) Expr { + prog := b.Prog + if !CanBuildStaticItab(rawIntf, concrete.raw.Type) { + panic("ssa: staticItab requires a statically known concrete type") + } + interfaceName, _ := prog.abi.TypeName(rawIntf) + concreteName, _ := prog.abi.TypeName(concrete.raw.Type) + key := staticItabKey{interfaceType: interfaceName, concreteType: concreteName} + if itab, ok := b.Pkg.staticItabs[key]; ok { + return itab + } + + methods := make([]llvm.Value, rawIntf.NumMethods()) + for index := range methods { + method := rawIntf.Method(index) + methodType := funcType(prog, method.Type()) + methodTypeName, _ := prog.abi.TypeName(methodType) + ifn, ok := b.Pkg.abiMethodIfns[abiMethodEntryKey{ + concrete: concreteName, + name: abiMethodName(method), + methodTyp: methodTypeName, + }] + if !ok || ifn.IsNil() || ifn.IsAConstant().IsNil() { + panic("ssa: concrete interface method is missing from ABI metadata") + } + methods[index] = ifn + } + + runtimeItab := prog.rtType("Itab") + interField := prog.Field(runtimeItab, 0) + typeField := prog.Field(runtimeItab, 1) + hashField := prog.Field(runtimeItab, 2) + funField := prog.Field(runtimeItab, 3) + textField := prog.Index(funField) + funArray := llvm.ArrayType(textField.ll, len(methods)) + layout := prog.ctx.StructType([]llvm.Type{ + interField.ll, + typeField.ll, + hashField.ll, + funArray, + }, false) + initializer := prog.ctx.ConstStruct([]llvm.Value{ + tintf.impl, + typ.impl, + prog.IntVal(uint64(abiTypeHash(concreteName)), hashField).impl, + llvm.ConstArray(textField.ll, methods), + }, false) + global := llvm.AddGlobal(b.Pkg.mod, layout, staticItabSymbol(b.Pkg.Path(), interfaceName, concreteName)) + global.SetInitializer(initializer) + global.SetLinkage(llvm.PrivateLinkage) + global.SetGlobalConstant(true) + global.SetUnnamedAddr(true) + global.SetAlignment(prog.td.ABITypeAlignment(runtimeItab.ll)) + ret := Expr{global, prog.Pointer(runtimeItab)} + if b.Pkg.staticItabs == nil { + b.Pkg.staticItabs = make(map[staticItabKey]Expr) + } + b.Pkg.staticItabs[key] = ret + return ret +} + +func (b Builder) unsafeConcreteInterface(rawIntf *types.Interface, concrete Type, typ Expr, data llvm.Value) llvm.Value { + if rawIntf.Empty() { + return b.unsafeEface(typ.impl, data) + } + tintf := b.abiType(rawIntf) + if CanBuildStaticItab(rawIntf, concrete.raw.Type) { + itab := b.staticItab(rawIntf, concrete, tintf, typ) + return b.unsafeIface(itab.impl, data) + } + itab := b.newItab(tintf, typ) + return b.unsafeIface(itab.impl, data) +} + func iMethodOf(rawIntf *types.Interface, name string) int { n := rawIntf.NumMethods() for i := 0; i < n; i++ { @@ -140,6 +237,22 @@ func (b Builder) imethod(intf Expr, method *types.Func, rawDataKnownNonNil bool) // t1 = make interface{} <- int (42:int) // t2 = make Stringer <- t0 func (b Builder) MakeInterface(tinter Type, x Expr) (ret Expr) { + return b.makeInterface(tinter, x, false) +} + +// MakeInterfaceFromConstant constructs an interface from a compile-time +// constant. Indirect interface payloads use immutable package-local backing +// storage instead of a fresh heap allocation. This is safe because an +// interface exposes a copy of the concrete value, not writable access to its +// backing storage. +func (b Builder) MakeInterfaceFromConstant(tinter Type, x Expr) (ret Expr) { + if x.IsNil() || x.impl.IsAConstant().IsNil() { + panic("ssa: MakeInterfaceFromConstant requires an LLVM constant") + } + return b.makeInterface(tinter, x, true) +} + +func (b Builder) makeInterface(tinter Type, x Expr, constantBacking bool) (ret Expr) { rawIntf := tinter.raw.Type.Underlying().(*types.Interface) dbgInstrf("MakeInterface %v, %v\n", rawIntf, x.impl) if x.kind == vkFuncDecl { @@ -151,16 +264,24 @@ func (b Builder) MakeInterface(tinter Type, x Expr) (ret Expr) { b.recordUseIface(typ) tabi := b.abiType(typ.raw.Type) if !directIfaceType(typ.raw.Type) { + if constantBacking { + vptr := b.Pkg.constantAddress(x) + return Expr{b.unsafeConcreteInterface(rawIntf, typ, tabi, vptr.impl), tinter} + } vptr := b.AllocU(typ) b.Store(vptr, x) - return Expr{b.unsafeInterface(rawIntf, tabi, vptr.impl), tinter} + return Expr{b.unsafeConcreteInterface(rawIntf, typ, tabi, vptr.impl), tinter} } kind, _, lvl := abi.DataKindOf(typ.raw.Type, 0, prog.is32Bits) switch kind { case abi.Indirect: + if constantBacking { + vptr := b.Pkg.constantAddress(x) + return Expr{b.unsafeConcreteInterface(rawIntf, typ, tabi, vptr.impl), tinter} + } vptr := b.AllocU(typ) b.Store(vptr, x) - return Expr{b.unsafeInterface(rawIntf, tabi, vptr.impl), tinter} + return Expr{b.unsafeConcreteInterface(rawIntf, typ, tabi, vptr.impl), tinter} } ximpl := x.impl if lvl > 0 { @@ -169,7 +290,7 @@ func (b Builder) MakeInterface(tinter Type, x Expr) (ret Expr) { var u llvm.Value switch kind { case abi.Pointer: - return Expr{b.unsafeInterface(rawIntf, tabi, ximpl), tinter} + return Expr{b.unsafeConcreteInterface(rawIntf, typ, tabi, ximpl), tinter} case abi.Integer: tu := prog.Uintptr() u = llvm.CreateIntCast(b.impl, ximpl, tu.ll) @@ -184,7 +305,7 @@ func (b Builder) MakeInterface(tinter Type, x Expr) (ret Expr) { panic("todo") } data := llvm.CreateIntToPtr(b.impl, u, prog.tyVoidPtr()) - return Expr{b.unsafeInterface(rawIntf, tabi, data), tinter} + return Expr{b.unsafeConcreteInterface(rawIntf, typ, tabi, data), tinter} } func (b Builder) MakeInterfaceFromPtr(tinter Type, ptr Expr) (ret Expr) { @@ -219,7 +340,7 @@ func (b Builder) makeInterfaceFromPtr(tinter Type, ptr Expr, knownNonNil bool) ( dst := b.Convert(prog.VoidPtr(), vptr) src := b.Convert(prog.VoidPtr(), ptr) b.Call(b.Pkg.rtFunc("Typedmemmove"), tabi, dst, src) - return Expr{b.unsafeInterface(rawIntf, tabi, vptr.impl), tinter} + return Expr{b.unsafeConcreteInterface(rawIntf, typ, tabi, vptr.impl), tinter} } func (b Builder) recordUseIface(typ Type) { diff --git a/ssa/memory.go b/ssa/memory.go index 5fddfa9861..b9f3544875 100644 --- a/ssa/memory.go +++ b/ssa/memory.go @@ -423,7 +423,16 @@ func (b Builder) AssertNilDeref(ptr Expr) { } nilPtr := llvm.ConstNull(ptr.impl.Type()) isNil := Expr{llvm.CreateICmp(b.impl, llvm.IntEQ, ptr.impl, nilPtr), b.Prog.Bool()} - b.InlineCall(b.Pkg.rtFunc("AssertNilDeref"), isNil) + // Keep the recoverable Go panic helper entirely on the nil edge. Calling + // AssertNilDeref(false) used to put one cross-package call after every + // ordinary pointer check; separate-object builds cannot reliably inline it, + // and even full LTO retained many calls whose false argument was already + // implied by surrounding control flow. Passing literal true on the cold edge + // also lets LLVM fold a proven-non-nil predecessor without inspecting the + // runtime package body. + b.IfThen(isNil, func() { + b.InlineCall(b.Pkg.rtFunc("AssertNilDeref"), b.Prog.BoolVal(true)) + }) } func (b Builder) NilDerefCheck(ptr Expr) Expr { diff --git a/ssa/memory_test.go b/ssa/memory_test.go index 23668fda8e..04338f19d5 100644 --- a/ssa/memory_test.go +++ b/ssa/memory_test.go @@ -13,6 +13,31 @@ func TestAssertNilDerefZeroExprNoPanic(t *testing.T) { b.AssertNilDeref(Expr{}) } +func TestAssertNilDerefCallsRuntimeOnlyOnNilEdge(t *testing.T) { + prog := NewProgram(nil) + prog.sizes = types.SizesFor("gc", runtime.GOARCH) + prog.SetRuntime(func() *types.Package { + pkg, err := importer.For("source", nil).Import(PkgRuntime) + if err != nil { + t.Fatal(err) + } + return pkg + }) + pkg := prog.NewPackage("memory-guard", "test/memory-guard") + param := types.NewVar(0, nil, "p", types.NewPointer(types.Typ[types.Int])) + sig := types.NewSignatureType(nil, nil, nil, types.NewTuple(param), nil, false) + fn := pkg.NewFunc("guard", sig, InGo) + b := fn.MakeBody(1) + b.AssertNilDeref(fn.Param(0)) + b.Return() + b.EndBuild() + body := fn.impl.String() + if !strings.Contains(body, "icmp eq ptr") || !strings.Contains(body, "br i1") || + !strings.Contains(body, "AssertNilDeref\"(i1 true)") { + t.Fatalf("nil guard is not a cold-edge runtime call:\n%s", body) + } +} + func TestLoadKnownNonNilZeroSizedSkipsNilDerefGuard(t *testing.T) { prog := NewProgram(nil) prog.sizes = types.SizesFor("gc", runtime.GOARCH) diff --git a/ssa/package.go b/ssa/package.go index 8471d6fc9e..1be795aedb 100644 --- a/ssa/package.go +++ b/ssa/package.go @@ -974,6 +974,8 @@ type aPackage struct { Meta *meta.PackageMeta metaBuilder *meta.Builder abiTypeWithUncommon map[llvm.Value]struct{} + abiMethodIfns map[abiMethodEntryKey]llvm.Value + staticItabs map[staticItabKey]Expr export map[string]string // pkgPath.nameInPkg => exportname preserveSyms map[string]struct{} // set of exported symbol names diff --git a/ssa/ssa_test.go b/ssa/ssa_test.go index 4dd51582f8..88cb62f037 100644 --- a/ssa/ssa_test.go +++ b/ssa/ssa_test.go @@ -1787,11 +1787,85 @@ func TestMakeInterfaceKinds(t *testing.T) { rawMeth := types.NewFunc(0, pkgTypes, "M", rawSig) nonEmpty := types.NewInterfaceType([]*types.Func{rawMeth}, nil) nonEmpty.Complete() + concrete := types.NewNamed( + types.NewTypeName(token.NoPos, pkgTypes, "methodInt", nil), + types.Typ[types.Int], nil, + ) + methodSig := types.NewSignatureType( + types.NewVar(token.NoPos, pkgTypes, "", concrete), nil, nil, nil, nil, false, + ) + concrete.AddMethod(types.NewFunc(token.NoPos, pkgTypes, "M", methodSig)) nonEmptyType := prog.Type(nonEmpty, InGo) sigNE := types.NewSignatureType(nil, nil, nil, nil, types.NewTuple(types.NewVar(0, nil, "", nonEmpty)), false) fnNE := pkg.NewFunc("nonEmptyIface", sigNE, InGo) bNE := fnNE.MakeBody(1) - bNE.Return(bNE.MakeInterface(nonEmptyType, prog.Val(7))) + bNE.Return(bNE.MakeInterface(nonEmptyType, prog.IntVal(7, prog.Type(concrete, InGo)))) +} + +func TestMakeInterfaceFromConstantUsesImmutableBacking(t *testing.T) { + prog := NewProgram(nil) + prog.sizes = types.SizesFor("gc", runtime.GOARCH) + prog.SetRuntime(func() *types.Package { + pkg, err := importer.For("source", nil).Import(PkgRuntime) + if err != nil { + t.Fatal(err) + } + return pkg + }) + pkg := prog.NewPackage("bar", "foo/bar") + rawPkg := types.NewPackage("foo/bar", "bar") + namedInt := types.NewNamed( + types.NewTypeName(token.NoPos, rawPkg, "Errno", nil), + types.Typ[types.Int], nil, + ) + stringResult := types.NewTuple(types.NewVar(token.NoPos, nil, "", types.Typ[types.String])) + concreteMethodSig := types.NewSignatureType( + types.NewVar(token.NoPos, rawPkg, "", namedInt), nil, nil, nil, stringResult, false, + ) + namedInt.AddMethod(types.NewFunc(token.NoPos, rawPkg, "Error", concreteMethodSig)) + interfaceMethodSig := types.NewSignatureType(nil, nil, nil, nil, stringResult, false) + errorIface := types.NewInterfaceType( + []*types.Func{types.NewFunc(token.NoPos, nil, "Error", interfaceMethodSig)}, nil, + ) + errorIface.Complete() + errorType := prog.Type(errorIface, InGo) + + valueType := prog.Type(namedInt, InGo) + result := types.NewTuple(types.NewVar(token.NoPos, nil, "", errorIface)) + + constantFn := pkg.NewFunc("constant", types.NewSignatureType(nil, nil, nil, nil, result, false), InGo) + constantBody := constantFn.MakeBody(1) + constantBody.Return(constantBody.MakeInterfaceFromConstant(errorType, prog.IntVal(4, valueType))) + + dynamicParams := types.NewTuple(types.NewVar(token.NoPos, nil, "value", namedInt)) + dynamicFn := pkg.NewFunc("dynamic", types.NewSignatureType(nil, nil, nil, dynamicParams, result, false), InGo) + dynamicBody := dynamicFn.MakeBody(1) + dynamicBody.Return(dynamicBody.MakeInterface(errorType, dynamicFn.Param(0))) + + constantIR := constantFn.impl.String() + if strings.Contains(constantIR, ".AllocU\"") { + t.Fatalf("constant interface conversion allocates:\n%s", constantIR) + } + if strings.Contains(constantIR, ".NewItab\"") { + t.Fatalf("constant concrete-to-interface conversion builds an itab at runtime:\n%s", constantIR) + } + if !strings.Contains(constantIR, "ret ") || !strings.Contains(constantIR, ", ptr @") { + t.Fatalf("constant interface conversion does not use static backing:\n%s", constantIR) + } + dynamicIR := dynamicFn.impl.String() + if !strings.Contains(dynamicIR, ".AllocU\"") { + t.Fatalf("dynamic interface conversion lost its required copy:\n%s", dynamicIR) + } + if strings.Contains(dynamicIR, ".NewItab\"") { + t.Fatalf("dynamic concrete-to-interface conversion builds an itab at runtime:\n%s", dynamicIR) + } + moduleIR := pkg.String() + if !strings.Contains(moduleIR, "private unnamed_addr constant i64 4") { + t.Fatalf("missing immutable constant interface backing:\n%s", moduleIR) + } + if !strings.Contains(moduleIR, "private unnamed_addr constant { ptr, ptr, i32, [1 x ptr] }") { + t.Fatalf("missing immutable static itab:\n%s", moduleIR) + } } func TestCheckExprAssignmentConversions(t *testing.T) { @@ -2414,6 +2488,27 @@ attributes #0 = { null_pointer_is_valid "frame-pointer"="non-leaf" } `) } +func TestIfWithBranchWeights(t *testing.T) { + prog := NewProgram(nil) + pkg := prog.NewPackage("bar", "foo/bar") + params := types.NewTuple(types.NewVar(0, nil, "a", types.Typ[types.Int])) + rets := types.NewTuple(types.NewVar(0, nil, "", types.Typ[types.Int])) + sig := types.NewSignatureType(nil, nil, nil, params, rets, false) + fn := pkg.NewFunc("weighted", sig, InGo) + b := fn.MakeBody(3) + iftrue := fn.Block(1) + iffalse := fn.Block(2) + cond := b.BinOp(token.GTR, fn.Param(0), prog.Val(0)) + b.IfWithBranchWeights(cond, iftrue, iffalse, 1000, 1) + b.SetBlock(iftrue).Return(prog.Val(1)) + b.SetBlock(iffalse).Return(prog.Val(0)) + ir := pkg.String() + if !strings.Contains(ir, `br i1 %1, label %_llgo_1, label %_llgo_2, !prof !0`) || + !strings.Contains(ir, `!0 = !{!"branch_weights", i32 1000, i32 1}`) { + t.Fatalf("weighted branch metadata is absent or malformed:\n%s", ir) + } +} + func TestPrintf(t *testing.T) { prog := NewProgram(nil) pkg := prog.NewPackage("bar", "foo/bar") @@ -2730,9 +2825,9 @@ func TestTargetMachineAndDataLayout(t *testing.T) { triple string }{ {"linux", "amd64", "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", "x86_64-unknown-linux"}, - {"linux", "arm64", "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32", "aarch64-unknown-linux"}, + {"linux", "arm64", "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32", "aarch64-unknown-linux"}, {"darwin", "amd64", "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", "x86_64-apple-macosx"}, - {"darwin", "arm64", "e-m:o-i64:64-i128:128-n32:64-S128-Fn32", "arm64-apple-macosx"}, + {"darwin", "arm64", "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32", "arm64-apple-macosx"}, } for _, tt := range tests { prog := NewProgram(&Target{GOOS: tt.goos, GOARCH: tt.goarch}) @@ -2810,11 +2905,19 @@ func TestAbiTables(t *testing.T) { rawMeth := types.NewFunc(0, pkgTypes, "M", rawSig) nonEmpty := types.NewInterfaceType([]*types.Func{rawMeth}, nil) nonEmpty.Complete() + concrete := types.NewNamed( + types.NewTypeName(token.NoPos, pkgTypes, "abiMethodInt", nil), + types.Typ[types.Int], nil, + ) + methodSig := types.NewSignatureType( + types.NewVar(token.NoPos, pkgTypes, "", concrete), nil, nil, nil, nil, false, + ) + concrete.AddMethod(types.NewFunc(token.NoPos, pkgTypes, "M", methodSig)) nonEmptyType := prog.Type(nonEmpty, InGo) sigNE := types.NewSignatureType(nil, nil, nil, nil, types.NewTuple(types.NewVar(0, nil, "", nonEmpty)), false) fnNE := pkg.NewFunc("nonEmptyIface", sigNE, InGo) bNE := fnNE.MakeBody(1) - bNE.Return(bNE.MakeInterface(nonEmptyType, prog.Val(7))) + bNE.Return(bNE.MakeInterface(nonEmptyType, prog.IntVal(7, prog.Type(concrete, InGo)))) fn := pkg.InitAbiTypes(pkg.Path() + ".init$abitables") s := fn.impl.String() diff --git a/ssa/stmt_builder.go b/ssa/stmt_builder.go index a0c73419e5..ee9c0457c2 100644 --- a/ssa/stmt_builder.go +++ b/ssa/stmt_builder.go @@ -254,6 +254,31 @@ func (b Builder) If(cond Expr, thenb, elseb BasicBlock) { b.impl.CreateCondBr(cond.impl, thenb.first, elseb.first) } +// IfWithBranchWeights emits an if instruction with relative, nonzero branch +// frequencies. The weights are optimization evidence only: both successors +// retain their ordinary control-flow and language semantics. +func (b Builder) IfWithBranchWeights( + cond Expr, thenb, elseb BasicBlock, thenWeight, elseWeight uint32, +) { + if b.Func != thenb.fn || b.Func != elseb.fn { + panic("mismatched function") + } + if thenWeight == 0 || elseWeight == 0 { + panic("branch weights must be nonzero") + } + dbgInstrf( + "IfWithBranchWeights %v, _llgo_%v, _llgo_%v, %d, %d\n", + cond.impl, thenb.idx, elseb.idx, thenWeight, elseWeight, + ) + branch := b.impl.CreateCondBr(cond.impl, thenb.first, elseb.first) + ctx := b.Prog.ctx + branch.SetMetadata(ctx.MDKindID("prof"), ctx.MDNode([]llvm.Metadata{ + ctx.MDString("branch_weights"), + llvm.ConstInt(b.Prog.tyInt32(), uint64(thenWeight), false).ConstantAsMetadata(), + llvm.ConstInt(b.Prog.tyInt32(), uint64(elseWeight), false).ConstantAsMetadata(), + })) +} + // IfThen emits an if-then instruction. func (b Builder) IfThen(cond Expr, then func()) { blks := b.Func.MakeBlocks(2) diff --git a/ssa/target_resolved_test.go b/ssa/target_resolved_test.go index 06d947810f..1e00c9d685 100644 --- a/ssa/target_resolved_test.go +++ b/ssa/target_resolved_test.go @@ -67,7 +67,7 @@ func TestResolvedTargetConfig(t *testing.T) { Features: "+bulk-memory,+mutable-globals,+nontrapping-fptoint,+sign-ext", }, wantPtrSize: 4, - wantLayout: "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128-ni:1:10:20", + wantLayout: "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-i128:128-n32:64-S128-ni:1:10:20", }, { name: "thumb", diff --git a/xtool/env/llvm/llvm.go b/xtool/env/llvm/llvm.go index d488fb9ec3..6bd046d13b 100644 --- a/xtool/env/llvm/llvm.go +++ b/xtool/env/llvm/llvm.go @@ -23,6 +23,7 @@ import ( "path/filepath" "runtime" "sort" + "strconv" "strings" "github.com/goplus/llgo/internal/env" @@ -37,6 +38,10 @@ import ( const ( // CrosscompileClangPath is the relative path from LLGO_ROOT to the clang installation CrosscompileClangPath = "crosscompile/clang" + + // SupportedMajorVersion is the sole LLVM release accepted by LLGo. Keep + // this gate aligned with the github.com/xgo-dev/llvm binding baseline. + SupportedMajorVersion = 22 ) // ----------------------------------------------------------------------------- @@ -88,25 +93,83 @@ func New(llvmConfigBin string) *Env { // means LLVM executables are assumed to be in PATH. func (e *Env) BinDir() string { return e.binDir } -// SetupPath makes the selected LLVM installation part of the process -// environment. Command entry points call it before starting builds; build -// requests and workers then inherit LLVM through the ordinary PATH snapshot. -func SetupPath() { - binDir := New("").BinDir() +// SetupPath validates the selected LLVM installation and makes it part of the +// process environment. Command entry points call it before starting builds; +// build requests and workers then inherit one LLVM 22 toolchain through the +// ordinary PATH snapshot. +func SetupPath() error { + llvmConfigBin := defaultLLVMConfigBin() + version, err := commandOutput(llvmConfigBin, "--version") + if err != nil { + return fmt.Errorf("run LLVM configuration %q: %w", llvmConfigBin, err) + } + major, err := parseMajorVersion(version) + if err != nil { + return fmt.Errorf("read LLVM version from %q: %w", llvmConfigBin, err) + } + if major != SupportedMajorVersion { + return fmt.Errorf( + "LLVM %s selected by %q is unsupported; LLGo requires LLVM %d", + version, llvmConfigBin, SupportedMajorVersion, + ) + } + binDir, err := commandOutput(llvmConfigBin, "--bindir") + if err != nil { + return fmt.Errorf("read LLVM binary directory from %q: %w", llvmConfigBin, err) + } if binDir == "" { - return + return fmt.Errorf("LLVM configuration %q returned an empty binary directory", llvmConfigBin) + } + if info, statErr := os.Stat(binDir); statErr != nil || !info.IsDir() { + if statErr != nil { + return fmt.Errorf("LLVM binary directory %q: %w", binDir, statErr) + } + return fmt.Errorf("LLVM binary directory %q is not a directory", binDir) } path := os.Getenv("PATH") for _, dir := range filepath.SplitList(path) { if samePath(dir, binDir) { - return + return nil } } if path != "" { binDir += string(os.PathListSeparator) + path } - _ = os.Setenv("PATH", binDir) + if err := os.Setenv("PATH", binDir); err != nil { + return fmt.Errorf("install LLVM %d binary directory in PATH: %w", SupportedMajorVersion, err) + } + return nil +} + +// SetupPathOrExit is the command-entry counterpart of [SetupPath]. A version +// mismatch is a configuration error, not a compatibility mode. +func SetupPathOrExit() { + if err := SetupPath(); err != nil { + fmt.Fprintln(os.Stderr, "llgo:", err) + os.Exit(2) + } +} + +func commandOutput(name string, args ...string) (string, error) { + output, err := exec.Command(name, args...).CombinedOutput() + text := strings.TrimSpace(string(output)) + if err != nil { + if text != "" { + return "", fmt.Errorf("%w: %s", err, text) + } + return "", err + } + return text, nil +} + +func parseMajorVersion(version string) (int, error) { + text, _, _ := strings.Cut(strings.TrimSpace(version), ".") + major, err := strconv.Atoi(text) + if err != nil || major <= 0 { + return 0, fmt.Errorf("invalid version %q", version) + } + return major, nil } func samePath(x, y string) bool { diff --git a/xtool/env/llvm/llvm_config_darwin_amd64_llvm15.go b/xtool/env/llvm/llvm_config_darwin_amd64_llvm15.go deleted file mode 100644 index 256ac0078a..0000000000 --- a/xtool/env/llvm/llvm_config_darwin_amd64_llvm15.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build !byollvm && darwin && amd64 && llvm15 - -/* - * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package llvm - -const ldLLVMConfigBin = "/usr/local/opt/llvm@15/bin/llvm-config" diff --git a/xtool/env/llvm/llvm_config_darwin_amd64_llvm16.go b/xtool/env/llvm/llvm_config_darwin_amd64_llvm16.go deleted file mode 100644 index b996ac05d8..0000000000 --- a/xtool/env/llvm/llvm_config_darwin_amd64_llvm16.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build !byollvm && darwin && amd64 && llvm16 - -/* - * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package llvm - -const ldLLVMConfigBin = "/usr/local/opt/llvm@16/bin/llvm-config" diff --git a/xtool/env/llvm/llvm_config_darwin_amd64_llvm17.go b/xtool/env/llvm/llvm_config_darwin_amd64_llvm17.go deleted file mode 100644 index b67ccbaa9d..0000000000 --- a/xtool/env/llvm/llvm_config_darwin_amd64_llvm17.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build !byollvm && darwin && amd64 && llvm17 - -/* - * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package llvm - -const ldLLVMConfigBin = "/usr/local/opt/llvm@17/bin/llvm-config" diff --git a/xtool/env/llvm/llvm_config_darwin_amd64_llvm18.go b/xtool/env/llvm/llvm_config_darwin_amd64_llvm18.go deleted file mode 100644 index 0e0668949d..0000000000 --- a/xtool/env/llvm/llvm_config_darwin_amd64_llvm18.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build !byollvm && darwin && amd64 && llvm18 - -/* - * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package llvm - -const ldLLVMConfigBin = "/usr/local/opt/llvm@18/bin/llvm-config" diff --git a/xtool/env/llvm/llvm_config_darwin_amd64_llvm19.go b/xtool/env/llvm/llvm_config_darwin_amd64_llvm19.go deleted file mode 100644 index c8959f7cdc..0000000000 --- a/xtool/env/llvm/llvm_config_darwin_amd64_llvm19.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build !byollvm && darwin && amd64 && !llvm14 && !llvm15 && !llvm16 && !llvm17 && !llvm18 && !llvm22 - -/* - * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package llvm - -const ldLLVMConfigBin = "/usr/local/opt/llvm@19/bin/llvm-config" diff --git a/xtool/env/llvm/llvm_config_darwin_amd64_llvm22.go b/xtool/env/llvm/llvm_config_darwin_amd64_llvm22.go index 077749b18e..fc5b1ec095 100644 --- a/xtool/env/llvm/llvm_config_darwin_amd64_llvm22.go +++ b/xtool/env/llvm/llvm_config_darwin_amd64_llvm22.go @@ -1,4 +1,4 @@ -//go:build !byollvm && darwin && amd64 && llvm22 +//go:build !byollvm && darwin && amd64 /* * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. diff --git a/xtool/env/llvm/llvm_config_darwin_llvm14.go b/xtool/env/llvm/llvm_config_darwin_llvm14.go deleted file mode 100644 index 20080b2555..0000000000 --- a/xtool/env/llvm/llvm_config_darwin_llvm14.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build !byollvm && darwin && !amd64 && llvm14 - -/* - * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package llvm - -const ldLLVMConfigBin = "/opt/homebrew/opt/llvm@14/bin/llvm-config" diff --git a/xtool/env/llvm/llvm_config_darwin_llvm15.go b/xtool/env/llvm/llvm_config_darwin_llvm15.go deleted file mode 100644 index 2164c1e9d0..0000000000 --- a/xtool/env/llvm/llvm_config_darwin_llvm15.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build !byollvm && darwin && !amd64 && llvm15 - -/* - * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package llvm - -const ldLLVMConfigBin = "/opt/homebrew/opt/llvm@15/bin/llvm-config" diff --git a/xtool/env/llvm/llvm_config_darwin_llvm16.go b/xtool/env/llvm/llvm_config_darwin_llvm16.go deleted file mode 100644 index 9abbcd7ad2..0000000000 --- a/xtool/env/llvm/llvm_config_darwin_llvm16.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build !byollvm && darwin && !amd64 && llvm16 - -/* - * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package llvm - -const ldLLVMConfigBin = "/opt/homebrew/opt/llvm@16/bin/llvm-config" diff --git a/xtool/env/llvm/llvm_config_darwin_llvm17.go b/xtool/env/llvm/llvm_config_darwin_llvm17.go deleted file mode 100644 index 83482032eb..0000000000 --- a/xtool/env/llvm/llvm_config_darwin_llvm17.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build !byollvm && darwin && !amd64 && llvm17 - -/* - * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package llvm - -const ldLLVMConfigBin = "/opt/homebrew/opt/llvm@17/bin/llvm-config" diff --git a/xtool/env/llvm/llvm_config_darwin_llvm18.go b/xtool/env/llvm/llvm_config_darwin_llvm18.go deleted file mode 100644 index cd3e1ad38b..0000000000 --- a/xtool/env/llvm/llvm_config_darwin_llvm18.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build !byollvm && darwin && !amd64 && llvm18 - -/* - * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package llvm - -const ldLLVMConfigBin = "/opt/homebrew/opt/llvm@18/bin/llvm-config" diff --git a/xtool/env/llvm/llvm_config_darwin_llvm19.go b/xtool/env/llvm/llvm_config_darwin_llvm19.go deleted file mode 100644 index df9a345e1a..0000000000 --- a/xtool/env/llvm/llvm_config_darwin_llvm19.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build !byollvm && darwin && !amd64 && !llvm14 && !llvm15 && !llvm16 && !llvm17 && !llvm18 && !llvm22 - -/* - * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package llvm - -const ldLLVMConfigBin = "/opt/homebrew/opt/llvm@19/bin/llvm-config" diff --git a/xtool/env/llvm/llvm_config_darwin_llvm22.go b/xtool/env/llvm/llvm_config_darwin_llvm22.go index 89b1c167e7..17d8e8779f 100644 --- a/xtool/env/llvm/llvm_config_darwin_llvm22.go +++ b/xtool/env/llvm/llvm_config_darwin_llvm22.go @@ -1,4 +1,4 @@ -//go:build !byollvm && darwin && !amd64 && llvm22 +//go:build !byollvm && darwin && !amd64 /* * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. diff --git a/xtool/env/llvm/llvm_config_linux_llvm14.go b/xtool/env/llvm/llvm_config_linux_llvm14.go deleted file mode 100644 index 688f4edc36..0000000000 --- a/xtool/env/llvm/llvm_config_linux_llvm14.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build !byollvm && linux && llvm14 - -/* - * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package llvm - -const ldLLVMConfigBin = "/usr/lib/llvm-14/bin/llvm-config" diff --git a/xtool/env/llvm/llvm_config_linux_llvm15.go b/xtool/env/llvm/llvm_config_linux_llvm15.go deleted file mode 100644 index c09d0ae94f..0000000000 --- a/xtool/env/llvm/llvm_config_linux_llvm15.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build !byollvm && linux && llvm15 - -/* - * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package llvm - -const ldLLVMConfigBin = "/usr/lib/llvm-15/bin/llvm-config" diff --git a/xtool/env/llvm/llvm_config_linux_llvm16.go b/xtool/env/llvm/llvm_config_linux_llvm16.go deleted file mode 100644 index 4f2c4484a7..0000000000 --- a/xtool/env/llvm/llvm_config_linux_llvm16.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build !byollvm && linux && llvm16 - -/* - * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package llvm - -const ldLLVMConfigBin = "/usr/lib/llvm-16/bin/llvm-config" diff --git a/xtool/env/llvm/llvm_config_linux_llvm17.go b/xtool/env/llvm/llvm_config_linux_llvm17.go deleted file mode 100644 index a285763d72..0000000000 --- a/xtool/env/llvm/llvm_config_linux_llvm17.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build !byollvm && linux && llvm17 - -/* - * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package llvm - -const ldLLVMConfigBin = "/usr/lib/llvm-17/bin/llvm-config" diff --git a/xtool/env/llvm/llvm_config_linux_llvm18.go b/xtool/env/llvm/llvm_config_linux_llvm18.go deleted file mode 100644 index 79cc83a74d..0000000000 --- a/xtool/env/llvm/llvm_config_linux_llvm18.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build !byollvm && linux && llvm18 - -/* - * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package llvm - -const ldLLVMConfigBin = "/usr/lib/llvm-18/bin/llvm-config" diff --git a/xtool/env/llvm/llvm_config_linux_llvm19.go b/xtool/env/llvm/llvm_config_linux_llvm19.go deleted file mode 100644 index 35e4c5ab9c..0000000000 --- a/xtool/env/llvm/llvm_config_linux_llvm19.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build !byollvm && linux && !llvm14 && !llvm15 && !llvm16 && !llvm17 && !llvm18 && !llvm22 - -/* - * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package llvm - -const ldLLVMConfigBin = "/usr/lib/llvm-19/bin/llvm-config" diff --git a/xtool/env/llvm/llvm_config_linux_llvm22.go b/xtool/env/llvm/llvm_config_linux_llvm22.go index 3c11c78b47..cbb0379cff 100644 --- a/xtool/env/llvm/llvm_config_linux_llvm22.go +++ b/xtool/env/llvm/llvm_config_linux_llvm22.go @@ -1,4 +1,4 @@ -//go:build !byollvm && linux && llvm22 +//go:build !byollvm && linux /* * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. diff --git a/xtool/env/llvm/llvm_config_windows_llvm20.go b/xtool/env/llvm/llvm_config_windows_llvm22.go similarity index 79% rename from xtool/env/llvm/llvm_config_windows_llvm20.go rename to xtool/env/llvm/llvm_config_windows_llvm22.go index 36bac6ca39..5ca3ac779d 100644 --- a/xtool/env/llvm/llvm_config_windows_llvm20.go +++ b/xtool/env/llvm/llvm_config_windows_llvm22.go @@ -1,7 +1,7 @@ -//go:build !byollvm && windows && !llvm14 && !llvm15 && !llvm16 && !llvm17 && !llvm18 && !llvm19 +//go:build !byollvm && windows /* - * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/xtool/env/llvm/llvm_test.go b/xtool/env/llvm/llvm_test.go index 16a45005fe..7ebe63a99d 100644 --- a/xtool/env/llvm/llvm_test.go +++ b/xtool/env/llvm/llvm_test.go @@ -6,6 +6,8 @@ import ( "os" "path/filepath" "runtime" + "strconv" + "strings" "testing" ) @@ -15,42 +17,97 @@ func TestSetupPath(t *testing.T) { } binDir := t.TempDir() - llvmConfig := filepath.Join(t.TempDir(), "llvm-config") - if err := os.WriteFile(llvmConfig, []byte("#!/bin/sh\nprintf '%s\\n' \"${LLGO_TEST_LLVM_BINDIR}\"\n"), 0o755); err != nil { - t.Fatal(err) - } + llvmConfig := writeLLVMConfig(t, "22.1.8", 0) t.Setenv("LLVM_CONFIG", llvmConfig) t.Setenv("LLGO_TEST_LLVM_BINDIR", binDir) original := filepath.Join(t.TempDir(), "original") t.Setenv("PATH", original) - SetupPath() + if err := SetupPath(); err != nil { + t.Fatal(err) + } want := binDir + string(os.PathListSeparator) + original if got := os.Getenv("PATH"); got != want { t.Fatalf("PATH = %q, want %q", got, want) } - SetupPath() + if err := SetupPath(); err != nil { + t.Fatal(err) + } if got := os.Getenv("PATH"); got != want { t.Fatalf("second setup changed PATH to %q, want %q", got, want) } } -func TestSetupPathIgnoresMissingBinDir(t *testing.T) { +func TestSetupPathRejectsUnsupportedLLVM(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("test helper uses a shell script") } - llvmConfig := filepath.Join(t.TempDir(), "llvm-config") - if err := os.WriteFile(llvmConfig, []byte("#!/bin/sh\nexit 1\n"), 0o755); err != nil { - t.Fatal(err) + llvmConfig := writeLLVMConfig(t, "19.1.7", 0) + binDir := t.TempDir() + t.Setenv("LLVM_CONFIG", llvmConfig) + t.Setenv("LLGO_TEST_LLVM_BINDIR", binDir) + t.Setenv("PATH", filepath.Join(t.TempDir(), "original")) + before := os.Getenv("PATH") + + err := SetupPath() + if err == nil || !strings.Contains(err.Error(), "LLGo requires LLVM 22") { + t.Fatalf("SetupPath error = %v, want LLVM 22 requirement", err) + } + if got := os.Getenv("PATH"); got != before { + t.Fatalf("PATH changed from %q to %q for unsupported LLVM", before, got) } +} + +func TestSetupPathRejectsBrokenLLVMConfig(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("test helper uses a shell script") + } + + llvmConfig := writeLLVMConfig(t, "22.1.8", 7) t.Setenv("LLVM_CONFIG", llvmConfig) t.Setenv("PATH", filepath.Join(t.TempDir(), "original")) before := os.Getenv("PATH") - SetupPath() + err := SetupPath() + if err == nil || !strings.Contains(err.Error(), "run LLVM configuration") { + t.Fatalf("SetupPath error = %v, want llvm-config execution failure", err) + } if got := os.Getenv("PATH"); got != before { - t.Fatalf("PATH changed from %q to %q without an LLVM bin directory", before, got) + t.Fatalf("PATH changed from %q to %q for broken llvm-config", before, got) + } +} + +func TestParseMajorVersion(t *testing.T) { + for _, test := range []struct { + version string + want int + ok bool + }{ + {version: "22.1.8", want: 22, ok: true}, + {version: "19.1.7", want: 19, ok: true}, + {version: "LLVM 22.1.8"}, + {version: ""}, + } { + got, err := parseMajorVersion(test.version) + if (err == nil) != test.ok || got != test.want { + t.Errorf("parseMajorVersion(%q) = %d, %v; want %d, ok=%v", test.version, got, err, test.want, test.ok) + } + } +} + +func writeLLVMConfig(t *testing.T, version string, versionExit int) string { + t.Helper() + path := filepath.Join(t.TempDir(), "llvm-config") + script := "#!/bin/sh\n" + + "case \"$1\" in\n" + + " --version) printf '%s\\n' '" + version + "'; exit " + strconv.Itoa(versionExit) + ";;\n" + + " --bindir) printf '%s\\n' \"${LLGO_TEST_LLVM_BINDIR}\";;\n" + + " *) exit 2;;\n" + + "esac\n" + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatal(err) } + return path }