From 7af15a5fd2b8d223a6a0fbe17ec8a8fdcc900419 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Sun, 2 Aug 2026 18:42:14 +0800 Subject: [PATCH 1/3] runtime/wasmresume: retain high-water frame blocks --- runtime/internal/runtime/proc_wasm_resume.go | 4 +- runtime/internal/wasmresume/resume.go | 8 +- runtime/internal/wasmresume/storage.go | 99 +++++++++++--- runtime/internal/wasmresume/storage_clear.go | 30 ++++ .../internal/wasmresume/storage_clear_llgo.go | 27 ++++ runtime/internal/wasmresume/storage_test.go | 129 +++++++++++++++--- 6 files changed, 248 insertions(+), 49 deletions(-) create mode 100644 runtime/internal/wasmresume/storage_clear.go create mode 100644 runtime/internal/wasmresume/storage_clear_llgo.go diff --git a/runtime/internal/runtime/proc_wasm_resume.go b/runtime/internal/runtime/proc_wasm_resume.go index 73d77e40a5..80667dff5c 100644 --- a/runtime/internal/runtime/proc_wasm_resume.go +++ b/runtime/internal/runtime/proc_wasm_resume.go @@ -176,7 +176,7 @@ func runWasmResumeContext(gp *g) wasmresume.Action { platform.unwind = unwind platform.unwindRoot = captureWasmResumeGCRoot() if c.Sigsetjmp(unwind, 0) != 0 { - if !platform.context.Unwind(unsafe.Pointer(gp.defer_), FreeRoot) { + if !platform.context.Unwind(unsafe.Pointer(gp.defer_)) { platform.unwind = previous platform.unwindRoot = previousRoot if gp.goexit { @@ -282,7 +282,7 @@ func wasmResumeAllocDynamic(ctx *wasmresume.Context, size, align uintptr) unsafe //go:linkname wasmResumeFree __llgo_wasm_resume_free func wasmResumeFree(ctx *wasmresume.Context, frame *wasmresume.Frame) { - ctx.ReleaseFrame(frame, FreeRoot) + ctx.ReleaseFrame(frame) } //go:linkname wasmResumeClose __llgo_wasm_resume_close diff --git a/runtime/internal/wasmresume/resume.go b/runtime/internal/wasmresume/resume.go index a03d3a72fd..fe68d84a2d 100644 --- a/runtime/internal/wasmresume/resume.go +++ b/runtime/internal/wasmresume/resume.go @@ -99,16 +99,16 @@ func (c *Context) AllocateFrame( } // ReleaseFrame reclaims the most recently completed generated frame. -func (c *Context) ReleaseFrame(frame *Frame, release Releaser) { +func (c *Context) ReleaseFrame(frame *Frame) { if frame == nil || frame.Descriptor == nil { panic("wasmresume: invalid completed frame") } - c.storage.releaseFrame(unsafe.Pointer(frame), frame.Descriptor.FrameSize, release) + c.storage.releaseFrame(unsafe.Pointer(frame), frame.Descriptor.FrameSize) } // Unwind discards frames above the defer owner and redirects that owner to its // generated panic/defer state. -func (c *Context) Unwind(deferFrame unsafe.Pointer, release Releaser) bool { +func (c *Context) Unwind(deferFrame unsafe.Pointer) bool { if deferFrame == nil { return false } @@ -131,7 +131,7 @@ func (c *Context) Unwind(deferFrame unsafe.Pointer, release Releaser) bool { for c.top != owner { frame := c.top c.top = frame.Parent - c.storage.releaseFrame(unsafe.Pointer(frame), frame.Descriptor.FrameSize, release) + c.storage.releaseFrame(unsafe.Pointer(frame), frame.Descriptor.FrameSize) } c.returned = nil owner.PC = owner.Descriptor.UnwindPC diff --git a/runtime/internal/wasmresume/storage.go b/runtime/internal/wasmresume/storage.go index 04db9c870c..2e90262d5e 100644 --- a/runtime/internal/wasmresume/storage.go +++ b/runtime/internal/wasmresume/storage.go @@ -21,7 +21,7 @@ import "unsafe" const defaultFrameBlockSize = uintptr(2 << 10) type frameBlock struct { - prev *frameBlock + prev, next *frameBlock begin, end uintptr stackPointer uintptr } @@ -39,6 +39,9 @@ func (s *frameStorage) allocate( if frame, ok := allocateFromBlock(s.current, size, align); ok { return frame } + if frame, ok := s.allocateFromRetainedBlock(size, align); ok { + return frame + } payload, ok := addUintptr(size, unsafe.Sizeof(uintptr(0))) if !ok { @@ -61,6 +64,14 @@ func (s *frameStorage) allocate( } block := (*frameBlock)(raw) block.prev = s.current + block.next = nil + if s.current != nil { + block.next = s.current.next + s.current.next = block + if block.next != nil { + block.next.prev = block + } + } block.begin, ok = addUintptr(uintptr(raw), unsafe.Sizeof(frameBlock{})) if !ok { panic("wasmresume: frame block address overflow") @@ -69,6 +80,7 @@ func (s *frameStorage) allocate( if !ok { panic("wasmresume: frame block address overflow") } + clearFrameStorage(block.begin, block.end) block.stackPointer = block.begin s.current = block frame, ok := allocateFromBlock(block, size, align) @@ -78,29 +90,70 @@ func (s *frameStorage) allocate( return frame } +func (s *frameStorage) allocateFromRetainedBlock( + size, align uintptr, +) (unsafe.Pointer, bool) { + if s.current == nil { + return nil, false + } + for block := s.current.next; block != nil; block = block.next { + if block.stackPointer != block.begin { + panic("wasmresume: active frame block follows current block") + } + if _, _, ok := frameBounds(block, size, align); !ok { + continue + } + if block != s.current.next { + block.prev.next = block.next + if block.next != nil { + block.next.prev = block.prev + } + block.prev = s.current + block.next = s.current.next + s.current.next.prev = block + s.current.next = block + } + s.current = block + frame, ok := allocateFromBlock(block, size, align) + if !ok { + panic("wasmresume: retained frame block became too small") + } + return frame, true + } + return nil, false +} + func allocateFromBlock(block *frameBlock, size, align uintptr) (unsafe.Pointer, bool) { if block == nil { return nil, false } - header, ok := addUintptr(block.stackPointer, unsafe.Sizeof(uintptr(0))) + frame, next, ok := frameBounds(block, size, align) if !ok { return nil, false } - frame, ok := alignUintptr(header, align) + *(*uintptr)(unsafe.Pointer(frame - unsafe.Sizeof(uintptr(0)))) = block.stackPointer + block.stackPointer = next + return unsafe.Pointer(frame), true +} + +func frameBounds(block *frameBlock, size, align uintptr) (frame, next uintptr, ok bool) { + header, ok := addUintptr(block.stackPointer, unsafe.Sizeof(uintptr(0))) if !ok { - return nil, false + return 0, 0, false + } + frame, ok = alignUintptr(header, align) + if !ok { + return 0, 0, false } - next, ok := addUintptr(frame, size) + next, ok = addUintptr(frame, size) if !ok || next > block.end { - return nil, false + return 0, 0, false } - *(*uintptr)(unsafe.Pointer(frame - unsafe.Sizeof(uintptr(0)))) = block.stackPointer - block.stackPointer = next - return unsafe.Pointer(frame), true + return frame, next, true } func (s *frameStorage) releaseFrame( - frame unsafe.Pointer, size uintptr, release Releaser, + frame unsafe.Pointer, size uintptr, ) { if s.current == nil || frame == nil || size == 0 { panic("wasmresume: invalid frame release") @@ -122,32 +175,34 @@ func (s *frameStorage) releaseFrame( if previous < block.begin || previous >= address { panic("wasmresume: invalid frame allocation header") } - if block != s.current && release == nil { - panic("wasmresume: missing frame block reclaimer") - } for s.current != block { current := s.current s.current = current.prev - release(unsafe.Pointer(current)) + clearFrameStorage(current.begin, current.stackPointer) + current.stackPointer = current.begin } + clearFrameStorage(previous, block.stackPointer) block.stackPointer = previous if previous == block.begin && block.prev != nil { - if release == nil { - panic("wasmresume: missing frame block reclaimer") - } s.current = block.prev - release(unsafe.Pointer(block)) } } func (s *frameStorage) close(release Releaser) { - if s.current != nil && release == nil { + if s.current == nil { + return + } + if release == nil { panic("wasmresume: missing frame block reclaimer") } - for block := s.current; block != nil; { - previous := block.prev + block := s.current + for block.prev != nil { + block = block.prev + } + for block != nil { + next := block.next release(unsafe.Pointer(block)) - block = previous + block = next } s.current = nil } diff --git a/runtime/internal/wasmresume/storage_clear.go b/runtime/internal/wasmresume/storage_clear.go new file mode 100644 index 0000000000..41bef83e90 --- /dev/null +++ b/runtime/internal/wasmresume/storage_clear.go @@ -0,0 +1,30 @@ +//go:build !llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wasmresume + +import "unsafe" + +func clearFrameStorage(begin, end uintptr) { + if end > begin { + storage := unsafe.Slice((*byte)(unsafe.Pointer(begin)), end-begin) + for i := range storage { + storage[i] = 0 + } + } +} diff --git a/runtime/internal/wasmresume/storage_clear_llgo.go b/runtime/internal/wasmresume/storage_clear_llgo.go new file mode 100644 index 0000000000..c957ef31e0 --- /dev/null +++ b/runtime/internal/wasmresume/storage_clear_llgo.go @@ -0,0 +1,27 @@ +//go:build llgo + +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wasmresume + +import c "github.com/goplus/llgo/runtime/internal/clite" + +func clearFrameStorage(begin, end uintptr) { + if end > begin { + c.Memset(c.Pointer(begin), 0, end-begin) + } +} diff --git a/runtime/internal/wasmresume/storage_test.go b/runtime/internal/wasmresume/storage_test.go index 76cfd3a00d..f092ccd19a 100644 --- a/runtime/internal/wasmresume/storage_test.go +++ b/runtime/internal/wasmresume/storage_test.go @@ -55,20 +55,20 @@ func TestFrameStorageAlignsAndReusesFrames(t *testing.T) { t.Fatalf("root block allocations = %d, want 1", roots.allocs) } - storage.releaseFrame(second, 64, roots.release) + storage.releaseFrame(second, 64) reused := storage.allocate(64, 64, roots.allocate) if reused != second { t.Fatalf("frame was not reused: got %p, want %p", reused, second) } - storage.releaseFrame(reused, 64, roots.release) - storage.releaseFrame(first, 31, roots.release) + storage.releaseFrame(reused, 64) + storage.releaseFrame(first, 31) storage.close(roots.release) if roots.frees != 1 || len(roots.blocks) != 0 { t.Fatalf("released roots = %d, remaining = %d", roots.frees, len(roots.blocks)) } } -func TestFrameStorageAddsAndReleasesSegments(t *testing.T) { +func TestFrameStorageRetainsAndReleasesSegments(t *testing.T) { var ( storage frameStorage roots testFrameRoots @@ -78,17 +78,85 @@ func TestFrameStorageAddsAndReleasesSegments(t *testing.T) { if first == nil || second == nil || roots.allocs != 2 { t.Fatalf("allocations = %d, first=%p second=%p", roots.allocs, first, second) } - storage.releaseFrame(second, 128, roots.release) - if roots.frees != 1 { - t.Fatalf("released child segments = %d, want 1", roots.frees) + storage.releaseFrame(second, 128) + if roots.frees != 0 { + t.Fatalf("released child segments = %d, want 0", roots.frees) } - storage.releaseFrame(first, defaultFrameBlockSize, roots.release) + reused := storage.allocate(128, 16, roots.allocate) + if reused != second || roots.allocs != 2 { + t.Fatalf("retained segment = %p with %d allocations, want %p with 2", reused, roots.allocs, second) + } + storage.releaseFrame(reused, 128) + storage.releaseFrame(first, defaultFrameBlockSize) storage.close(roots.release) if roots.frees != 2 || len(roots.blocks) != 0 { t.Fatalf("released roots = %d, remaining = %d", roots.frees, len(roots.blocks)) } } +func TestFrameStorageFindsFittingRetainedSegment(t *testing.T) { + var ( + storage frameStorage + roots testFrameRoots + ) + base := storage.allocate(defaultFrameBlockSize, 16, roots.allocate) + small := storage.allocate(64, 16, roots.allocate) + large := storage.allocate(2*defaultFrameBlockSize, 16, roots.allocate) + if base == nil || small == nil || large == nil || roots.allocs != 3 { + t.Fatalf("allocations = %d, base=%p small=%p large=%p", roots.allocs, base, small, large) + } + + storage.releaseFrame(large, 2*defaultFrameBlockSize) + storage.releaseFrame(small, 64) + reused := storage.allocate(2*defaultFrameBlockSize, 16, roots.allocate) + if reused != large || roots.allocs != 3 { + t.Fatalf("fitting retained segment = %p with %d allocations, want %p with 3", reused, roots.allocs, large) + } + if storage.current.next == nil || storage.current.next.begin > uintptr(small) || storage.current.next.end < uintptr(small) { + t.Fatal("smaller retained segment was lost while moving the fitting segment") + } + + storage.releaseFrame(reused, 2*defaultFrameBlockSize) + storage.releaseFrame(base, defaultFrameBlockSize) + storage.close(roots.release) + if roots.frees != 3 || len(roots.blocks) != 0 { + t.Fatalf("released roots = %d, remaining = %d", roots.frees, len(roots.blocks)) + } +} + +func TestFrameStorageClearsReleasedRoots(t *testing.T) { + var ( + storage frameStorage + roots testFrameRoots + ) + base := storage.allocate(defaultFrameBlockSize, 16, roots.allocate) + frame := storage.allocate(64, 16, roots.allocate) + if base == nil || frame == nil { + t.Fatal("frame allocation failed") + } + bytes := unsafe.Slice((*byte)(frame), 64) + for i := range bytes { + bytes[i] = 0xa5 + } + header := (*uintptr)(unsafe.Pointer(uintptr(frame) - unsafe.Sizeof(uintptr(0)))) + if *header == 0 { + t.Fatal("frame allocation header was not initialized") + } + + storage.releaseFrame(frame, 64) + if *header != 0 { + t.Fatalf("released frame header = %#x, want 0", *header) + } + for i, value := range bytes { + if value != 0 { + t.Fatalf("released frame byte %d = %#x, want 0", i, value) + } + } + + storage.releaseFrame(base, defaultFrameBlockSize) + storage.close(roots.release) +} + func TestContextOwnsGeneratedFrameStorage(t *testing.T) { var ( ctx Context @@ -101,7 +169,7 @@ func TestContextOwnsGeneratedFrameStorage(t *testing.T) { } frame := (*testLeafFrame)(raw) frame.Descriptor = &Descriptor{FrameSize: size} - ctx.ReleaseFrame(&frame.Frame, roots.release) + ctx.ReleaseFrame(&frame.Frame) ctx.Close(roots.release) if roots.allocs != 1 || roots.frees != 1 { t.Fatalf("root lifecycle = %d allocs, %d frees", roots.allocs, roots.frees) @@ -122,7 +190,7 @@ func TestContextReleaseFrameDiscardsDynamicStorage(t *testing.T) { t.Fatal("dynamic frame storage allocation failed") } - ctx.ReleaseFrame(frame, roots.release) + ctx.ReleaseFrame(frame) reused := ctx.AllocateFrame(frameSize, 8, roots.allocate) if reused != raw { t.Fatalf("frame storage was not rewound: got %p, want %p", reused, raw) @@ -156,7 +224,7 @@ func TestContextUnwindReclaimsChildrenAndRedirectsOwner(t *testing.T) { child.Descriptor = childDescriptor ctx.top = &child.Frame - if !ctx.Unwind(unsafe.Pointer(&token), roots.release) { + if !ctx.Unwind(unsafe.Pointer(&token)) { t.Fatal("Context.Unwind did not find the defer owner") } if ctx.top != &owner.Frame || owner.PC != ownerDescriptor.UnwindPC { @@ -171,7 +239,7 @@ func TestContextUnwindReclaimsChildrenAndRedirectsOwner(t *testing.T) { func TestContextUnwindRejectsMissingOwner(t *testing.T) { var ctx Context - if ctx.Unwind(nil, nil) || ctx.Unwind(unsafe.Pointer(new(byte)), nil) { + if ctx.Unwind(nil) || ctx.Unwind(unsafe.Pointer(new(byte))) { t.Fatal("Context.Unwind accepted a missing defer owner") } } @@ -188,7 +256,7 @@ func TestContextUnwindIgnoresIncompleteDescriptor(t *testing.T) { UnwindOffset: unsafe.Offsetof(frame.deferFrame), } ctx.top = &frame.Frame - if ctx.Unwind(unsafe.Pointer(&token), nil) { + if ctx.Unwind(unsafe.Pointer(&token)) { t.Fatal("Context.Unwind accepted a descriptor without an unwind PC") } } @@ -232,7 +300,7 @@ func TestFrameStorageRejectsInvalidReleaseState(t *testing.T) { assertPanic("empty", func() { var storage frameStorage - storage.releaseFrame(unsafe.Pointer(new(byte)), 1, nil) + storage.releaseFrame(unsafe.Pointer(new(byte)), 1) }) var ( @@ -241,18 +309,18 @@ func TestFrameStorageRejectsInvalidReleaseState(t *testing.T) { ) frame := storage.allocate(8, 8, roots.allocate) assertPanic("nil frame", func() { - storage.releaseFrame(nil, 8, roots.release) + storage.releaseFrame(nil, 8) }) assertPanic("zero size", func() { - storage.releaseFrame(frame, 0, roots.release) + storage.releaseFrame(frame, 0) }) assertPanic("foreign frame", func() { - storage.releaseFrame(unsafe.Pointer(new(byte)), 1, roots.release) + storage.releaseFrame(unsafe.Pointer(new(byte)), 1) }) assertPanic("invalid header", func() { header := unsafe.Pointer(uintptr(frame) - unsafe.Sizeof(uintptr(0))) *(*uintptr)(header) = 0 - storage.releaseFrame(frame, 8, roots.release) + storage.releaseFrame(frame, 8) }) storage.close(roots.release) @@ -271,7 +339,7 @@ func TestContextRejectsFrameWithoutDescriptor(t *testing.T) { t.Fatal("ReleaseFrame accepted an untyped frame") } }() - ctx.ReleaseFrame(&Frame{}, nil) + ctx.ReleaseFrame(&Frame{}) } func BenchmarkFrameStorageHotAllocateRelease(b *testing.B) { @@ -280,11 +348,30 @@ func BenchmarkFrameStorageHotAllocateRelease(b *testing.B) { roots testFrameRoots ) frame := storage.allocate(64, 16, roots.allocate) - storage.releaseFrame(frame, 64, roots.release) + storage.releaseFrame(frame, 64) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + frame = storage.allocate(64, 16, roots.allocate) + storage.releaseFrame(frame, 64) + } +} + +func BenchmarkFrameStorageRetainedOverflow(b *testing.B) { + var ( + storage frameStorage + roots testFrameRoots + ) + base := storage.allocate(defaultFrameBlockSize, 16, roots.allocate) + frame := storage.allocate(64, 16, roots.allocate) + storage.releaseFrame(frame, 64) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { frame = storage.allocate(64, 16, roots.allocate) - storage.releaseFrame(frame, 64, roots.release) + storage.releaseFrame(frame, 64) } + b.StopTimer() + storage.releaseFrame(base, defaultFrameBlockSize) + storage.close(roots.release) } From 51864ccb9a676d3065928dc8a7eef16df01c2f79 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Sun, 2 Aug 2026 19:44:23 +0800 Subject: [PATCH 2/3] runtime/wasmresume: reuse compatibility frame arenas --- internal/wasmresume/abi.go | 3 +- internal/wasmresume/boundary.go | 6 ++- internal/wasmresume/boundary_test.go | 3 +- internal/wasmresume/compat.go | 12 +++-- internal/wasmresume/state.go | 18 +++++-- internal/wasmresume/state_test.go | 11 ++++- runtime/internal/runtime/proc_wasm_resume.go | 52 ++++++++++++++++++-- 7 files changed, 89 insertions(+), 16 deletions(-) diff --git a/internal/wasmresume/abi.go b/internal/wasmresume/abi.go index 37ea167cc7..82a619a08a 100644 --- a/internal/wasmresume/abi.go +++ b/internal/wasmresume/abi.go @@ -26,7 +26,8 @@ const ( resumeEntryPrefix = "__llgo_wasm_resume." startEntryPrefix = "__llgo_wasm_start." descriptorPrefix = "__llgo_wasm_resume_desc." - frameCloseName = "__llgo_wasm_resume_close" + compatEnterName = "__llgo_wasm_resume_compat_enter" + compatLeaveName = "__llgo_wasm_resume_compat_leave" actionContinue = 0 actionReturn = 1 actionSuspend = 2 diff --git a/internal/wasmresume/boundary.go b/internal/wasmresume/boundary.go index 0ce5dd9827..63e1dfd12b 100644 --- a/internal/wasmresume/boundary.go +++ b/internal/wasmresume/boundary.go @@ -29,7 +29,8 @@ const ( runtimeFrameAlloc = "__llgo_wasm_resume_alloc" runtimeDynamicAlloc = "__llgo_wasm_resume_alloc_dynamic" runtimeFrameFree = "__llgo_wasm_resume_free" - runtimeFrameClose = "__llgo_wasm_resume_close" + runtimeCompatEnter = "__llgo_wasm_resume_compat_enter" + runtimeCompatLeave = "__llgo_wasm_resume_compat_leave" ) // IsRuntimeABIImplementation reports functions which implement the resumable @@ -71,5 +72,6 @@ func IsNonSuspendingBoundary(name string) bool { name == runtimeFrameAlloc || name == runtimeDynamicAlloc || name == runtimeFrameFree || - name == runtimeFrameClose + name == runtimeCompatEnter || + name == runtimeCompatLeave } diff --git a/internal/wasmresume/boundary_test.go b/internal/wasmresume/boundary_test.go index c98c08e3e4..e12a02455e 100644 --- a/internal/wasmresume/boundary_test.go +++ b/internal/wasmresume/boundary_test.go @@ -24,7 +24,8 @@ func TestRuntimeBoundaries(t *testing.T) { runtimeFrameAlloc, runtimeDynamicAlloc, runtimeFrameFree, - runtimeFrameClose, + runtimeCompatEnter, + runtimeCompatLeave, runtimeRunWasmResumeContext, "github.com/goplus/llgo/runtime/internal/runtime.AllocU", "github.com/goplus/llgo/runtime/internal/runtime.AllocZ", diff --git a/internal/wasmresume/compat.go b/internal/wasmresume/compat.go index 846fe12da9..fb056d1958 100644 --- a/internal/wasmresume/compat.go +++ b/internal/wasmresume/compat.go @@ -56,6 +56,12 @@ func emitCompatibilityWrapper( llvm.ConstInt(abi.uintptrType, targetData.TypeAllocSize(abi.contextType), false), llvm.ConstInt(ctx.Int1Type(), 0, false), }, "") + owner := builder.CreateCall( + declareCompatEnter(mod, abi).GlobalValueType(), + declareCompatEnter(mod, abi), + []llvm.Value{context}, + "resume.arena.owner", + ) builder.CreateIntrinsic(ctx.VoidType(), llvm.LookupIntrinsicID("llvm.memset"), []llvm.Value{ root, llvm.ConstInt(ctx.Int8Type(), 0, false), @@ -123,9 +129,9 @@ func emitCompatibilityWrapper( builder.SetInsertPointAtEnd(finished) builder.CreateCall( - declareFrameClose(mod, abi).GlobalValueType(), - declareFrameClose(mod, abi), - []llvm.Value{context}, + declareCompatLeave(mod, abi).GlobalValueType(), + declareCompatLeave(mod, abi), + []llvm.Value{context, owner}, "", ) if lowered.layout.plan.resultSlot == 0 { diff --git a/internal/wasmresume/state.go b/internal/wasmresume/state.go index 81e14abc5d..2448cbc633 100644 --- a/internal/wasmresume/state.go +++ b/internal/wasmresume/state.go @@ -390,11 +390,21 @@ func declareFrameFree(mod llvm.Module, abi resumeABI) llvm.Value { return fn } -func declareFrameClose(mod llvm.Module, abi resumeABI) llvm.Value { - fn := mod.NamedFunction(frameCloseName) +func declareCompatEnter(mod llvm.Module, abi resumeABI) llvm.Value { + fn := mod.NamedFunction(compatEnterName) if fn.IsNil() { - fn = llvm.AddFunction(mod, frameCloseName, llvm.FunctionType( - abi.ctx.VoidType(), []llvm.Type{abi.ptr}, false, + fn = llvm.AddFunction(mod, compatEnterName, llvm.FunctionType( + abi.ptr, []llvm.Type{abi.ptr}, false, + )) + } + return fn +} + +func declareCompatLeave(mod llvm.Module, abi resumeABI) llvm.Value { + fn := mod.NamedFunction(compatLeaveName) + if fn.IsNil() { + fn = llvm.AddFunction(mod, compatLeaveName, llvm.FunctionType( + abi.ctx.VoidType(), []llvm.Type{abi.ptr, abi.ptr}, false, )) } return fn diff --git a/internal/wasmresume/state_test.go b/internal/wasmresume/state_test.go index 6b77f63e96..44c36b0075 100644 --- a/internal/wasmresume/state_test.go +++ b/internal/wasmresume/state_test.go @@ -168,6 +168,8 @@ func TestLowerPrototypeBuildsDirectCallStateMachine(t *testing.T) { `i32 1, label %resume.1`, `call ptr @__llgo_wasm_resume_alloc(ptr %0,`, `call void @llvm.memset`, + `call ptr @__llgo_wasm_resume_compat_enter(`, + `call void @__llgo_wasm_resume_compat_leave(`, `ret i8 0`, `%returned = load ptr`, `call void @__llgo_wasm_resume_free(ptr %0, ptr %returned)`, @@ -388,8 +390,13 @@ func defineStateMachineHarness( builder.SetInsertPointAtEnd(block) builder.CreateRetVoid() - close := mod.NamedFunction(frameCloseName) - block = ctx.AddBasicBlock(close, "entry") + enter := mod.NamedFunction(compatEnterName) + block = ctx.AddBasicBlock(enter, "entry") + builder.SetInsertPointAtEnd(block) + builder.CreateRet(enter.Param(0)) + + leave := mod.NamedFunction(compatLeaveName) + block = ctx.AddBasicBlock(leave, "entry") builder.SetInsertPointAtEnd(block) builder.CreateRetVoid() diff --git a/runtime/internal/runtime/proc_wasm_resume.go b/runtime/internal/runtime/proc_wasm_resume.go index 80667dff5c..cf490e2533 100644 --- a/runtime/internal/runtime/proc_wasm_resume.go +++ b/runtime/internal/runtime/proc_wasm_resume.go @@ -38,6 +38,7 @@ type runtimeContextPlatform struct { var wasmSched struct { m m p p + compat wasmresume.Context gcRoot wasmGCRootContext runq runqueue.Queue[*g] started bool @@ -46,6 +47,13 @@ var wasmSched struct { mainExited bool } +var ( + // The frame owner stays on runtime-owned storage while the compatibility + // owner follows nested stack-local wrappers. + wasmResumeFrameOwner *wasmresume.Context + wasmResumeCompatOwner *wasmresume.Context +) + func initRuntimeContext(ctx *runtimeContext, callergp *g, status uint32) *g { gp := initG(ctx, callergp, status) if wasmGCRootEnabled { @@ -138,6 +146,7 @@ func RunWasmMain() { if gp.isMain { releaseWasmContext(gp) stopWasmResumeHost() + wasmSched.compat.Close(FreeRoot) wasmSched.running = false return } @@ -175,7 +184,13 @@ func runWasmResumeContext(gp *g) wasmresume.Action { previousRoot := platform.unwindRoot platform.unwind = unwind platform.unwindRoot = captureWasmResumeGCRoot() + previousFrameOwner := wasmResumeFrameOwner + previousCompatOwner := wasmResumeCompatOwner + wasmResumeFrameOwner = &platform.context + wasmResumeCompatOwner = &platform.context if c.Sigsetjmp(unwind, 0) != 0 { + wasmResumeFrameOwner = previousFrameOwner + wasmResumeCompatOwner = previousCompatOwner if !platform.context.Unwind(unsafe.Pointer(gp.defer_)) { platform.unwind = previous platform.unwindRoot = previousRoot @@ -197,6 +212,8 @@ func runWasmResumeContext(gp *g) wasmresume.Action { } } action := platform.context.Run() + wasmResumeFrameOwner = previousFrameOwner + wasmResumeCompatOwner = previousCompatOwner platform.unwind = previous platform.unwindRoot = previousRoot if wasmGCRootEnabled { @@ -272,22 +289,51 @@ func goexitBackend(gp *g) { //go:linkname wasmResumeAlloc __llgo_wasm_resume_alloc func wasmResumeAlloc(ctx *wasmresume.Context, size, align uintptr) unsafe.Pointer { + if ctx == wasmResumeCompatOwner && wasmResumeFrameOwner != nil { + ctx = wasmResumeFrameOwner + } return ctx.AllocateFrame(size, align, AllocRoot) } //go:linkname wasmResumeAllocDynamic __llgo_wasm_resume_alloc_dynamic func wasmResumeAllocDynamic(ctx *wasmresume.Context, size, align uintptr) unsafe.Pointer { + if ctx == wasmResumeCompatOwner && wasmResumeFrameOwner != nil { + ctx = wasmResumeFrameOwner + } return ctx.AllocateFrame(size, align, AllocRoot) } //go:linkname wasmResumeFree __llgo_wasm_resume_free func wasmResumeFree(ctx *wasmresume.Context, frame *wasmresume.Frame) { + if ctx == wasmResumeCompatOwner && wasmResumeFrameOwner != nil { + ctx = wasmResumeFrameOwner + } ctx.ReleaseFrame(frame) } -//go:linkname wasmResumeClose __llgo_wasm_resume_close -func wasmResumeClose(ctx *wasmresume.Context) { - ctx.Close(FreeRoot) +//go:linkname wasmResumeCompatEnter __llgo_wasm_resume_compat_enter +func wasmResumeCompatEnter(ctx *wasmresume.Context) unsafe.Pointer { + if ctx == nil { + fatal("runtime: invalid WebAssembly compatibility arena entry") + return nil + } + owner := wasmResumeCompatOwner + if owner == nil { + owner = &wasmSched.compat + wasmResumeFrameOwner = owner + } + wasmResumeCompatOwner = ctx + return unsafe.Pointer(owner) +} + +//go:linkname wasmResumeCompatLeave __llgo_wasm_resume_compat_leave +func wasmResumeCompatLeave(ctx *wasmresume.Context, rawOwner unsafe.Pointer) { + owner := (*wasmresume.Context)(rawOwner) + if owner == nil || wasmResumeCompatOwner != ctx { + fatal("runtime: invalid WebAssembly compatibility arena exit") + return + } + wasmResumeCompatOwner = owner } // CurrentGForTesting returns an opaque handle suitable for ReadyForTesting. From 86d59afe4b35a22692629c80f36113d7b820fc5b Mon Sep 17 00:00:00 2001 From: Li Jie Date: Sun, 2 Aug 2026 20:33:20 +0800 Subject: [PATCH 3/3] compiler/wasmresume: add frame arena fast paths --- internal/wasmresume/arena.go | 233 ++++++++++++++++++++ internal/wasmresume/arena_test.go | 78 +++++++ internal/wasmresume/dynamic.go | 10 - internal/wasmresume/execution_test.go | 34 +++ internal/wasmresume/state.go | 20 -- internal/wasmresume/state_test.go | 31 ++- runtime/internal/wasmresume/storage_test.go | 22 ++ 7 files changed, 396 insertions(+), 32 deletions(-) create mode 100644 internal/wasmresume/arena.go create mode 100644 internal/wasmresume/arena_test.go diff --git a/internal/wasmresume/arena.go b/internal/wasmresume/arena.go new file mode 100644 index 0000000000..c1b8c2bac5 --- /dev/null +++ b/internal/wasmresume/arena.go @@ -0,0 +1,233 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wasmresume + +import "github.com/xgo-dev/llvm" + +// Fast paths know only the pointer-sized frameBlock prefix shared with the +// runtime. Block creation, retained-block selection, and final reclamation stay +// behind the runtime ABI. +const ( + frameAllocFastName = "__llgo_wasm_resume_alloc.fast" + frameDynamicAllocFastName = "__llgo_wasm_resume_alloc_dynamic.fast" + frameFreeFastName = "__llgo_wasm_resume_free.fast" +) + +func declareFrameAllocator(mod llvm.Module, abi resumeABI) llvm.Value { + return defineFastFrameAllocator(mod, abi, frameAllocFastName, frameAllocName) +} + +func declareDynamicAllocator(mod llvm.Module, abi resumeABI) llvm.Value { + return defineFastFrameAllocator( + mod, abi, frameDynamicAllocFastName, frameDynamicAllocName, + ) +} + +func declareFrameFree(mod llvm.Module, abi resumeABI) llvm.Value { + fn := mod.NamedFunction(frameFreeFastName) + if !fn.IsNil() { + return fn + } + + ctx := mod.Context() + fnType := llvm.FunctionType(ctx.VoidType(), []llvm.Type{abi.ptr, abi.ptr}, false) + fn = llvm.AddFunction(mod, frameFreeFastName, fnType) + fn.SetLinkage(llvm.InternalLinkage) + fn.AddFunctionAttr(ctx.CreateEnumAttribute(llvm.AttributeKindID("noinline"), 0)) + + slow := mod.NamedFunction(frameFreeName) + if slow.IsNil() { + slow = llvm.AddFunction(mod, frameFreeName, fnType) + } + blockType := frameBlockType(abi) + entry := ctx.AddBasicBlock(fn, "entry") + bounds := ctx.AddBasicBlock(fn, "bounds") + header := ctx.AddBasicBlock(fn, "header") + fast := ctx.AddBasicBlock(fn, "fast") + slowPath := ctx.AddBasicBlock(fn, "slow") + + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(entry) + currentField := builder.CreateStructGEP(abi.contextType, fn.Param(0), 2, "") + block := builder.CreateLoad(abi.ptr, currentField, "frame.block") + builder.CreateCondBr( + builder.CreateICmp(llvm.IntNE, block, llvm.ConstNull(abi.ptr), ""), + bounds, + slowPath, + ) + + builder.SetInsertPointAtEnd(bounds) + begin := builder.CreateLoad( + abi.uintptrType, builder.CreateStructGEP(blockType, block, 2, ""), "frame.begin", + ) + stackPointerField := builder.CreateStructGEP(blockType, block, 4, "") + stackPointer := builder.CreateLoad(abi.uintptrType, stackPointerField, "frame.sp") + frameAddress := builder.CreatePtrToInt(fn.Param(1), abi.uintptrType, "frame.address") + pointerSize := llvm.ConstInt(abi.uintptrType, uint64(abi.uintptrType.IntTypeWidth()/8), false) + minimum := builder.CreateAdd(begin, pointerSize, "frame.minimum") + inBounds := builder.CreateAnd( + builder.CreateICmp(llvm.IntUGE, frameAddress, minimum, ""), + builder.CreateICmp(llvm.IntULT, frameAddress, stackPointer, ""), + "frame.in.bounds", + ) + builder.CreateCondBr(inBounds, header, slowPath) + + builder.SetInsertPointAtEnd(header) + headerAddress := builder.CreateSub(frameAddress, pointerSize, "frame.header") + previous := builder.CreateLoad( + abi.uintptrType, + builder.CreateIntToPtr(headerAddress, abi.ptr, "frame.header.ptr"), + "frame.previous", + ) + previousBlock := builder.CreateLoad( + abi.ptr, builder.CreateStructGEP(blockType, block, 0, ""), "frame.previous.block", + ) + validHeader := builder.CreateAnd( + builder.CreateICmp(llvm.IntUGE, previous, begin, ""), + builder.CreateICmp(llvm.IntULT, previous, frameAddress, ""), + "frame.valid.header", + ) + needsBlockPop := builder.CreateAnd( + builder.CreateICmp(llvm.IntEQ, previous, begin, ""), + builder.CreateICmp(llvm.IntNE, previousBlock, llvm.ConstNull(abi.ptr), ""), + "frame.needs.block.pop", + ) + builder.CreateCondBr( + builder.CreateAnd(validHeader, builder.CreateNot(needsBlockPop, ""), "frame.fast"), + fast, + slowPath, + ) + + builder.SetInsertPointAtEnd(fast) + builder.CreateIntrinsic(ctx.VoidType(), llvm.LookupIntrinsicID("llvm.memset"), []llvm.Value{ + builder.CreateIntToPtr(previous, abi.ptr, "frame.clear.begin"), + llvm.ConstInt(ctx.Int8Type(), 0, false), + builder.CreateSub(stackPointer, previous, "frame.clear.size"), + llvm.ConstInt(ctx.Int1Type(), 0, false), + }, "") + builder.CreateStore(previous, stackPointerField) + builder.CreateRetVoid() + + builder.SetInsertPointAtEnd(slowPath) + builder.CreateCall(slow.GlobalValueType(), slow, []llvm.Value{fn.Param(0), fn.Param(1)}, "") + builder.CreateRetVoid() + return fn +} + +func defineFastFrameAllocator( + mod llvm.Module, + abi resumeABI, + fastName, slowName string, +) llvm.Value { + fn := mod.NamedFunction(fastName) + if !fn.IsNil() { + return fn + } + + ctx := mod.Context() + fnType := llvm.FunctionType( + abi.ptr, []llvm.Type{abi.ptr, abi.uintptrType, abi.uintptrType}, false, + ) + fn = llvm.AddFunction(mod, fastName, fnType) + fn.SetLinkage(llvm.InternalLinkage) + fn.AddFunctionAttr(ctx.CreateEnumAttribute(llvm.AttributeKindID("noinline"), 0)) + + slow := mod.NamedFunction(slowName) + if slow.IsNil() { + slow = llvm.AddFunction(mod, slowName, fnType) + } + blockType := frameBlockType(abi) + entry := ctx.AddBasicBlock(fn, "entry") + check := ctx.AddBasicBlock(fn, "check") + fast := ctx.AddBasicBlock(fn, "fast") + slowPath := ctx.AddBasicBlock(fn, "slow") + + builder := ctx.NewBuilder() + defer builder.Dispose() + builder.SetInsertPointAtEnd(entry) + currentField := builder.CreateStructGEP(abi.contextType, fn.Param(0), 2, "") + block := builder.CreateLoad(abi.ptr, currentField, "frame.block") + builder.CreateCondBr( + builder.CreateICmp(llvm.IntNE, block, llvm.ConstNull(abi.ptr), ""), + check, + slowPath, + ) + + builder.SetInsertPointAtEnd(check) + stackPointerField := builder.CreateStructGEP(blockType, block, 4, "") + stackPointer := builder.CreateLoad(abi.uintptrType, stackPointerField, "frame.sp") + end := builder.CreateLoad( + abi.uintptrType, builder.CreateStructGEP(blockType, block, 3, ""), "frame.end", + ) + pointerSize := llvm.ConstInt(abi.uintptrType, uint64(abi.uintptrType.IntTypeWidth()/8), false) + header := builder.CreateAdd(stackPointer, pointerSize, "frame.header") + alignMask := builder.CreateSub( + fn.Param(2), llvm.ConstInt(abi.uintptrType, 1, false), "frame.align.mask", + ) + padded := builder.CreateAdd(header, alignMask, "frame.padded") + negativeAlign := builder.CreateSub( + llvm.ConstNull(abi.uintptrType), fn.Param(2), "frame.negative.align", + ) + frameAddress := builder.CreateAnd(padded, negativeAlign, "frame.address") + next := builder.CreateAdd(frameAddress, fn.Param(1), "frame.next") + fits := builder.CreateAnd( + builder.CreateAnd( + builder.CreateICmp(llvm.IntUGE, header, stackPointer, ""), + builder.CreateICmp(llvm.IntUGE, padded, header, ""), + "frame.header.valid", + ), + builder.CreateAnd( + builder.CreateICmp(llvm.IntUGE, frameAddress, header, ""), + builder.CreateAnd( + builder.CreateICmp(llvm.IntUGE, next, frameAddress, ""), + builder.CreateICmp(llvm.IntULE, next, end, ""), + "frame.end.valid", + ), + "frame.bounds.valid", + ), + "frame.fits", + ) + builder.CreateCondBr(fits, fast, slowPath) + + builder.SetInsertPointAtEnd(fast) + headerAddress := builder.CreateSub(frameAddress, pointerSize, "frame.header.address") + builder.CreateStore( + stackPointer, builder.CreateIntToPtr(headerAddress, abi.ptr, "frame.header.ptr"), + ) + builder.CreateStore(next, stackPointerField) + builder.CreateRet(builder.CreateIntToPtr(frameAddress, abi.ptr, "frame.ptr")) + + builder.SetInsertPointAtEnd(slowPath) + allocated := builder.CreateCall( + slow.GlobalValueType(), slow, + []llvm.Value{fn.Param(0), fn.Param(1), fn.Param(2)}, + "frame.slow", + ) + builder.CreateRet(allocated) + return fn +} + +func frameBlockType(abi resumeABI) llvm.Type { + return abi.ctx.StructType([]llvm.Type{ + abi.ptr, + abi.ptr, + abi.uintptrType, + abi.uintptrType, + abi.uintptrType, + }, false) +} diff --git a/internal/wasmresume/arena_test.go b/internal/wasmresume/arena_test.go new file mode 100644 index 0000000000..d284c26e69 --- /dev/null +++ b/internal/wasmresume/arena_test.go @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wasmresume + +import ( + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestFrameArenaABILayout(t *testing.T) { + for _, layout := range []string{ + "e-m:e-p:32:32-i64:64-n32:64-S128", + "e-m:e-p:64:64-i64:64-n32:64-S128", + } { + t.Run(layout, func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + targetData := llvm.NewTargetData(layout) + defer targetData.Dispose() + abi := newResumeABI(ctx, targetData) + pointerSize := uint64(targetData.PointerSize()) + + if got, want := targetData.TypeAllocSize(abi.contextType), 3*pointerSize; got != want { + t.Fatalf("Context size = %d, want %d", got, want) + } + if got, want := targetData.ElementOffset(abi.contextType, 2), 2*pointerSize; got != want { + t.Fatalf("Context storage offset = %d, want %d", got, want) + } + + block := frameBlockType(abi) + if got, want := targetData.TypeAllocSize(block), 5*pointerSize; got != want { + t.Fatalf("frameBlock size = %d, want %d", got, want) + } + for field := range 5 { + if got, want := targetData.ElementOffset(block, field), uint64(field)*pointerSize; got != want { + t.Fatalf("frameBlock field %d offset = %d, want %d", field, got, want) + } + } + }) + } +} + +func TestFrameArenaFastPathsAreNotInlined(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("frame-arena-fast-paths") + defer mod.Dispose() + targetData := llvm.NewTargetData("e-m:e-p:32:32-i64:64-n32:64-S128") + defer targetData.Dispose() + abi := newResumeABI(ctx, targetData) + + functions := []llvm.Value{ + declareFrameAllocator(mod, abi), + declareDynamicAllocator(mod, abi), + declareFrameFree(mod, abi), + } + kind := llvm.AttributeKindID("noinline") + for _, fn := range functions { + if fn.GetEnumFunctionAttribute(kind).IsNil() { + t.Errorf("%s is missing the noinline attribute", fn.Name()) + } + } +} diff --git a/internal/wasmresume/dynamic.go b/internal/wasmresume/dynamic.go index f45a10097f..a9558a9b23 100644 --- a/internal/wasmresume/dynamic.go +++ b/internal/wasmresume/dynamic.go @@ -111,13 +111,3 @@ func isCallToIntrinsic(value llvm.Value, name string) bool { return !callee.IsAFunction().IsNil() && (callee.Name() == name || strings.HasPrefix(callee.Name(), name+".")) } - -func declareDynamicAllocator(mod llvm.Module, abi resumeABI) llvm.Value { - fn := mod.NamedFunction(frameDynamicAllocName) - if fn.IsNil() { - fn = llvm.AddFunction(mod, frameDynamicAllocName, llvm.FunctionType( - abi.ptr, []llvm.Type{abi.ptr, abi.uintptrType, abi.uintptrType}, false, - )) - } - return fn -} diff --git a/internal/wasmresume/execution_test.go b/internal/wasmresume/execution_test.go index 0b8d31f869..0dddce8ed2 100644 --- a/internal/wasmresume/execution_test.go +++ b/internal/wasmresume/execution_test.go @@ -162,6 +162,7 @@ func buildExecutableWasmResumeObject(t *testing.T, triple string) []byte { defineStateMachineHarness(mod, targetData, root, []llvm.Value{ llvm.ConstInt(i32, 6, false), }) + defineExecutableMemset(mod, targetData) if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { t.Fatalf("verify %s executable: %v\n%s", triple, err, mod.String()) } @@ -177,3 +178,36 @@ func buildExecutableWasmResumeObject(t *testing.T, triple string) []byte { defer buffer.Dispose() return append([]byte(nil), buffer.Bytes()...) } + +func defineExecutableMemset(mod llvm.Module, targetData llvm.TargetData) { + ctx := mod.Context() + ptr := llvm.PointerType(ctx.Int8Type(), 0) + uintptrType := ctx.IntType(targetData.PointerSize() * 8) + fn := llvm.AddFunction(mod, "memset", llvm.FunctionType( + ptr, []llvm.Type{ptr, ctx.Int32Type(), uintptrType}, false, + )) + entry := ctx.AddBasicBlock(fn, "entry") + loop := ctx.AddBasicBlock(fn, "loop") + done := ctx.AddBasicBlock(fn, "done") + builder := ctx.NewBuilder() + defer builder.Dispose() + + zero := llvm.ConstNull(uintptrType) + builder.SetInsertPointAtEnd(entry) + builder.CreateCondBr( + builder.CreateICmp(llvm.IntEQ, fn.Param(2), zero, ""), done, loop, + ) + + builder.SetInsertPointAtEnd(loop) + index := builder.CreatePHI(uintptrType, "index") + address := builder.CreateInBoundsGEP(ctx.Int8Type(), fn.Param(0), []llvm.Value{index}, "") + builder.CreateStore(builder.CreateTrunc(fn.Param(1), ctx.Int8Type(), ""), address) + next := builder.CreateAdd(index, llvm.ConstInt(uintptrType, 1, false), "next") + builder.CreateCondBr( + builder.CreateICmp(llvm.IntULT, next, fn.Param(2), ""), loop, done, + ) + index.AddIncoming([]llvm.Value{zero, next}, []llvm.BasicBlock{entry, loop}) + + builder.SetInsertPointAtEnd(done) + builder.CreateRet(fn.Param(0)) +} diff --git a/internal/wasmresume/state.go b/internal/wasmresume/state.go index 2448cbc633..8e09578fb2 100644 --- a/internal/wasmresume/state.go +++ b/internal/wasmresume/state.go @@ -370,26 +370,6 @@ func callFramePrefix(ctx llvm.Context, typ llvm.Type) llvm.Type { return ctx.StructType(fields, false) } -func declareFrameAllocator(mod llvm.Module, abi resumeABI) llvm.Value { - fn := mod.NamedFunction(frameAllocName) - if fn.IsNil() { - fn = llvm.AddFunction(mod, frameAllocName, llvm.FunctionType( - abi.ptr, []llvm.Type{abi.ptr, abi.uintptrType, abi.uintptrType}, false, - )) - } - return fn -} - -func declareFrameFree(mod llvm.Module, abi resumeABI) llvm.Value { - fn := mod.NamedFunction(frameFreeName) - if fn.IsNil() { - fn = llvm.AddFunction(mod, frameFreeName, llvm.FunctionType( - abi.ctx.VoidType(), []llvm.Type{abi.ptr, abi.ptr}, false, - )) - } - return fn -} - func declareCompatEnter(mod llvm.Module, abi resumeABI) llvm.Value { fn := mod.NamedFunction(compatEnterName) if fn.IsNil() { diff --git a/internal/wasmresume/state_test.go b/internal/wasmresume/state_test.go index 44c36b0075..5135121359 100644 --- a/internal/wasmresume/state_test.go +++ b/internal/wasmresume/state_test.go @@ -166,13 +166,15 @@ func TestLowerPrototypeBuildsDirectCallStateMachine(t *testing.T) { `switch i32 %pc, label %invalid-pc [`, `i32 0, label %entry`, `i32 1, label %resume.1`, - `call ptr @__llgo_wasm_resume_alloc(ptr %0,`, + `define internal ptr @__llgo_wasm_resume_alloc.fast`, + `call ptr @__llgo_wasm_resume_alloc.fast(ptr %0,`, `call void @llvm.memset`, `call ptr @__llgo_wasm_resume_compat_enter(`, `call void @__llgo_wasm_resume_compat_leave(`, `ret i8 0`, `%returned = load ptr`, - `call void @__llgo_wasm_resume_free(ptr %0, ptr %returned)`, + `define internal void @__llgo_wasm_resume_free.fast`, + `call void @__llgo_wasm_resume_free.fast(ptr %0, ptr %returned)`, `ret i8 1`, } { if !strings.Contains(ir, want) { @@ -372,6 +374,9 @@ func defineStateMachineHarness( childStorage.SetAlignment(16) childOffset := llvm.AddGlobal(mod, i32, "child.offset") childOffset.SetInitializer(llvm.ConstInt(i32, 0, false)) + storageBlockType := frameBlockType(abi) + storageBlock := llvm.AddGlobal(mod, storageBlockType, "child.storage.block") + storageBlock.SetInitializer(llvm.ConstNull(storageBlockType)) alloc := mod.NamedFunction(frameAllocName) block := ctx.AddBasicBlock(alloc, "entry") @@ -415,6 +420,28 @@ func defineStateMachineHarness( failedBlock := ctx.AddBasicBlock(run, "failed") builder.SetInsertPointAtEnd(entryBlock) + storageBegin := builder.CreatePtrToInt(childStorage, abi.uintptrType, "storage.begin") + builder.CreateStore( + llvm.ConstNull(abi.ptr), builder.CreateStructGEP(storageBlockType, storageBlock, 0, ""), + ) + builder.CreateStore( + llvm.ConstNull(abi.ptr), builder.CreateStructGEP(storageBlockType, storageBlock, 1, ""), + ) + builder.CreateStore( + storageBegin, builder.CreateStructGEP(storageBlockType, storageBlock, 2, ""), + ) + builder.CreateStore( + builder.CreateAdd( + storageBegin, llvm.ConstInt(abi.uintptrType, 4096, false), "storage.end", + ), + builder.CreateStructGEP(storageBlockType, storageBlock, 3, ""), + ) + builder.CreateStore( + storageBegin, builder.CreateStructGEP(storageBlockType, storageBlock, 4, ""), + ) + builder.CreateStore( + storageBlock, builder.CreateStructGEP(abi.contextType, context, 2, ""), + ) builder.CreateStore( llvm.ConstNull(abi.ptr), builder.CreateStructGEP(lowered.layout.typ, root, 0, ""), diff --git a/runtime/internal/wasmresume/storage_test.go b/runtime/internal/wasmresume/storage_test.go index f092ccd19a..aa74f60e64 100644 --- a/runtime/internal/wasmresume/storage_test.go +++ b/runtime/internal/wasmresume/storage_test.go @@ -16,6 +16,28 @@ type testUnwindFrame struct { deferFrame unsafe.Pointer } +func TestFrameArenaABILayout(t *testing.T) { + pointerSize := unsafe.Sizeof(uintptr(0)) + if got, want := unsafe.Offsetof(Context{}.storage), 2*pointerSize; got != want { + t.Fatalf("Context storage offset = %d, want %d", got, want) + } + if got, want := unsafe.Sizeof(frameBlock{}), 5*pointerSize; got != want { + t.Fatalf("frameBlock size = %d, want %d", got, want) + } + offsets := [...]uintptr{ + unsafe.Offsetof(frameBlock{}.prev), + unsafe.Offsetof(frameBlock{}.next), + unsafe.Offsetof(frameBlock{}.begin), + unsafe.Offsetof(frameBlock{}.end), + unsafe.Offsetof(frameBlock{}.stackPointer), + } + for field, got := range offsets { + if want := uintptr(field) * pointerSize; got != want { + t.Fatalf("frameBlock field %d offset = %d, want %d", field, got, want) + } + } +} + func (r *testFrameRoots) allocate(size uintptr) unsafe.Pointer { block := make([]byte, size) if len(block) == 0 {