diff --git a/code.go b/code.go index 96fcf6fc..53ab454d 100644 --- a/code.go +++ b/code.go @@ -443,6 +443,7 @@ func (t *translator) readCodeForFunction(fn *funcCompiler) error { fn.pushConst(r) } } + fn.memMayGrow() } case 0x0f: // return @@ -720,7 +721,7 @@ func (t *translator) readCodeForFunction(fn *funcCompiler) error { &ast.BinaryExpr{ X: &ast.CallExpr{ Fun: newID("len"), - Args: []ast.Expr{t.memory.selector}}, + Args: []ast.Expr{fn.mem()}}, Op: token.SHR, Y: literal16, }, t.memory.stype())) @@ -732,6 +733,7 @@ func (t *translator) readCodeForFunction(fn *funcCompiler) error { Sel: newID("Grow")}, Args: []ast.Expr{literal0, newID("maxMem")}}, t.memory.stype())) + fn.memMayGrow() default: needsUnsafe("shared memory") @@ -742,6 +744,7 @@ func (t *translator) readCodeForFunction(fn *funcCompiler) error { &ast.UnaryExpr{Op: token.AND, X: t.memory.selector}, literal0, newID("maxMem")}}, t.memory.stype())) + fn.memMayGrow() } case 0x40: // memory.grow @@ -773,6 +776,7 @@ func (t *translator) readCodeForFunction(fn *funcCompiler) error { newID("maxMem")}}, t.memory.stype())) } + fn.memMayGrow() case 0x41: // i32.const e, err := t.constI32() diff --git a/code_atomic.go b/code_atomic.go index bdef4e6c..ee2f09c1 100644 --- a/code_atomic.go +++ b/code_atomic.go @@ -16,6 +16,7 @@ func (t *translator) readOpcodeAtomic(fn *funcCompiler) error { _, err = t.in.ReadByte() // 0x00 fn.helpers.add("atomic_fence") fn.emit(&ast.ExprStmt{X: &ast.CallExpr{Fun: newID("atomic_fence")}}) + fn.memMayGrow() return nil } @@ -170,6 +171,7 @@ func (t *translator) readOpcodeAtomic(fn *funcCompiler) error { default: return fmt.Errorf("unsupported opcode (atomic): 0xFE 0x%02X", code) } + fn.memMayGrow() return nil } @@ -179,7 +181,7 @@ func (fn *funcCompiler) atomicLoad(typ, name string, offset uint64) ast.Expr { fn.helpers.add(name) return convert(&ast.CallExpr{ Fun: newID(name), - Args: []ast.Expr{fn.memory.selector, addr}, + Args: []ast.Expr{fn.mem(), addr}, }, typ) } @@ -193,7 +195,7 @@ func (fn *funcCompiler) atomicStore(name string, offset uint64) ast.Stmt { fn.helpers.add("atomic_ptr" + bits) return &ast.ExprStmt{X: &ast.CallExpr{ Fun: newID(name), - Args: []ast.Expr{fn.memory.selector, addr, val}, + Args: []ast.Expr{fn.mem(), addr, val}, }} } @@ -207,7 +209,7 @@ func (fn *funcCompiler) atomicRmw(typ, name string, offset uint64) ast.Expr { fn.helpers.add("atomic_ptr" + bits) return convert(&ast.CallExpr{ Fun: newID(name), - Args: []ast.Expr{fn.memory.selector, addr, val}, + Args: []ast.Expr{fn.mem(), addr, val}, }, typ) } @@ -223,7 +225,7 @@ func (fn *funcCompiler) atomicCmpxchg(typ, name string, offset uint64) ast.Expr fn.helpers.add("atomic_ptr" + bits) return convert(&ast.CallExpr{ Fun: newID(name), - Args: []ast.Expr{fn.memory.selector, addr, old, new}, + Args: []ast.Expr{fn.mem(), addr, old, new}, }, typ) } @@ -237,7 +239,7 @@ func (fn *funcCompiler) atomicNotify(name string, offset uint64) ast.Expr { return &ast.CallExpr{ Fun: newID(name), Args: []ast.Expr{ - fn.memory.selector, addr, count, + fn.mem(), addr, count, modSel(newID("waiters"))}} } @@ -255,6 +257,6 @@ func (fn *funcCompiler) atomicWait(name string, offset uint64) ast.Expr { return &ast.CallExpr{ Fun: newID(name), Args: []ast.Expr{ - fn.memory.selector, addr, exp, timeout, + fn.mem(), addr, exp, timeout, modSel(newID("waiters"))}} } diff --git a/code_ext.go b/code_ext.go index d3f55253..6675a8c0 100644 --- a/code_ext.go +++ b/code_ext.go @@ -47,7 +47,7 @@ func (t *translator) readOpcodeExtended(fn *funcCompiler) error { X: &ast.CallExpr{ Fun: newID("memory_init"), Args: []ast.Expr{ - t.memory.selector, + fn.mem(), t.dataExpr(int(i)), dst, src, n}}}) case 0x09: // data.drop @@ -75,7 +75,7 @@ func (t *translator) readOpcodeExtended(fn *funcCompiler) error { X: &ast.CallExpr{ Fun: newID("memory_copy"), Args: []ast.Expr{ - t.memory.selector, + fn.mem(), dst, src, n}}}) case 0x0b: // memory.fill @@ -93,7 +93,7 @@ func (t *translator) readOpcodeExtended(fn *funcCompiler) error { X: &ast.CallExpr{ Fun: newID("memory_zero"), Args: []ast.Expr{ - t.memory.selector, + fn.mem(), dest, n}}}) } else { fn.helpers.add("memory_fill") @@ -101,7 +101,7 @@ func (t *translator) readOpcodeExtended(fn *funcCompiler) error { X: &ast.CallExpr{ Fun: newID("memory_fill"), Args: []ast.Expr{ - t.memory.selector, + fn.mem(), dest, val, n}}}) } diff --git a/fncompiler.go b/fncompiler.go index 3096dd8f..15da859c 100644 --- a/fncompiler.go +++ b/fncompiler.go @@ -3,6 +3,7 @@ package main import ( "go/ast" "go/token" + "slices" "strconv" "strings" @@ -21,6 +22,12 @@ type funcCompiler struct { labels int temps int + memSlice *ast.ParenExpr + memReloads []memReload + memReads int + memFirstTop int + memLastTop int + provided bool } @@ -132,7 +139,7 @@ func (fn *funcCompiler) branch(n uint64) (stmts []ast.Stmt) { // Returns an expression that loads a byte from memory (an l-value). func (fn *funcCompiler) load8(offset uint64) ast.Expr { return &ast.IndexExpr{ - X: fn.memory.selector, + X: fn.mem(), Index: fn.popAddr(offset)} } @@ -145,7 +152,7 @@ func (fn *funcCompiler) load(typ string, offset uint64) (expr ast.Expr) { fn.helpers.add("load" + bits) expr = &ast.CallExpr{ Fun: newID("load" + bits), - Args: []ast.Expr{fn.memory.selector, addr}} + Args: []ast.Expr{fn.mem(), addr}} switch { case strings.HasPrefix(typ, "float"): @@ -180,7 +187,7 @@ func (fn *funcCompiler) store(typ string, offset uint64) ast.Stmt { fn.helpers.add("store" + bits) return &ast.ExprStmt{X: &ast.CallExpr{ Fun: newID("store" + bits), - Args: []ast.Expr{fn.memory.selector, addr, val}}} + Args: []ast.Expr{fn.mem(), addr, val}}} } // Pushes expr (a literal, constant or materialized temporary) to the value stack. @@ -597,7 +604,110 @@ func (fn *funcCompiler) newLabel() *ast.Ident { return id } +const loopWeight = 8 + +func (fn *funcCompiler) inLoop() bool { + for i := range fn.blocks { + if fn.blocks[i].loopPos != 0 { + return true + } + } + return false +} + +type memReload struct { + body *ast.BlockStmt + pos int + topIdx int + prevReads int + inLoop bool +} + +func (fn *funcCompiler) mem() ast.Expr { + if *noopt || fn.blocks.top().unreachable { + return fn.memory.selector + } + if fn.memSlice == nil { + fn.memSlice = &ast.ParenExpr{X: fn.memory.selector} + } + topIdx := len(fn.blocks[0].body.List) + if len(fn.blocks) > 1 { + topIdx-- + } + if fn.memReads == 0 { + fn.memFirstTop = topIdx + } + if fn.inLoop() { + fn.memReads += loopWeight + } else { + fn.memReads += 1 + } + fn.memLastTop = topIdx + return fn.memSlice +} + +func (fn *funcCompiler) memMayGrow() { + blk := fn.blocks.top() + if *noopt || fn.memory == nil || blk.unreachable { + return + } + fn.memReloads = append(fn.memReloads, memReload{ + body: blk.body, + pos: len(blk.body.List), + topIdx: len(fn.blocks[0].body.List) - 1, + prevReads: fn.memReads, + inLoop: fn.inLoop(), + }) +} + +func (fn *funcCompiler) memFinish() { + if fn.memReads == 0 { + return + } + + cost := 1 + var reloads []memReload + for _, r := range fn.memReloads { + // Skip reloads before the definition, and after the last use: + // only a loop can reach a use that comes earlier. + if fn.memFirstTop <= r.topIdx && r.topIdx <= fn.memLastTop && + (r.inLoop || r.prevReads < fn.memReads) { + if r.inLoop { + cost += loopWeight + } else { + cost += 1 + } + reloads = append(reloads, r) + } + } + if cost >= fn.memReads { + return + } + + mem := newID("mem") + fn.memSlice.X = mem + + // Backwards, so positions stay valid. + for _, r := range slices.Backward(reloads) { + r.body.List = slices.Insert(r.body.List, r.pos, ast.Stmt(&ast.AssignStmt{ + Tok: token.ASSIGN, + Lhs: []ast.Expr{mem}, + Rhs: []ast.Expr{fn.memory.selector}, + })) + } + + // Define at the top level, before the statement with the first use. + top := fn.blocks[0].body + top.List = slices.Insert(top.List, fn.memFirstTop, ast.Stmt(&ast.AssignStmt{ + Tok: token.DEFINE, + Lhs: []ast.Expr{mem}, + Rhs: []ast.Expr{fn.memory.selector}, + })) +} + func (fn *funcCompiler) cleanup() { + fn.memFinish() + // Sanity checks. passes.CheckMaterialized(fn.decl) diff --git a/ids.go b/ids.go index c360fc41..efba8c0f 100644 --- a/ids.go +++ b/ids.go @@ -27,6 +27,7 @@ import ( // // Method code may have: // - module selector (m) +// - local memory slice (mem) // - local variables, arguments (v+number) // - materialized temporaries (t+number) // - temporary variables (p+number) diff --git a/regression_test.go b/regression_test.go index 72e10d06..d5fa9225 100644 --- a/regression_test.go +++ b/regression_test.go @@ -41,6 +41,12 @@ func Test_regression_store_grow(t *testing.T) { if got := m.Xsize(); got != 2 { t.Errorf("size = %d, want 2", got) } + if got := m.Xgrow_mid(); got != 12345 { + t.Errorf("grow_mid() = %d, want 12345 (local mem must reload after memory.grow)", got) + } + if got := m.Xsize(); got != 3 { + t.Errorf("size = %d, want 3", got) + } } func Test_regression_oob_trap(t *testing.T) { diff --git a/testdata/loops/loops.go b/testdata/loops/loops.go index 5c8f6882..0de3f826 100644 --- a/testdata/loops/loops.go +++ b/testdata/loops/loops.go @@ -44,13 +44,14 @@ func (m *Module) fn1() int32 { func (m *Module) Xadd_all(v0, v1 int32) int32 { var v2, v3, v4 int32 v2 = i32(0) + mem := *m.memory l1: { if v2 >= v1 { return v4 } v3 = v0 + v2*i32(4) - t0 := int32(load32(*m.memory, uint32(v3))) + t0 := int32(load32(mem, uint32(v3))) v4 = v4 + t0 v2 = v2 + i32(1) goto l1 diff --git a/testdata/regression/store_grow/store_grow.go b/testdata/regression/store_grow/store_grow.go index af060001..1c70d2ff 100644 --- a/testdata/regression/store_grow/store_grow.go +++ b/testdata/regression/store_grow/store_grow.go @@ -30,10 +30,22 @@ func (m *Module) fn0() int32 { } func (m *Module) Xtest() int32 { t0 := m.fn0() - store32(m.memory, uint32(i32(0)), uint32(t0)) - t1 := int32(load32(m.memory, uint32(i32(0)))) + mem := m.memory + store32(mem, uint32(i32(0)), uint32(t0)) + t1 := int32(load32(mem, uint32(i32(0)))) return t1 } +func (m *Module) Xgrow_mid() int32 { + var v0 int32 + mem := m.memory + t0 := int32(len(mem) >> 16) + v0 = t0 << 16 + t1 := m.fn0() + mem = m.memory + store32(mem, uint32(v0), uint32(t1)) + t2 := int32(load32(mem, uint32(v0))) + return t2 +} func (m *Module) Xsize() int32 { t0 := int32(len(m.memory) >> 16) return t0 diff --git a/testdata/regression/store_grow/store_grow.wasm b/testdata/regression/store_grow/store_grow.wasm index 08ff5aea..b755236f 100644 Binary files a/testdata/regression/store_grow/store_grow.wasm and b/testdata/regression/store_grow/store_grow.wasm differ diff --git a/testdata/regression/store_grow/store_grow.wat b/testdata/regression/store_grow/store_grow.wat index 545aa84d..a305e48a 100644 --- a/testdata/regression/store_grow/store_grow.wat +++ b/testdata/regression/store_grow/store_grow.wat @@ -6,5 +6,9 @@ (func (export "test") (result i32) (i32.store (i32.const 0) (call $alloc)) (i32.load (i32.const 0))) + (func (export "grow_mid") (result i32) (local $off i32) + (local.set $off (i32.shl (memory.size) (i32.const 16))) + (i32.store (local.get $off) (call $alloc)) + (i32.load (local.get $off))) (func (export "size") (result i32) (memory.size)))