From 1eb0df37e683063bcf0a825499d298fb76f84c9e Mon Sep 17 00:00:00 2001 From: Abmcar Date: Fri, 10 Apr 2026 02:09:15 +0800 Subject: [PATCH 1/9] perf(evm): replace udiv/urem with ushr/and in dynamic shift lowering Replace OP_udiv(ShiftAmount, 64) with OP_ushr(ShiftAmount, 6) and OP_urem(ShiftAmount, 64) with OP_and(ShiftAmount, 63) in SHL/SHR/SAR dynamic shift paths. Since the divisor is always 64 (2^6), these strength reductions avoid expensive x86 DIV instructions (~30 cycles). Co-Authored-By: Claude Opus 4.6 (1M context) --- .../evm_frontend/evm_mir_compiler.cpp | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/compiler/evm_frontend/evm_mir_compiler.cpp b/src/compiler/evm_frontend/evm_mir_compiler.cpp index fa748f4c3..7c5a3a656 100644 --- a/src/compiler/evm_frontend/evm_mir_compiler.cpp +++ b/src/compiler/evm_frontend/evm_mir_compiler.cpp @@ -2816,10 +2816,13 @@ EVMMirBuilder::handleLeftShift(const U256Inst &Value, MInstruction *ShiftAmount, // shift_mod = shift % 64 (shift amount within 64-bit range) // shift_comp = shift / 64 (which component index shift from) // remaining_bits = 64 - shift_mod (remaining bits for carry calculation) + // Strength-reduce: shift % 64 == shift & 63, shift / 64 == shift >> 6 + MInstruction *Const63 = createIntConstInstruction(MirI64Type, 63); + MInstruction *Const6 = createIntConstInstruction(MirI64Type, 6); MInstruction *ShiftMod64 = createInstruction( - false, OP_urem, MirI64Type, ShiftAmount, Const64); + false, OP_and, MirI64Type, ShiftAmount, Const63); MInstruction *ComponentShift = createInstruction( - false, OP_udiv, MirI64Type, ShiftAmount, Const64); + false, OP_ushr, MirI64Type, ShiftAmount, Const6); MInstruction *RemainingBits = createInstruction( false, OP_sub, MirI64Type, Const64, ShiftMod64); @@ -3009,10 +3012,13 @@ EVMMirBuilder::handleLogicalRightShift(const U256Inst &Value, // DMIR implementation maps 256-bit shift to 4x64-bit components // shift_mod = shift % 64 (shift amount within 64-bit range) // shift_comp = shift / 64 (which component index shift from) + // Strength-reduce: shift % 64 == shift & 63, shift / 64 == shift >> 6 + MInstruction *Const63 = createIntConstInstruction(MirI64Type, 63); + MInstruction *Const6 = createIntConstInstruction(MirI64Type, 6); MInstruction *ShiftMod64 = createInstruction( - false, OP_urem, MirI64Type, ShiftAmount, Const64); + false, OP_and, MirI64Type, ShiftAmount, Const63); MInstruction *ComponentShift = createInstruction( - false, OP_udiv, MirI64Type, ShiftAmount, Const64); + false, OP_ushr, MirI64Type, ShiftAmount, Const6); MInstruction *MaxIndex = createIntConstInstruction(MirI64Type, EVM_ELEMENTS_COUNT); @@ -3206,11 +3212,13 @@ EVMMirBuilder::handleArithmeticRightShift(const U256Inst &Value, // intra-component shifts = shift % 64 // shift_comp = shift / 64 (which component index shift from) + // Strength-reduce: shift % 64 == shift & 63, shift / 64 == shift >> 6 MInstruction *Const64 = createIntConstInstruction(MirI64Type, 64); + MInstruction *Const6 = createIntConstInstruction(MirI64Type, 6); MInstruction *ShiftMod64 = createInstruction( - false, OP_urem, MirI64Type, ShiftAmount, Const64); + false, OP_and, MirI64Type, ShiftAmount, Const63); MInstruction *ComponentShift = createInstruction( - false, OP_udiv, MirI64Type, ShiftAmount, Const64); + false, OP_ushr, MirI64Type, ShiftAmount, Const6); MInstruction *MaxIndex = createIntConstInstruction(MirI64Type, EVM_ELEMENTS_COUNT); From ab690509f9fc3b34b0aa5dfafa85bd7fc06a6d7e Mon Sep 17 00:00:00 2001 From: Abmcar Date: Fri, 10 Apr 2026 02:36:53 +0800 Subject: [PATCH 2/9] perf(evm): inline ADDMOD fast path to eliminate runtime call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inline the intx::addmod fast-path algorithm at the MIR level for non-constant ADDMOD operations. When mod[3] != 0 (modulus >= 2^192) and both operands' high limbs are <= mod's high limb, the result is computed entirely in JIT-generated code using conditional normalize, add, and subtract-and-select — avoiding the runtime function call to evmGetAddMod/intx::addmod. The fast path performs: 1. Normalize augend: if augend >= mod, use augend - mod 2. Normalize addend: if addend >= mod, use addend - mod 3. Add normalized values, detect overflow via sum < augend 4. Subtract mod from sum, detect borrow via sum < mod 5. Select result: if (overflow || !borrow) use difference, else sum Falls back to the existing runtime call for small moduli (mod[3] == 0) and the mod == 0 edge case. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../evm_frontend/evm_mir_compiler.cpp | 231 +++++++++++++++++- 1 file changed, 227 insertions(+), 4 deletions(-) diff --git a/src/compiler/evm_frontend/evm_mir_compiler.cpp b/src/compiler/evm_frontend/evm_mir_compiler.cpp index 7c5a3a656..705b7c80f 100644 --- a/src/compiler/evm_frontend/evm_mir_compiler.cpp +++ b/src/compiler/evm_frontend/evm_mir_compiler.cpp @@ -2101,10 +2101,233 @@ typename EVMMirBuilder::Operand EVMMirBuilder::handleAddMod(Operand AugendOp, return Operand(intxToU256Value(Result)); } - const auto &RuntimeFunctions = getRuntimeFunctionTable(); - return callRuntimeFor( - RuntimeFunctions.GetAddMod, AugendOp, AddendOp, ModulusOp); + MType *I64Type = EVMFrontendContext::getMIRTypeFromEVMType(EVMType::UINT64); + MInstruction *Zero = createIntConstInstruction(I64Type, 0); + + U256Inst Augend = extractU256Operand(AugendOp); + U256Inst Addend = extractU256Operand(AddendOp); + U256Inst Modulus = extractU256Operand(ModulusOp); + + // Fast-path eligibility: mod[3] != 0 && x[3] <= mod[3] && y[3] <= mod[3] + // Note: mod[3] != 0 implies mod != 0, so the mod == 0 case is handled + // by the slow path (runtime call) which returns 0 for zero modulus. + MInstruction *ModHi = Modulus[3]; + MInstruction *ModHiNonZero = createInstruction( + false, CmpInstruction::ICMP_NE, I64Type, ModHi, Zero); + MInstruction *AugendHiLE = createInstruction( + false, CmpInstruction::ICMP_ULE, I64Type, Augend[3], ModHi); + MInstruction *AddendHiLE = createInstruction( + false, CmpInstruction::ICMP_ULE, I64Type, Addend[3], ModHi); + + // FastEligible = ModHiNonZero && AugendHiLE && AddendHiLE + MInstruction *FastEligible = createInstruction( + false, OP_and, I64Type, ModHiNonZero, AugendHiLE); + FastEligible = createInstruction(false, OP_and, I64Type, + FastEligible, AddendHiLE); + + // Prepare result variables + U256Var ResultVars = {}; + for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) { + ResultVars[I] = CurFunc->createVariable(I64Type); + } + + auto storeResult = [&](const U256Inst &Values) { + for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) { + createInstruction(true, &(Ctx.VoidType), Values[I], + ResultVars[I]->getVarIdx()); + } + }; + + auto loadResult = [&]() -> U256Inst { + U256Inst Values = {}; + for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) { + Values[I] = loadVariable(ResultVars[I]); + } + return Values; + }; + + // Branch: FastEligible ? FastBB : SlowBB + MBasicBlock *FastBB = createBasicBlock(); + MBasicBlock *SlowBB = createBasicBlock(); + MBasicBlock *AfterBB = createBasicBlock(); + createInstruction(true, Ctx, FastEligible, FastBB, SlowBB); + addSuccessor(FastBB); + addSuccessor(SlowBB); + + // === Fast path (inline addmod) === + setInsertBlock(FastBB); + { + // Step 1: Normalize augend — if augend >= mod, use augend - mod + // Compute augend < mod using cascaded limb comparison + MInstruction *AugLtMod = nullptr; + { + // u256 LT comparison: augend < modulus (high-to-low cascade) + MInstruction *CmpResult = nullptr; + MInstruction *AllEqual = nullptr; + for (int I = EVM_ELEMENTS_COUNT - 1; I >= 0; --I) { + MInstruction *LT = createInstruction( + false, CmpInstruction::ICMP_ULT, I64Type, Augend[I], Modulus[I]); + MInstruction *EQ = createInstruction( + false, CmpInstruction::ICMP_EQ, I64Type, Augend[I], Modulus[I]); + if (CmpResult == nullptr) { + CmpResult = LT; + AllEqual = EQ; + } else { + CmpResult = createInstruction( + false, I64Type, AllEqual, LT, CmpResult); + AllEqual = createInstruction( + false, OP_and, I64Type, AllEqual, EQ); + } + } + AugLtMod = CmpResult; + } + + // Compute augend - mod (u256 SUB chain) + Operand AugSubMod = + handleBinaryArithmetic(AugendOp, ModulusOp); + U256Inst AugDiff = extractU256Operand(AugSubMod); + + // NormAugend = AugLtMod ? Augend : AugDiff (if augend < mod, keep augend) + U256Inst NormAugend = {}; + for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) { + NormAugend[I] = createInstruction( + false, I64Type, AugLtMod, Augend[I], AugDiff[I]); + } + + // Step 2: Normalize addend — if addend >= mod, use addend - mod + MInstruction *AddLtMod = nullptr; + { + MInstruction *CmpResult = nullptr; + MInstruction *AllEqual = nullptr; + for (int I = EVM_ELEMENTS_COUNT - 1; I >= 0; --I) { + MInstruction *LT = createInstruction( + false, CmpInstruction::ICMP_ULT, I64Type, Addend[I], Modulus[I]); + MInstruction *EQ = createInstruction( + false, CmpInstruction::ICMP_EQ, I64Type, Addend[I], Modulus[I]); + if (CmpResult == nullptr) { + CmpResult = LT; + AllEqual = EQ; + } else { + CmpResult = createInstruction( + false, I64Type, AllEqual, LT, CmpResult); + AllEqual = createInstruction( + false, OP_and, I64Type, AllEqual, EQ); + } + } + AddLtMod = CmpResult; + } + + Operand AddSubMod = + handleBinaryArithmetic(AddendOp, ModulusOp); + U256Inst AddDiff = extractU256Operand(AddSubMod); + + // NormAddend = AddLtMod ? Addend : AddDiff + U256Inst NormAddend = {}; + for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) { + NormAddend[I] = createInstruction( + false, I64Type, AddLtMod, Addend[I], AddDiff[I]); + } + + // Step 3: Sum = NormAugend + NormAddend (u256 ADD chain) + Operand NormAugendOp(NormAugend, EVMType::UINT256); + Operand NormAddendOp(NormAddend, EVMType::UINT256); + Operand SumOp = handleBinaryArithmetic( + NormAugendOp, NormAddendOp); + U256Inst Sum = extractU256Operand(SumOp); + + // Detect overflow: sum < NormAugend (unsigned comparison) + MInstruction *Overflow = nullptr; + { + MInstruction *CmpResult = nullptr; + MInstruction *AllEqual = nullptr; + for (int I = EVM_ELEMENTS_COUNT - 1; I >= 0; --I) { + MInstruction *LT = createInstruction( + false, CmpInstruction::ICMP_ULT, I64Type, Sum[I], NormAugend[I]); + MInstruction *EQ = createInstruction( + false, CmpInstruction::ICMP_EQ, I64Type, Sum[I], NormAugend[I]); + if (CmpResult == nullptr) { + CmpResult = LT; + AllEqual = EQ; + } else { + CmpResult = createInstruction( + false, I64Type, AllEqual, LT, CmpResult); + AllEqual = createInstruction( + false, OP_and, I64Type, AllEqual, EQ); + } + } + Overflow = CmpResult; + } + + // Step 4: SumSubMod = Sum - Mod (u256 SUB chain) + Operand ModulusOpLocal(Modulus, EVMType::UINT256); + Operand SumSubModOp = + handleBinaryArithmetic(SumOp, ModulusOpLocal); + U256Inst SumSubMod = extractU256Operand(SumSubModOp); + + // Detect borrow: Sum < Mod + MInstruction *SumLtMod = nullptr; + { + MInstruction *CmpResult = nullptr; + MInstruction *AllEqual = nullptr; + for (int I = EVM_ELEMENTS_COUNT - 1; I >= 0; --I) { + MInstruction *LT = createInstruction( + false, CmpInstruction::ICMP_ULT, I64Type, Sum[I], Modulus[I]); + MInstruction *EQ = createInstruction( + false, CmpInstruction::ICMP_EQ, I64Type, Sum[I], Modulus[I]); + if (CmpResult == nullptr) { + CmpResult = LT; + AllEqual = EQ; + } else { + CmpResult = createInstruction( + false, I64Type, AllEqual, LT, CmpResult); + AllEqual = createInstruction( + false, OP_and, I64Type, AllEqual, EQ); + } + } + SumLtMod = CmpResult; + } + + // Result selection: if (overflow || !borrow) use SumSubMod, else Sum + // !borrow means Sum >= Mod, i.e., !SumLtMod + // overflow || !SumLtMod => use SumSubMod + // + // Equivalently: use Sum only when (!overflow && SumLtMod) + // UseSumSubMod = Overflow || !SumLtMod + // Since SumLtMod is 0 or 1: !SumLtMod = (SumLtMod == 0) + MInstruction *One = createIntConstInstruction(I64Type, 1); + MInstruction *NotBorrow = createInstruction( + false, OP_xor, I64Type, SumLtMod, One); + MInstruction *UseSumSubMod = createInstruction( + false, OP_or, I64Type, Overflow, NotBorrow); + + U256Inst FastResult = {}; + for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) { + FastResult[I] = createInstruction( + false, I64Type, UseSumSubMod, SumSubMod[I], Sum[I]); + } + + storeResult(FastResult); + createInstruction(true, Ctx, AfterBB); + addSuccessor(AfterBB); + } + + // === Slow path (runtime call, handles mod == 0 and small moduli) === + setInsertBlock(SlowBB); + { + // Runtime handles all cases including mod == 0. + const auto &RuntimeFunctions = getRuntimeFunctionTable(); + U256Inst SlowResult = extractU256Operand( + callRuntimeFor( + RuntimeFunctions.GetAddMod, AugendOp, AddendOp, ModulusOp)); + storeResult(SlowResult); + createInstruction(true, Ctx, AfterBB); + addSuccessor(AfterBB); + } + + // === After block: load result === + setInsertBlock(AfterBB); + return Operand(loadResult(), EVMType::UINT256); } typename EVMMirBuilder::Operand From df7d5c5b271577bab7657ce3bd6f8c7098c78347 Mon Sep 17 00:00:00 2001 From: Abmcar Date: Fri, 10 Apr 2026 00:36:51 +0800 Subject: [PATCH 3/9] perf(evm): eliminate dead-carry protectUnsafeValue barriers in u64 fast paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove protectUnsafeValue barriers in handleAddU64Const and handleSubU64Const where the x86 carry flag is provably dead: - handleAddU64Const: Remove barriers on RHS constants (MOV imm never clobbers flags) and on the last ADC result (no subsequent ADC consumes the carry). Intermediate ADC barriers are preserved to force immediate execution while CF is live between ADD→ADC→ADC. - handleSubU64Const: Remove all barriers. This function uses explicit borrow computation (SUB + ICMP_ULT + zeroExtend) rather than SBB instructions, so the carry flag is never live. Generic u256 ADD/SUB chains (evm_mir_compiler.h) are intentionally left unchanged: their last ADC/SBB barriers ensure the instruction executes while CF from the previous ADC/SBB is still valid, and removing them causes deferred lowering to execute after CF is clobbered. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../evm_frontend/evm_mir_compiler.cpp | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/compiler/evm_frontend/evm_mir_compiler.cpp b/src/compiler/evm_frontend/evm_mir_compiler.cpp index 705b7c80f..bf5a049db 100644 --- a/src/compiler/evm_frontend/evm_mir_compiler.cpp +++ b/src/compiler/evm_frontend/evm_mir_compiler.cpp @@ -2791,20 +2791,25 @@ EVMMirBuilder::handleAddU64Const(const Operand &FullOp, for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) { LHS[I] = protectUnsafeValue(LHS[I], MirI64Type); } - RHS0 = protectUnsafeValue(RHS0, MirI64Type); - MInstruction *ProtectedZero = protectUnsafeValue(RHSZero, MirI64Type); + // RHS constants (immediate MOV) never clobber flags — no barrier needed. U256Inst Result = {}; // Limb 0: ADD with the actual u64 value Result[0] = protectUnsafeValue(createInstruction( false, OP_add, MirI64Type, LHS[0], RHS0), MirI64Type); - // Limbs 1-3: ADC with shared zero (carry propagation only) + // Limbs 1-3: ADC with zero (carry propagation only) for (size_t I = 1; I < EVM_ELEMENTS_COUNT; ++I) { - Result[I] = - protectUnsafeValue(createInstruction( - false, MirI64Type, LHS[I], ProtectedZero, Carry), - MirI64Type); + MInstruction *AdcInst = createInstruction( + false, MirI64Type, LHS[I], RHSZero, Carry); + // Last ADC: carry flag is dead after this point, no subsequent ADC + // consumes it. The barrier is only needed for intermediate ADCs to + // force immediate execution while CF is live. + if (I < EVM_ELEMENTS_COUNT - 1) { + Result[I] = protectUnsafeValue(AdcInst, MirI64Type); + } else { + Result[I] = AdcInst; + } } return Operand(Result, EVMType::UINT256); } @@ -2829,13 +2834,15 @@ EVMMirBuilder::handleSubU64Const(const Operand &LHSOp, Borrow = zeroExtendToI64(Borrow); U256Inst Result = {}; - Result[0] = protectUnsafeValue(Diff0, MirI64Type); + // No SBB instructions used — explicit borrow computation means carry flag + // is never live, so no protectUnsafeValue barriers are needed. + Result[0] = Diff0; // Limbs 1-3: only subtract the borrow (RHS is 0 for upper limbs) for (size_t I = 1; I < EVM_ELEMENTS_COUNT; ++I) { MInstruction *Diff = createInstruction( false, OP_sub, MirI64Type, LHS[I], Borrow); - Result[I] = protectUnsafeValue(Diff, MirI64Type); + Result[I] = Diff; if (I < EVM_ELEMENTS_COUNT - 1) { // New borrow: LHS[I] < Borrow From 718dab0b649a013ebdb2b42c378f71fd8e1d75b6 Mon Sep 17 00:00:00 2001 From: Abmcar Date: Fri, 10 Apr 2026 21:13:11 +0800 Subject: [PATCH 4/9] perf(evm): add value-range narrowing for u256 arithmetic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add lightweight ValueRange tracking (U64/U128/U256) to Operand class to detect when u256 operands provably fit in narrower types, then emit single-instruction fast paths instead of expensive multi-limb arithmetic. Range sources: - Constants: auto-derived from limb values - AND masks: result narrows to the smaller operand range - BYTE: always U64 (single byte value 0..255) - Comparisons: always U64 (result is 0 or 1) Range-based fast paths: - ADD(U64,U64): single add + carry → U128 result - MUL(U64,U64): single 64×64→128 multiply → U128 result - DIV(U64,U64): single udiv with div-by-zero guard → U64 result - MOD(U64,U64): single urem with div-by-zero guard → U64 result Co-Authored-By: Claude Opus 4.6 (1M context) --- .../evm_frontend/evm_mir_compiler.cpp | 73 ++++++++++++++- src/compiler/evm_frontend/evm_mir_compiler.h | 89 ++++++++++++++++++- 2 files changed, 155 insertions(+), 7 deletions(-) diff --git a/src/compiler/evm_frontend/evm_mir_compiler.cpp b/src/compiler/evm_frontend/evm_mir_compiler.cpp index bf5a049db..02a83388f 100644 --- a/src/compiler/evm_frontend/evm_mir_compiler.cpp +++ b/src/compiler/evm_frontend/evm_mir_compiler.cpp @@ -1754,6 +1754,21 @@ typename EVMMirBuilder::Operand EVMMirBuilder::handleMul(Operand MultiplicandOp, return MultiplicandOp; } + // Phase 1: Range-based u64×u64 → u128 fast path + // When both operands provably fit in u64, a single 64×64→128 multiply + // replaces the expensive 4×4 schoolbook multiplication. + if (Operand::bothFitU64(MultiplicandOp, MultiplierOp) && + !MultiplicandOp.isConstant() && !MultiplierOp.isConstant()) { + U256Inst A = extractU256Operand(MultiplicandOp); + U256Inst B = extractU256Operand(MultiplierOp); + MType *I64Type = &Ctx.I64Type; + MInstruction *Zero = createIntConstInstruction(I64Type, 0); + MInstruction *MulLo = createEvmUmul128(A[0], B[0]); + MInstruction *MulHi = createEvmUmul128Hi(MulLo); + U256Inst Result = {MulLo, MulHi, Zero, Zero}; + return Operand(Result, EVMType::UINT256, ValueRange::U128); + } + // Phase 4: u64 fast path - one operand fits in u64 (4x1 multiplication) bool AIsU64 = MultiplicandOp.isConstU64(); bool BIsU64 = MultiplierOp.isConstU64(); @@ -1867,6 +1882,31 @@ typename EVMMirBuilder::Operand EVMMirBuilder::handleDiv(Operand DividendOp, } } + // Range-based u64÷u64 fast path: single OP_udiv with div-by-zero guard. + // DIV(u64, u64) → u64 result. Inserted before the isConstU64() path so + // that non-constant operands with proven narrow range benefit too. + if (Operand::bothFitU64(DividendOp, DivisorOp) && !DividendOp.isConstant() && + !DivisorOp.isConstant()) { + U256Inst A = extractU256Operand(DividendOp); + U256Inst B = extractU256Operand(DivisorOp); + MType *I64Type = &Ctx.I64Type; + MInstruction *Zero = createIntConstInstruction(I64Type, 0); + MInstruction *One = createIntConstInstruction(I64Type, 1); + + // Guard against div-by-zero: EVM DIV(x, 0) = 0, but x86 traps + MInstruction *IsZero = createInstruction( + false, CmpInstruction::ICMP_EQ, I64Type, B[0], Zero); + MInstruction *SafeB = + createInstruction(false, I64Type, IsZero, One, B[0]); + MInstruction *Quotient = createInstruction( + false, OP_udiv, I64Type, A[0], SafeB); + MInstruction *DivResult = createInstruction( + false, I64Type, IsZero, Zero, Quotient); + + U256Inst Result = {DivResult, Zero, Zero, Zero}; + return Operand(Result, EVMType::UINT256, ValueRange::U64); + } + // u64 divisor: inline cascading 128/64 division if (DivisorOp.isConstU64()) { uint64_t D = DivisorOp.getConstValue()[0]; @@ -2016,6 +2056,30 @@ typename EVMMirBuilder::Operand EVMMirBuilder::handleMod(Operand DividendOp, } } + // Range-based u64%u64 fast path: single OP_urem with div-by-zero guard. + // MOD(u64, u64) → u64 result (remainder < divisor). + if (Operand::bothFitU64(DividendOp, DivisorOp) && !DividendOp.isConstant() && + !DivisorOp.isConstant()) { + U256Inst A = extractU256Operand(DividendOp); + U256Inst B = extractU256Operand(DivisorOp); + MType *I64Type = &Ctx.I64Type; + MInstruction *Zero = createIntConstInstruction(I64Type, 0); + MInstruction *One = createIntConstInstruction(I64Type, 1); + + // Guard against div-by-zero: EVM MOD(x, 0) = 0 + MInstruction *IsZero = createInstruction( + false, CmpInstruction::ICMP_EQ, I64Type, B[0], Zero); + MInstruction *SafeB = + createInstruction(false, I64Type, IsZero, One, B[0]); + MInstruction *Remainder = createInstruction( + false, OP_urem, I64Type, A[0], SafeB); + MInstruction *ModResult = createInstruction( + false, I64Type, IsZero, Zero, Remainder); + + U256Inst Result = {ModResult, Zero, Zero, Zero}; + return Operand(Result, EVMType::UINT256, ValueRange::U64); + } + // u64 divisor: inline cascading 128/64 mod if (DivisorOp.isConstU64()) { uint64_t D = DivisorOp.getConstValue()[0]; @@ -2884,7 +2948,7 @@ EVMMirBuilder::handleCompareEqU64(const Operand &FullOp, uint64_t U64Val) { for (size_t I = 1; I < EVM_ELEMENTS_COUNT; ++I) { Result[I] = Zero; } - return Operand(Result, EVMType::UINT256); + return Operand(Result, EVMType::UINT256, ValueRange::U64); } typename EVMMirBuilder::Operand @@ -2919,7 +2983,7 @@ EVMMirBuilder::handleCompareLtRhsU64(const Operand &LHSOp, uint64_t RhsU64) { for (size_t I = 1; I < EVM_ELEMENTS_COUNT; ++I) { Result[I] = Zero; } - return Operand(Result, EVMType::UINT256); + return Operand(Result, EVMType::UINT256, ValueRange::U64); } typename EVMMirBuilder::Operand @@ -2955,7 +3019,7 @@ EVMMirBuilder::handleCompareGtRhsU64(const Operand &LHSOp, uint64_t RhsU64) { for (size_t I = 1; I < EVM_ELEMENTS_COUNT; ++I) { Result[I] = Zero; } - return Operand(Result, EVMType::UINT256); + return Operand(Result, EVMType::UINT256, ValueRange::U64); } typename EVMMirBuilder::Operand @@ -3571,7 +3635,8 @@ typename EVMMirBuilder::Operand EVMMirBuilder::handleByte(Operand IndexOp, for (size_t I = 1; I < EVM_ELEMENTS_COUNT; ++I) { ResultComponents[I] = Zero; } - return Operand(ResultComponents, EVMType::UINT256); + // BYTE always produces a single byte value (0..255) + return Operand(ResultComponents, EVMType::UINT256, ValueRange::U64); }; if (IndexOp.isConstant() && ValueOp.isConstant()) { diff --git a/src/compiler/evm_frontend/evm_mir_compiler.h b/src/compiler/evm_frontend/evm_mir_compiler.h index 34b88c1e9..0020561c8 100644 --- a/src/compiler/evm_frontend/evm_mir_compiler.h +++ b/src/compiler/evm_frontend/evm_mir_compiler.h @@ -115,6 +115,14 @@ class EVMMirBuilder final { using U256Value = std::array; using U256ConstInt = std::array; + // Range classification for u256 operands. Narrower ranges enable + // single-instruction fast paths instead of expensive multi-limb arithmetic. + enum class ValueRange : uint8_t { + U64, // Fits in 64 bits (limbs [1..3] == 0) + U128, // Fits in 128 bits (limbs [2..3] == 0) + U256, // Full 256 bits — conservative default + }; + EVMMirBuilder(CompilerContext &Context, MFunction &MFunc); class Operand { @@ -136,6 +144,13 @@ class EVMMirBuilder final { ZEN_ASSERT(Type == EVMType::UINT256 && "Multi-component only for U256"); } + // Constructor for U256 multi-component with explicit range + Operand(U256Inst Components, EVMType Type, ValueRange Range) + : Type(Type), Range(Range), U256Components(Components), + IsU256MultiComponent(true) { + ZEN_ASSERT(Type == EVMType::UINT256 && "Multi-component only for U256"); + } + Operand(U256Var VarComponents, EVMType Type) : Type(Type), U256VarComponents(VarComponents), IsU256MultiComponent(true) { @@ -143,7 +158,16 @@ class EVMMirBuilder final { } Operand(const U256Value &ConstValue) - : Type(EVMType::UINT256), ConstValue(ConstValue), IsConstant(true) {} + : Type(EVMType::UINT256), ConstValue(ConstValue), IsConstant(true) { + // Auto-derive range from constant value + if (ConstValue[1] == 0 && ConstValue[2] == 0 && ConstValue[3] == 0) { + Range = ValueRange::U64; + } else if (ConstValue[2] == 0 && ConstValue[3] == 0) { + Range = ValueRange::U128; + } else { + Range = ValueRange::U256; + } + } static Operand createDeferredBitwiseNot(U256Inst BaseComponents) { Operand Result; @@ -224,6 +248,14 @@ class EVMMirBuilder final { return U256Components; } + // Provable value range — narrower ranges enable fast arithmetic paths + ValueRange getRange() const { return Range; } + + // Check whether both operands provably fit in u64 + static bool bothFitU64(const Operand &A, const Operand &B) { + return A.getRange() == ValueRange::U64 && B.getRange() == ValueRange::U64; + } + constexpr bool isReg() { return false; } constexpr bool isTempReg() { return true; } @@ -231,6 +263,7 @@ class EVMMirBuilder final { MInstruction *Instr = nullptr; Variable *Var = nullptr; EVMType Type = EVMType::VOID; + ValueRange Range = ValueRange::U256; // For EVMU256Type: 4 I64 components [0]=low, [1]=mid-low, [2]=mid-high, // [3]=high @@ -330,6 +363,29 @@ class EVMMirBuilder final { } } + // Phase 1: Range-based u64 fast path for ADD + // When both operands provably fit in u64, emit single ADD + carry + // instead of the full 4-limb ADC chain. Result fits in u128. + if constexpr (Operator == BinaryOperator::BO_ADD) { + if (Operand::bothFitU64(LHSOp, RHSOp) && !LHSOp.isConstant() && + !RHSOp.isConstant()) { + MType *MirI64Type = + EVMFrontendContext::getMIRTypeFromEVMType(EVMType::UINT64); + MInstruction *Zero = createIntConstInstruction(MirI64Type, 0); + U256Inst LHS = extractU256Operand(LHSOp); + U256Inst RHS = extractU256Operand(RHSOp); + MInstruction *Sum = createInstruction( + false, OP_add, MirI64Type, LHS[0], RHS[0]); + Sum = protectUnsafeValue(Sum, MirI64Type); + // Carry = (Sum < LHS[0]) ? 1 : 0 + MInstruction *CarryCmp = createInstruction( + false, CmpInstruction::ICMP_ULT, MirI64Type, Sum, LHS[0]); + MInstruction *CarryExt = zeroExtendToI64(CarryCmp); + U256Inst Result = {Sum, CarryExt, Zero, Zero}; + return Operand(Result, EVMType::UINT256, ValueRange::U128); + } + } + // Phase 2: u64 fast path for ADD - share zero const for upper RHS limbs if constexpr (Operator == BinaryOperator::BO_ADD) { bool LHSIsU64 = LHSOp.isConstU64(); @@ -498,7 +554,8 @@ class EVMMirBuilder final { } U256Inst Result = handleCompareImpl(LHSOp, RHSOp, &Ctx.I64Type); - return Operand(Result, EVMType::UINT256); + // Comparison results are always 0 or 1 + return Operand(Result, EVMType::UINT256, ValueRange::U64); } // EVM bitwise opcode: and, or, xor @@ -562,7 +619,33 @@ class EVMMirBuilder final { for (size_t I = 1; I < EVM_ELEMENTS_COUNT; ++I) { Result[I] = Zero; } - return Operand(Result, EVMType::UINT256); + return Operand(Result, EVMType::UINT256, ValueRange::U64); + } + + // Non-constant AND with a U128 mask: result fits in U128 + if (LHSOp.getRange() <= ValueRange::U128 || + RHSOp.getRange() <= ValueRange::U128) { + // AND narrows to the smaller operand range + ValueRange NarrowRange = std::min(LHSOp.getRange(), RHSOp.getRange()); + U256Inst LHS = extractU256Operand(LHSOp); + U256Inst RHS = extractU256Operand(RHSOp); + MType *MirI64Type = + EVMFrontendContext::getMIRTypeFromEVMType(EVMType::UINT64); + MInstruction *Zero = createIntConstInstruction(MirI64Type, 0); + U256Inst Result = {}; + for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) { + if (NarrowRange == ValueRange::U64 && I >= 1) { + Result[I] = Zero; + } else if (NarrowRange == ValueRange::U128 && I >= 2) { + Result[I] = Zero; + } else { + Result[I] = protectUnsafeValue( + createInstruction(false, OP_and, MirI64Type, + LHS[I], RHS[I]), + MirI64Type); + } + } + return Operand(Result, EVMType::UINT256, NarrowRange); } } From 0c5afd81cbba4bfe3bff05fa475c5bc3200ca790 Mon Sep 17 00:00:00 2001 From: Abmcar Date: Fri, 10 Apr 2026 21:10:43 +0800 Subject: [PATCH 5/9] perf(evm): inline general u256 DIV/MOD to eliminate runtime call Replace the runtime function call fallback in handleDiv and handleMod with inline runtime divisor-size branching. At runtime, check if the divisor fits in a single 64-bit limb (upper 3 limbs all zero). If so, use the existing cascading 128/64 division pattern inline, avoiding the expensive indirect call, caller-save register spills, and stack frame overhead. Multi-limb divisors still fall back to the runtime call. Also handles the divisor==0 case inline (EVM spec: returns 0) to avoid entering the runtime for a common edge case. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../evm_frontend/evm_mir_compiler.cpp | 129 ++++++++++++++++-- src/compiler/evm_frontend/evm_mir_compiler.h | 5 + 2 files changed, 126 insertions(+), 8 deletions(-) diff --git a/src/compiler/evm_frontend/evm_mir_compiler.cpp b/src/compiler/evm_frontend/evm_mir_compiler.cpp index 02a83388f..5e231db47 100644 --- a/src/compiler/evm_frontend/evm_mir_compiler.cpp +++ b/src/compiler/evm_frontend/evm_mir_compiler.cpp @@ -1733,6 +1733,125 @@ EVMMirBuilder::handleModU64Dividend(uint64_t Dividend, return Operand(Result, EVMType::UINT256); } +typename EVMMirBuilder::Operand EVMMirBuilder::handleDivModGeneral( + const Operand &DividendOp, const Operand &DivisorOp, bool WantQuotient) { + MType *I64Type = &Ctx.I64Type; + MInstruction *Zero = createIntConstInstruction(I64Type, 0); + + U256Inst A = extractU256Operand(DividendOp); + U256Inst B = extractU256Operand(DivisorOp); + + // Check if divisor upper limbs are all zero (runtime 1-limb divisor) + MInstruction *UpperOr = createInstruction( + false, OP_or, I64Type, B[1], + createInstruction(false, OP_or, I64Type, B[2], B[3])); + MInstruction *HasUpperLimbs = createInstruction( + false, CmpInstruction::ICMP_NE, I64Type, UpperOr, Zero); + + // Also check if B[0] is zero (combined with upper limbs being zero means + // divisor == 0 → EVM returns 0) + MInstruction *IsB0Zero = createInstruction( + false, CmpInstruction::ICMP_EQ, I64Type, B[0], Zero); + + // Result variables for cross-BB communication + U256Var ResultVars = {}; + for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) { + ResultVars[I] = CurFunc->createVariable(I64Type); + } + + auto storeResult = [&](const U256Inst &Values) { + for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) { + createInstruction(true, &(Ctx.VoidType), Values[I], + ResultVars[I]->getVarIdx()); + } + }; + + auto loadResult = [&]() -> U256Inst { + U256Inst Values = {}; + for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) { + Values[I] = loadVariable(ResultVars[I]); + } + return Values; + }; + + // Branch structure: + // if (HasUpperLimbs) goto MultiLimbBB else goto SingleLimbCheckBB + // SingleLimbCheckBB: + // if (IsB0Zero) goto ZeroDivisorBB else goto SingleLimbBB + // ZeroDivisorBB: result = 0; goto AfterBB + // SingleLimbBB: inline cascading 128/64 div; goto AfterBB + // MultiLimbBB: runtime call; goto AfterBB + // AfterBB: phi + MBasicBlock *MultiLimbBB = createBasicBlock(); + MBasicBlock *SingleLimbCheckBB = createBasicBlock(); + MBasicBlock *ZeroDivisorBB = createBasicBlock(); + MBasicBlock *SingleLimbBB = createBasicBlock(); + MBasicBlock *AfterBB = createBasicBlock(); + + createInstruction(true, Ctx, HasUpperLimbs, MultiLimbBB, + SingleLimbCheckBB); + addSuccessor(MultiLimbBB); + addSuccessor(SingleLimbCheckBB); + + // --- SingleLimbCheckBB: check for zero divisor --- + setInsertBlock(SingleLimbCheckBB); + createInstruction(true, Ctx, IsB0Zero, ZeroDivisorBB, + SingleLimbBB); + addSuccessor(ZeroDivisorBB); + addSuccessor(SingleLimbBB); + + // --- ZeroDivisorBB: divisor == 0, EVM returns 0 --- + setInsertBlock(ZeroDivisorBB); + U256Inst ZeroResult = {Zero, Zero, Zero, Zero}; + storeResult(ZeroResult); + createInstruction(true, Ctx, AfterBB); + addSuccessor(AfterBB); + + // --- SingleLimbBB: 1-limb divisor, cascading 128/64 division --- + setInsertBlock(SingleLimbBB); + // Cascading: (0:A[3])/B[0], (R3:A[2])/B[0], (R2:A[1])/B[0], + // (R1:A[0])/B[0] + MInstruction *Div3 = createEvmUdiv128By64(Zero, A[3], B[0]); + MInstruction *Rem3 = createEvmUrem128By64(Div3); + MInstruction *Div2 = createEvmUdiv128By64(Rem3, A[2], B[0]); + MInstruction *Rem2 = createEvmUrem128By64(Div2); + MInstruction *Div1 = createEvmUdiv128By64(Rem2, A[1], B[0]); + MInstruction *Rem1 = createEvmUrem128By64(Div1); + MInstruction *Div0 = createEvmUdiv128By64(Rem1, A[0], B[0]); + MInstruction *Rem0 = createEvmUrem128By64(Div0); + + if (WantQuotient) { + U256Inst QuotResult = {Div0, Div1, Div2, Div3}; + storeResult(QuotResult); + } else { + U256Inst RemResult = {Rem0, Zero, Zero, Zero}; + storeResult(RemResult); + } + createInstruction(true, Ctx, AfterBB); + addSuccessor(AfterBB); + + // --- MultiLimbBB: multi-limb divisor, fall back to runtime --- + setInsertBlock(MultiLimbBB); + const auto &RuntimeFunctions = getRuntimeFunctionTable(); + Operand RuntimeResult; + if (WantQuotient) { + RuntimeResult = callRuntimeFor( + RuntimeFunctions.GetDiv, DividendOp, DivisorOp); + } else { + RuntimeResult = callRuntimeFor( + RuntimeFunctions.GetMod, DividendOp, DivisorOp); + } + storeResult(extractU256Operand(RuntimeResult)); + createInstruction(true, Ctx, AfterBB); + addSuccessor(AfterBB); + + // --- AfterBB: merge results --- + setInsertBlock(AfterBB); + return Operand(loadResult(), EVMType::UINT256); +} + typename EVMMirBuilder::Operand EVMMirBuilder::handleMul(Operand MultiplicandOp, Operand MultiplierOp) { // Phase 0: Constant folding @@ -1980,10 +2099,7 @@ typename EVMMirBuilder::Operand EVMMirBuilder::handleDiv(Operand DividendOp, return handleDivU64Dividend(A, DivisorOp); } - const auto &RuntimeFunctions = getRuntimeFunctionTable(); - return callRuntimeFor(RuntimeFunctions.GetDiv, - DividendOp, DivisorOp); + return handleDivModGeneral(DividendOp, DivisorOp, /*WantQuotient=*/true); } typename EVMMirBuilder::Operand EVMMirBuilder::handleSDiv(Operand DividendOp, @@ -2093,10 +2209,7 @@ typename EVMMirBuilder::Operand EVMMirBuilder::handleMod(Operand DividendOp, return handleModU64Dividend(A, DivisorOp); } - const auto &RuntimeFunctions = getRuntimeFunctionTable(); - return callRuntimeFor(RuntimeFunctions.GetMod, - DividendOp, DivisorOp); + return handleDivModGeneral(DividendOp, DivisorOp, /*WantQuotient=*/false); } typename EVMMirBuilder::Operand EVMMirBuilder::handleSMod(Operand DividendOp, diff --git a/src/compiler/evm_frontend/evm_mir_compiler.h b/src/compiler/evm_frontend/evm_mir_compiler.h index 0020561c8..c235588e3 100644 --- a/src/compiler/evm_frontend/evm_mir_compiler.h +++ b/src/compiler/evm_frontend/evm_mir_compiler.h @@ -1011,6 +1011,11 @@ class EVMMirBuilder final { Operand handleDivU64Dividend(uint64_t Dividend, const Operand &DivisorOp); Operand handleModU64Dividend(uint64_t Dividend, const Operand &DivisorOp); + // General u256 div/mod with runtime divisor-size branching. + // WantQuotient=true returns quotient (DIV), false returns remainder (MOD). + Operand handleDivModGeneral(const Operand &DividendOp, + const Operand &DivisorOp, bool WantQuotient); + // ==================== EVM to MIR Opcode Mapping ==================== Opcode getMirOpcode(BinaryOperator BinOpr); From f4fc7cee72a730155b3413eefc561b0eea27358f Mon Sep 17 00:00:00 2001 From: Abmcar Date: Sat, 11 Apr 2026 00:00:20 +0800 Subject: [PATCH 6/9] fix(evm): fix register coalescer crash in inline DIV/MOD The 5-block CFG pattern in handleDivModGeneral caused a live range calculation assertion failure (TheVNI != nullptr) during register allocation. The root cause was cross-block value references spanning two intermediate blocks (EntryBB -> SingleLimbCheckBB -> ZeroDivisorBB), which the live range calculator could not resolve. Replace the ZeroDivisorBB branch with a select-based zero guard: SafeB0 = select(IsB0Zero, 1, B[0]) prevents hardware DIV-by-zero, then select(IsB0Zero, 0, result) zeros the output when divisor was 0. This reduces the CFG from 5 blocks to 3, matching the proven handleDiv pattern (EntryBB -> SingleLimbBB/MultiLimbBB -> AfterBB). Co-Authored-By: Claude Opus 4.6 (1M context) --- .../evm_frontend/evm_mir_compiler.cpp | 65 +++++++++---------- 1 file changed, 31 insertions(+), 34 deletions(-) diff --git a/src/compiler/evm_frontend/evm_mir_compiler.cpp b/src/compiler/evm_frontend/evm_mir_compiler.cpp index 5e231db47..f434d1d5f 100644 --- a/src/compiler/evm_frontend/evm_mir_compiler.cpp +++ b/src/compiler/evm_frontend/evm_mir_compiler.cpp @@ -1737,6 +1737,7 @@ typename EVMMirBuilder::Operand EVMMirBuilder::handleDivModGeneral( const Operand &DividendOp, const Operand &DivisorOp, bool WantQuotient) { MType *I64Type = &Ctx.I64Type; MInstruction *Zero = createIntConstInstruction(I64Type, 0); + MInstruction *One = createIntConstInstruction(I64Type, 1); U256Inst A = extractU256Operand(DividendOp); U256Inst B = extractU256Operand(DivisorOp); @@ -1748,10 +1749,13 @@ typename EVMMirBuilder::Operand EVMMirBuilder::handleDivModGeneral( MInstruction *HasUpperLimbs = createInstruction( false, CmpInstruction::ICMP_NE, I64Type, UpperOr, Zero); - // Also check if B[0] is zero (combined with upper limbs being zero means - // divisor == 0 → EVM returns 0) + // Guard B[0] against zero to prevent hardware divide-by-zero trap. + // EVM DIV/MOD(a, 0) = 0, but x86 DIV with zero divisor raises SIGFPE. + // Replace zero divisor with 1 (harmless), then select result to 0 afterward. MInstruction *IsB0Zero = createInstruction( false, CmpInstruction::ICMP_EQ, I64Type, B[0], Zero); + MInstruction *SafeB0 = + createInstruction(false, I64Type, IsB0Zero, One, B[0]); // Result variables for cross-BB communication U256Var ResultVars = {}; @@ -1774,57 +1778,50 @@ typename EVMMirBuilder::Operand EVMMirBuilder::handleDivModGeneral( return Values; }; - // Branch structure: - // if (HasUpperLimbs) goto MultiLimbBB else goto SingleLimbCheckBB - // SingleLimbCheckBB: - // if (IsB0Zero) goto ZeroDivisorBB else goto SingleLimbBB - // ZeroDivisorBB: result = 0; goto AfterBB - // SingleLimbBB: inline cascading 128/64 div; goto AfterBB + // Branch structure (3-block, matches handleDiv pattern): + // if (HasUpperLimbs) goto MultiLimbBB else goto SingleLimbBB + // SingleLimbBB: inline cascading 128/64 div with SafeB0; goto AfterBB // MultiLimbBB: runtime call; goto AfterBB - // AfterBB: phi + // AfterBB: load result MBasicBlock *MultiLimbBB = createBasicBlock(); - MBasicBlock *SingleLimbCheckBB = createBasicBlock(); - MBasicBlock *ZeroDivisorBB = createBasicBlock(); MBasicBlock *SingleLimbBB = createBasicBlock(); MBasicBlock *AfterBB = createBasicBlock(); createInstruction(true, Ctx, HasUpperLimbs, MultiLimbBB, - SingleLimbCheckBB); - addSuccessor(MultiLimbBB); - addSuccessor(SingleLimbCheckBB); - - // --- SingleLimbCheckBB: check for zero divisor --- - setInsertBlock(SingleLimbCheckBB); - createInstruction(true, Ctx, IsB0Zero, ZeroDivisorBB, SingleLimbBB); - addSuccessor(ZeroDivisorBB); + addSuccessor(MultiLimbBB); addSuccessor(SingleLimbBB); - // --- ZeroDivisorBB: divisor == 0, EVM returns 0 --- - setInsertBlock(ZeroDivisorBB); - U256Inst ZeroResult = {Zero, Zero, Zero, Zero}; - storeResult(ZeroResult); - createInstruction(true, Ctx, AfterBB); - addSuccessor(AfterBB); - // --- SingleLimbBB: 1-limb divisor, cascading 128/64 division --- + // Uses SafeB0 (guaranteed non-zero) to avoid hardware DIV-by-zero. + // If B[0] was actually zero, IsB0Zero selects result to 0 afterward. setInsertBlock(SingleLimbBB); - // Cascading: (0:A[3])/B[0], (R3:A[2])/B[0], (R2:A[1])/B[0], - // (R1:A[0])/B[0] - MInstruction *Div3 = createEvmUdiv128By64(Zero, A[3], B[0]); + // Cascading: (0:A[3])/SafeB0, (R3:A[2])/SafeB0, ... + MInstruction *Div3 = createEvmUdiv128By64(Zero, A[3], SafeB0); MInstruction *Rem3 = createEvmUrem128By64(Div3); - MInstruction *Div2 = createEvmUdiv128By64(Rem3, A[2], B[0]); + MInstruction *Div2 = createEvmUdiv128By64(Rem3, A[2], SafeB0); MInstruction *Rem2 = createEvmUrem128By64(Div2); - MInstruction *Div1 = createEvmUdiv128By64(Rem2, A[1], B[0]); + MInstruction *Div1 = createEvmUdiv128By64(Rem2, A[1], SafeB0); MInstruction *Rem1 = createEvmUrem128By64(Div1); - MInstruction *Div0 = createEvmUdiv128By64(Rem1, A[0], B[0]); + MInstruction *Div0 = createEvmUdiv128By64(Rem1, A[0], SafeB0); MInstruction *Rem0 = createEvmUrem128By64(Div0); if (WantQuotient) { - U256Inst QuotResult = {Div0, Div1, Div2, Div3}; + // Select each quotient limb to 0 when divisor was zero + U256Inst QuotResult = {createInstruction( + false, I64Type, IsB0Zero, Zero, Div0), + createInstruction( + false, I64Type, IsB0Zero, Zero, Div1), + createInstruction( + false, I64Type, IsB0Zero, Zero, Div2), + createInstruction( + false, I64Type, IsB0Zero, Zero, Div3)}; storeResult(QuotResult); } else { - U256Inst RemResult = {Rem0, Zero, Zero, Zero}; + // Select remainder to 0 when divisor was zero (only limb 0 matters) + U256Inst RemResult = {createInstruction( + false, I64Type, IsB0Zero, Zero, Rem0), + Zero, Zero, Zero}; storeResult(RemResult); } createInstruction(true, Ctx, AfterBB); From b3ff51bed479604bc70d7aa61e50e9552a4a454a Mon Sep 17 00:00:00 2001 From: Abmcar Date: Wed, 15 Apr 2026 13:48:01 +0800 Subject: [PATCH 7/9] refactor(evm): extract u256 LT helper + document ADDMOD fast-path invariant Address Copilot review feedback on PR #458: 1. Extract the cascaded u256 unsigned-LT comparison (previously inlined four times in handleAddMod for AugLtMod, AddLtMod, Overflow, and SumLtMod) into a single u256UnsignedLT(A, B) lambda returning i64 {0,1}. No behavior change; reduces risk of future divergent edits across the four call sites. 2. Strengthen the fast-path eligibility comment to document the invariant the guard establishes: mod[3] != 0 && x[3] <= mod[3] implies x < 2*mod (and symmetrically for y), which is exactly what the single-subtraction normalization step requires. Includes the proof sketch so future refactors don't weaken the check. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../evm_frontend/evm_mir_compiler.cpp | 122 +++++------------- 1 file changed, 33 insertions(+), 89 deletions(-) diff --git a/src/compiler/evm_frontend/evm_mir_compiler.cpp b/src/compiler/evm_frontend/evm_mir_compiler.cpp index f434d1d5f..757d6cfda 100644 --- a/src/compiler/evm_frontend/evm_mir_compiler.cpp +++ b/src/compiler/evm_frontend/evm_mir_compiler.cpp @@ -2282,9 +2282,13 @@ typename EVMMirBuilder::Operand EVMMirBuilder::handleAddMod(Operand AugendOp, U256Inst Addend = extractU256Operand(AddendOp); U256Inst Modulus = extractU256Operand(ModulusOp); - // Fast-path eligibility: mod[3] != 0 && x[3] <= mod[3] && y[3] <= mod[3] - // Note: mod[3] != 0 implies mod != 0, so the mod == 0 case is handled - // by the slow path (runtime call) which returns 0 for zero modulus. + // Fast-path eligibility: mod[3] != 0 && x[3] <= mod[3] && y[3] <= mod[3]. + // Invariant established: x < 2*mod and y < 2*mod, so a single conditional + // subtraction is sufficient to normalize each operand into [0, mod). + // Proof: x < (x[3]+1)*2^192 <= (mod[3]+1)*2^192 <= 2*mod[3]*2^192 <= 2*mod, + // where the last two steps use mod[3] >= 1. Any weaker check (e.g. dropping + // the high-limb bound) breaks this invariant and the fast path. + // The mod == 0 case is routed to the runtime slow path, which returns 0. MInstruction *ModHi = Modulus[3]; MInstruction *ModHiNonZero = createInstruction( false, CmpInstruction::ICMP_NE, I64Type, ModHi, Zero); @@ -2299,6 +2303,28 @@ typename EVMMirBuilder::Operand EVMMirBuilder::handleAddMod(Operand AugendOp, FastEligible = createInstruction(false, OP_and, I64Type, FastEligible, AddendHiLE); + // Cascaded u256 unsigned LT: returns i64 {0,1} for A < B, high limb first. + auto u256UnsignedLT = [&](const U256Inst &A, const U256Inst &B) { + MInstruction *CmpResult = nullptr; + MInstruction *AllEqual = nullptr; + for (int I = EVM_ELEMENTS_COUNT - 1; I >= 0; --I) { + MInstruction *LT = createInstruction( + false, CmpInstruction::ICMP_ULT, I64Type, A[I], B[I]); + MInstruction *EQ = createInstruction( + false, CmpInstruction::ICMP_EQ, I64Type, A[I], B[I]); + if (CmpResult == nullptr) { + CmpResult = LT; + AllEqual = EQ; + } else { + CmpResult = createInstruction( + false, I64Type, AllEqual, LT, CmpResult); + AllEqual = createInstruction(false, OP_and, I64Type, + AllEqual, EQ); + } + } + return CmpResult; + }; + // Prepare result variables U256Var ResultVars = {}; for (size_t I = 0; I < EVM_ELEMENTS_COUNT; ++I) { @@ -2332,29 +2358,7 @@ typename EVMMirBuilder::Operand EVMMirBuilder::handleAddMod(Operand AugendOp, setInsertBlock(FastBB); { // Step 1: Normalize augend — if augend >= mod, use augend - mod - // Compute augend < mod using cascaded limb comparison - MInstruction *AugLtMod = nullptr; - { - // u256 LT comparison: augend < modulus (high-to-low cascade) - MInstruction *CmpResult = nullptr; - MInstruction *AllEqual = nullptr; - for (int I = EVM_ELEMENTS_COUNT - 1; I >= 0; --I) { - MInstruction *LT = createInstruction( - false, CmpInstruction::ICMP_ULT, I64Type, Augend[I], Modulus[I]); - MInstruction *EQ = createInstruction( - false, CmpInstruction::ICMP_EQ, I64Type, Augend[I], Modulus[I]); - if (CmpResult == nullptr) { - CmpResult = LT; - AllEqual = EQ; - } else { - CmpResult = createInstruction( - false, I64Type, AllEqual, LT, CmpResult); - AllEqual = createInstruction( - false, OP_and, I64Type, AllEqual, EQ); - } - } - AugLtMod = CmpResult; - } + MInstruction *AugLtMod = u256UnsignedLT(Augend, Modulus); // Compute augend - mod (u256 SUB chain) Operand AugSubMod = @@ -2369,27 +2373,7 @@ typename EVMMirBuilder::Operand EVMMirBuilder::handleAddMod(Operand AugendOp, } // Step 2: Normalize addend — if addend >= mod, use addend - mod - MInstruction *AddLtMod = nullptr; - { - MInstruction *CmpResult = nullptr; - MInstruction *AllEqual = nullptr; - for (int I = EVM_ELEMENTS_COUNT - 1; I >= 0; --I) { - MInstruction *LT = createInstruction( - false, CmpInstruction::ICMP_ULT, I64Type, Addend[I], Modulus[I]); - MInstruction *EQ = createInstruction( - false, CmpInstruction::ICMP_EQ, I64Type, Addend[I], Modulus[I]); - if (CmpResult == nullptr) { - CmpResult = LT; - AllEqual = EQ; - } else { - CmpResult = createInstruction( - false, I64Type, AllEqual, LT, CmpResult); - AllEqual = createInstruction( - false, OP_and, I64Type, AllEqual, EQ); - } - } - AddLtMod = CmpResult; - } + MInstruction *AddLtMod = u256UnsignedLT(Addend, Modulus); Operand AddSubMod = handleBinaryArithmetic(AddendOp, ModulusOp); @@ -2410,27 +2394,7 @@ typename EVMMirBuilder::Operand EVMMirBuilder::handleAddMod(Operand AugendOp, U256Inst Sum = extractU256Operand(SumOp); // Detect overflow: sum < NormAugend (unsigned comparison) - MInstruction *Overflow = nullptr; - { - MInstruction *CmpResult = nullptr; - MInstruction *AllEqual = nullptr; - for (int I = EVM_ELEMENTS_COUNT - 1; I >= 0; --I) { - MInstruction *LT = createInstruction( - false, CmpInstruction::ICMP_ULT, I64Type, Sum[I], NormAugend[I]); - MInstruction *EQ = createInstruction( - false, CmpInstruction::ICMP_EQ, I64Type, Sum[I], NormAugend[I]); - if (CmpResult == nullptr) { - CmpResult = LT; - AllEqual = EQ; - } else { - CmpResult = createInstruction( - false, I64Type, AllEqual, LT, CmpResult); - AllEqual = createInstruction( - false, OP_and, I64Type, AllEqual, EQ); - } - } - Overflow = CmpResult; - } + MInstruction *Overflow = u256UnsignedLT(Sum, NormAugend); // Step 4: SumSubMod = Sum - Mod (u256 SUB chain) Operand ModulusOpLocal(Modulus, EVMType::UINT256); @@ -2439,27 +2403,7 @@ typename EVMMirBuilder::Operand EVMMirBuilder::handleAddMod(Operand AugendOp, U256Inst SumSubMod = extractU256Operand(SumSubModOp); // Detect borrow: Sum < Mod - MInstruction *SumLtMod = nullptr; - { - MInstruction *CmpResult = nullptr; - MInstruction *AllEqual = nullptr; - for (int I = EVM_ELEMENTS_COUNT - 1; I >= 0; --I) { - MInstruction *LT = createInstruction( - false, CmpInstruction::ICMP_ULT, I64Type, Sum[I], Modulus[I]); - MInstruction *EQ = createInstruction( - false, CmpInstruction::ICMP_EQ, I64Type, Sum[I], Modulus[I]); - if (CmpResult == nullptr) { - CmpResult = LT; - AllEqual = EQ; - } else { - CmpResult = createInstruction( - false, I64Type, AllEqual, LT, CmpResult); - AllEqual = createInstruction( - false, OP_and, I64Type, AllEqual, EQ); - } - } - SumLtMod = CmpResult; - } + MInstruction *SumLtMod = u256UnsignedLT(Sum, Modulus); // Result selection: if (overflow || !borrow) use SumSubMod, else Sum // !borrow means Sum >= Mod, i.e., !SumLtMod From 66235af525bc3c6839059812d41c2754baca193d Mon Sep 17 00:00:00 2001 From: Abmcar Date: Sat, 25 Apr 2026 12:30:33 +0800 Subject: [PATCH 8/9] test(evm): add ADDMOD fast-path boundary fixtures Add three .easm/.expected pairs to exercise the inline ADDMOD fast path introduced in earlier u256 batch commits. The existing addmod fixtures cover only small modulus / 0xFF carry / mod==0 cases; none of them set mod[3] != 0, so the fast path's eligibility branch and normalize-subtract chain were untested. New cases: - addmod_fastpath_boundary: mod[3]=x[3]=y[3]=1 with x>mod and y>mod, forces the conditional subtract on both operands before sum. - addmod_fastpath_x_eq_mod: x == mod, exercising the AugLtMod=false / NormAugend=0 path while y < mod stays in place. - addmod_fastpath_overflow_into_high_limb: x[3]=y[3]=mod[3]=2^63 with low limbs maxed, sum carries out of limb 3 -> Overflow detection forces selecting SumSubMod regardless of borrow flag. Verified with run_evm_tests.py in both multipass JIT and interpreter modes (6/6 pass each), and via evmone-unittests multipass (223/223). --- tests/evm_asm/addmod_fastpath_boundary.easm | 15 +++++++++++++++ tests/evm_asm/addmod_fastpath_boundary.expected | 8 ++++++++ .../addmod_fastpath_overflow_into_high_limb.easm | 16 ++++++++++++++++ ...mod_fastpath_overflow_into_high_limb.expected | 8 ++++++++ tests/evm_asm/addmod_fastpath_x_eq_mod.easm | 13 +++++++++++++ tests/evm_asm/addmod_fastpath_x_eq_mod.expected | 8 ++++++++ 6 files changed, 68 insertions(+) create mode 100644 tests/evm_asm/addmod_fastpath_boundary.easm create mode 100644 tests/evm_asm/addmod_fastpath_boundary.expected create mode 100644 tests/evm_asm/addmod_fastpath_overflow_into_high_limb.easm create mode 100644 tests/evm_asm/addmod_fastpath_overflow_into_high_limb.expected create mode 100644 tests/evm_asm/addmod_fastpath_x_eq_mod.easm create mode 100644 tests/evm_asm/addmod_fastpath_x_eq_mod.expected diff --git a/tests/evm_asm/addmod_fastpath_boundary.easm b/tests/evm_asm/addmod_fastpath_boundary.easm new file mode 100644 index 000000000..efd582f4e --- /dev/null +++ b/tests/evm_asm/addmod_fastpath_boundary.easm @@ -0,0 +1,15 @@ +// Fast-path boundary: x[3] == y[3] == mod[3] != 0, x > mod, y > mod. +// Forces normalize-subtract on both operands before sum. +// mod = 2^192 + 5; x = mod + 1 = 2^192 + 6; y = mod + 2 = 2^192 + 7 +// (x + y) % mod = ((2^192 + 6) + (2^192 + 7)) % (2^192 + 5) +// = (2^193 + 13) % (2^192 + 5) +// = 3 (after normalize: x'=1, y'=2, sum=3 < mod) +PUSH32 0x0000000000000001000000000000000000000000000000000000000000000005 +PUSH32 0x0000000000000001000000000000000000000000000000000000000000000007 +PUSH32 0x0000000000000001000000000000000000000000000000000000000000000006 +ADDMOD +PUSH1 0x00 +MSTORE +PUSH1 0x20 +PUSH1 0x00 +RETURN diff --git a/tests/evm_asm/addmod_fastpath_boundary.expected b/tests/evm_asm/addmod_fastpath_boundary.expected new file mode 100644 index 000000000..a0864239b --- /dev/null +++ b/tests/evm_asm/addmod_fastpath_boundary.expected @@ -0,0 +1,8 @@ +status: success +error_code: 0 +stack: [] +memory: '0000000000000000000000000000000000000000000000000000000000000003' +storage: {} +transient_storage: {} +return: '0000000000000000000000000000000000000000000000000000000000000003' +events: [] diff --git a/tests/evm_asm/addmod_fastpath_overflow_into_high_limb.easm b/tests/evm_asm/addmod_fastpath_overflow_into_high_limb.easm new file mode 100644 index 000000000..64bb784c9 --- /dev/null +++ b/tests/evm_asm/addmod_fastpath_overflow_into_high_limb.easm @@ -0,0 +1,16 @@ +// Fast-path with carry out of limb 3: x + y overflows 256-bit, exercises Overflow detection. +// mod = 2^255 (mod[3] = 1<<63, eligible for fast path) +// x = y = (1<<63)<<192 | (2^192 - 1) -> x[3] = mod[3], x < mod +// sum_257 = 2^256 + 2^193 - 2; sum_256 = 2^193 - 2; overflow = 1 +// SumLtMod = (sum_256 < mod) = true, NotBorrow = 0 +// UseSumSubMod = overflow | NotBorrow = 1 -> result = SumSubMod = 2^256 + 2^193 - 2 - 2^255 +// Equivalent to (2x) mod 2^255 = 2^193 - 2 +PUSH32 0x8000000000000000000000000000000000000000000000000000000000000000 +PUSH32 0x8000000000000000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF +PUSH32 0x8000000000000000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF +ADDMOD +PUSH1 0x00 +MSTORE +PUSH1 0x20 +PUSH1 0x00 +RETURN diff --git a/tests/evm_asm/addmod_fastpath_overflow_into_high_limb.expected b/tests/evm_asm/addmod_fastpath_overflow_into_high_limb.expected new file mode 100644 index 000000000..b411135ea --- /dev/null +++ b/tests/evm_asm/addmod_fastpath_overflow_into_high_limb.expected @@ -0,0 +1,8 @@ +status: success +error_code: 0 +stack: [] +memory: '0000000000000001FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE' +storage: {} +transient_storage: {} +return: '0000000000000001FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE' +events: [] diff --git a/tests/evm_asm/addmod_fastpath_x_eq_mod.easm b/tests/evm_asm/addmod_fastpath_x_eq_mod.easm new file mode 100644 index 000000000..c6ef430f7 --- /dev/null +++ b/tests/evm_asm/addmod_fastpath_x_eq_mod.easm @@ -0,0 +1,13 @@ +// Fast-path with x == mod: AugLtMod evaluates false, normalize subtract zeroes x. +// y < mod stays as-is, sum = 0 + y = y, no overflow, sum < mod -> result = y. +// mod = 2^192 + 2^128 + 12345; x = mod; y = 2^192 + 7 (< mod since mod[2]=1, y[2]=0) +// (x + y) % mod = (mod + y) % mod = y +PUSH32 0x0000000000000001000000000000000100000000000000000000000000003039 +PUSH32 0x0000000000000001000000000000000000000000000000000000000000000007 +PUSH32 0x0000000000000001000000000000000100000000000000000000000000003039 +ADDMOD +PUSH1 0x00 +MSTORE +PUSH1 0x20 +PUSH1 0x00 +RETURN diff --git a/tests/evm_asm/addmod_fastpath_x_eq_mod.expected b/tests/evm_asm/addmod_fastpath_x_eq_mod.expected new file mode 100644 index 000000000..63758b3e5 --- /dev/null +++ b/tests/evm_asm/addmod_fastpath_x_eq_mod.expected @@ -0,0 +1,8 @@ +status: success +error_code: 0 +stack: [] +memory: '0000000000000001000000000000000000000000000000000000000000000007' +storage: {} +transient_storage: {} +return: '0000000000000001000000000000000000000000000000000000000000000007' +events: [] From 20cdbd59eed636d370159b6d89ccedb0dc148165 Mon Sep 17 00:00:00 2001 From: Abmcar Date: Mon, 27 Apr 2026 16:37:50 +0800 Subject: [PATCH 9/9] docs(compiler): add change doc for u256 arithmetic optimizations Light-tier change doc covering the five MIR-level u256 lowering optimizations on this branch (shift strength reduction, ADDMOD inline fast path, dead-carry barrier elimination, value-range narrowing, inline DIV/MOD) plus the 3-block CFG workaround for the register coalescer assertion. Records the suite-level geomean +0.69% [-1.80%, +3.44%] and targeted +6%-18% wins on u256-arithmetic-heavy benches, and the dropped Barrett MULMOD variant. --- .../2026-04-10-u256-arithmetic-opts/README.md | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 docs/changes/2026-04-10-u256-arithmetic-opts/README.md diff --git a/docs/changes/2026-04-10-u256-arithmetic-opts/README.md b/docs/changes/2026-04-10-u256-arithmetic-opts/README.md new file mode 100644 index 000000000..8f9ea67e9 --- /dev/null +++ b/docs/changes/2026-04-10-u256-arithmetic-opts/README.md @@ -0,0 +1,89 @@ +# Change: Inline u256 arithmetic optimizations + +- **Status**: Implemented +- **Date**: 2026-04-10 +- **Tier**: Light + +## Overview + +Five MIR-level optimizations to u256 arithmetic lowering in the EVM frontend, +plus a register-allocator workaround uncovered by the new inline DIV/MOD path. +All changes are contained in `src/compiler/evm_frontend/evm_mir_compiler.{cpp,h}`. + +- **Shift strength reduction** — In dynamic-shift lowering for `SHL`/`SHR`/`SAR`, + replace `OP_udiv`/`OP_urem` by 64 with `OP_ushr 6` / `OP_and 63`. Avoids the + ~30-cycle x86 `DIV` on the dynamic shift path. +- **ADDMOD inline fast path** — For non-constant `ADDMOD` with `mod[3] != 0` + (modulus ≥ 2^192), inline `intx::addmod` at MIR level: normalize augend/addend + against `mod`, add with overflow detection, subtract `mod` with borrow + detection, select. Small moduli (`mod[3] == 0`) and `mod == 0` continue to + call the runtime helper. +- **Dead-carry barrier elimination** — Drop `protectUnsafeValue` barriers in + `handleAddU64Const` / `handleSubU64Const` where x86 `CF` is provably dead + (RHS-immediate MOVs and terminal ADC; SUB/ICMP_ULT/zeroExtend borrow chain in + SUB never lives `CF`). Generic u256 ADD/SUB chains in the header are + intentionally unchanged. +- **Value-range narrowing** — Add `ValueRange{U64,U128,U256}` on `Operand`, + auto-derived from constants and propagated through AND masks, `BYTE`, and + comparisons. When both operands of `ADD`/`MUL`/`DIV`/`MOD` are provably U64, + emit a single-instruction fast path instead of the 4-limb chain. +- **Inline u256 DIV/MOD** — Replace the runtime fallback in `handleDiv` / + `handleMod` with a runtime divisor-size check. Single-limb divisors (upper + three limbs zero) use the existing cascading 128/64 division pattern inline; + multi-limb divisors still call the runtime. + +A 5-block CFG variant of inline DIV/MOD (with a dedicated `ZeroDivisorBB`) +tripped a `CgLiveRangeCalc` assertion (`TheVNI != nullptr`) during register +allocation. The committed form uses a 3-block CFG matching `handleDiv`: a +branchless guard `SafeB0 = select(B[0] == 0, 1, B[0])` feeds the hardware +`DIV`, then `select(B[0] == 0, 0, result)` zeros the output. This path is +distinct from the `findReachingDefs` fix in PR #456. + +## Motivation + +U256 arithmetic lowering currently calls into runtime helpers for several +operations that have closed-form MIR sequences. Inlining them removes a call, +exposes the operation to the existing peephole / register-allocation passes, +and on operands that are provably narrow lets the JIT collapse a 4-limb chain +to a single-instruction fast path. The carry-barrier removals are a peephole +cleanup: `protectUnsafeValue` barriers force stack spills for live `CF`, and +several call sites do not actually have a downstream `CF` consumer, so the +barrier is structural noise that survived earlier passes. + +A Barrett-reduction MULMOD variant was prototyped in the same line of work +but abandoned — both inline and C-helper forms crashed the register coalescer +on contracts with thousands of `MULMOD` call sites in one function and showed +no measurable win. + +## Impact + +- Module: `docs/modules/compiler/` (EVM frontend MIR lowering) +- 8 files changed, +547/-34 + - `src/compiler/evm_frontend/evm_mir_compiler.{cpp,h}` — all five + optimizations + range tracking + reg-alloc workaround + - `tests/evm_asm/addmod_fastpath_*.{easm,expected}` — 3 boundary fixtures +- Performance (evmone-bench, multipass, 10 reps, suite of 27 + `external/total/{main,micro}` benches, CPU-pinned, single session, baseline + `upstream/main@ec3c9f9`): + - Suite-level geomean speedup: **1.0069 (+0.69%)**, 95% bootstrap CI + `[-1.80%, +3.44%]` — within per-bench noise; not statistically distinguishable + from parity at suite level + - Targeted wins on u256-arithmetic-heavy benches: weierstrudel/15 +18.3%, + weierstrudel/1 +13.5%, swap_math/received +9.7%, snailtracer/benchmark + +7.9%, swap_math/spent +6.7%, swap_math/insufficient_liquidity +6.6%, + structarray_alloc/nfts_rank +3.9% + - Suite-level geomean is in the noise band because integer-arithmetic wins + on JIT-bound micro benches have largely been absorbed by recent upstream + work (#428 BMI2 lowering, #435 peephole, #395 SSA) + - No regressions outside per-bench noise on the bottom benches (CV > 7%) +- Correctness: gas and output match in all paths +- No host or ABI changes; multipass-only — interpreter path is unaffected + +## Checklist + +- [x] Implementation complete +- [x] Tests added/updated — `tests/evm_asm/addmod_fastpath_*` (3 new boundary + fixtures, multipass + interpreter both 6/6); `evmone-unittests` multipass + 223/223, interpreter 215/215; `evmone-statetest` `fork_Cancun` 2723/2723 +- [ ] Module specs in `docs/modules/` updated (if affected) +- [x] Build and tests pass — CI 17/17 green on PR #458 HEAD `66235af`