diff --git a/src/compiler/evm_frontend/evm_memory_analysis.h b/src/compiler/evm_frontend/evm_memory_analysis.h index a837ae03..aedc5f0b 100644 --- a/src/compiler/evm_frontend/evm_memory_analysis.h +++ b/src/compiler/evm_frontend/evm_memory_analysis.h @@ -879,6 +879,9 @@ class MemoryGuaranteedMinBytesAnalysis { case MemoryOpKind::MStore: case MemoryOpKind::MStore8: return Op.Writes.size() == 1 && getIntervalEnd(Op.Writes[0], End); + case MemoryOpKind::CallDataCopy: + case MemoryOpKind::CodeCopy: + return Op.Writes.size() == 1 && getIntervalEnd(Op.Writes[0], End); case MemoryOpKind::MCopy: { if (Op.Reads.size() != 1 || Op.Writes.size() != 1) { return false; diff --git a/src/compiler/evm_frontend/evm_mir_compiler.cpp b/src/compiler/evm_frontend/evm_mir_compiler.cpp index 9ff0e22f..a74cd413 100644 --- a/src/compiler/evm_frontend/evm_mir_compiler.cpp +++ b/src/compiler/evm_frontend/evm_mir_compiler.cpp @@ -4740,7 +4740,11 @@ void EVMMirBuilder::handleCodeCopy(Operand DestOffsetComponents, Operand OffsetComponents, Operand SizeComponents) { const auto &RuntimeFunctions = getRuntimeFunctionTable(); - preExpandCopyMemory(DestOffsetComponents, SizeComponents); + if (preExpandCopyMemory(DestOffsetComponents, SizeComponents)) { +#ifdef ZEN_ENABLE_MULTIPASS_JIT_LOGGING + ++MemStats.CopyGuaranteedElisionCount; +#endif + } MInstruction *Size = extractKnownU64LowOperand(SizeComponents); chargeWordCopyGasIR(Size); uint64_t Non64Value = std::numeric_limits::max(); @@ -6789,9 +6793,20 @@ void EVMMirBuilder::preExpandKeccakTwoWordMemory(Operand &OffsetComponents) { expandMemoryIR(RequiredSize, Overflow); } -void EVMMirBuilder::preExpandCopyMemory(Operand &DestOffsetComponents, +bool EVMMirBuilder::preExpandCopyMemory(Operand &DestOffsetComponents, Operand &SizeComponents) { + const bool DestWasConstU64 = DestOffsetComponents.isConstU64(); + const uint64_t ConstDest = + DestWasConstU64 ? DestOffsetComponents.getConstValue()[0] : 0; + const bool SizeWasConstU64 = SizeComponents.isConstU64(); + const uint64_t ConstSize = + SizeWasConstU64 ? SizeComponents.getConstValue()[0] : 0; + normalizeOffsetWithSize(DestOffsetComponents, SizeComponents); + if (DestWasConstU64 && SizeWasConstU64 && + tryUseGuaranteedMinBytesExpansionElision(true, ConstDest, ConstSize)) { + return true; + } MType *I64Type = &Ctx.I64Type; MInstruction *DestOffset = extractKnownU64LowOperand(DestOffsetComponents); @@ -6802,6 +6817,7 @@ void EVMMirBuilder::preExpandCopyMemory(Operand &DestOffsetComponents, false, CmpInstruction::Predicate::ICMP_ULT, I64Type, RequiredSize, DestOffset); expandMemoryIR(RequiredSize, Overflow); + return false; } void EVMMirBuilder::chargeWordCopyGasIR(MInstruction *Size) { @@ -7214,7 +7230,11 @@ void EVMMirBuilder::handleCallDataCopy(Operand DestOffsetComponents, Operand SizeComponents) { const auto &RuntimeFunctions = getRuntimeFunctionTable(); uint64_t Non64Value = std::numeric_limits::max(); - preExpandCopyMemory(DestOffsetComponents, SizeComponents); + if (preExpandCopyMemory(DestOffsetComponents, SizeComponents)) { +#ifdef ZEN_ENABLE_MULTIPASS_JIT_LOGGING + ++MemStats.CopyGuaranteedElisionCount; +#endif + } MInstruction *Size = extractKnownU64LowOperand(SizeComponents); chargeWordCopyGasIR(Size); normalizeOperandU64(OffsetComponents, &Non64Value); @@ -7312,6 +7332,7 @@ typename EVMMirBuilder::Operand EVMMirBuilder::handleReturnDataSize() { bool EVMMirBuilder::hasMemoryCompileStats() const { return MemStats.MLoadExpandCount != 0 || MemStats.MStoreExpandCount != 0 || MemStats.MStore8ExpandCount != 0 || MemStats.MCopyExpandCount != 0 || + MemStats.CopyGuaranteedElisionCount != 0 || MemStats.BlockConstPrecheckCount != 0 || MemStats.MemoryExpansionPlanCount != 0 || MemStats.LinearU64AddrFastPathCount != 0 || @@ -7674,7 +7695,8 @@ void EVMMirBuilder::dumpMemoryCompileStats() const { ZEN_LOG_DEBUG( "[EVM-MEM-SUMMARY] mload_expand=%llu mstore_expand=%llu " "mstore8_expand=%llu mcopy_expand=%llu " - "mcopy_guaranteed_elision=%llu memory_dse_candidates=%llu " + "mcopy_guaranteed_elision=%llu copy_guaranteed_elision=%llu " + "memory_dse_candidates=%llu " "memory_dse_eliminated_writes=%llu memory_load_forward_candidates=%llu " "memory_load_forwarded=%llu block_const_precheck=%llu " "block_linear_precheck=%llu prechecked_mload_ops=%llu " @@ -7768,6 +7790,7 @@ void EVMMirBuilder::dumpMemoryCompileStats() const { static_cast(MemStats.MStore8ExpandCount), static_cast(MemStats.MCopyExpandCount), static_cast(MemStats.MCopyGuaranteedElisionCount), + static_cast(MemStats.CopyGuaranteedElisionCount), static_cast(MemStats.MemoryDSEStoreCandidates), static_cast(MemStats.MemoryDSEEliminatedWrites), static_cast(MemStats.MemoryLoadForwardCandidates), @@ -8639,6 +8662,7 @@ void EVMMirBuilder::noteMemoryOpcodeInBlock(evmc_opcode Opcode, uint64_t PC) { } void EVMMirBuilder::noteHelperOpcodeInBlock(evmc_opcode Opcode, uint64_t PC) { + CurrentMemoryOpPC = PC; if (!CurBlockMemStats.Active) { return; } diff --git a/src/compiler/evm_frontend/evm_mir_compiler.h b/src/compiler/evm_frontend/evm_mir_compiler.h index b2c045f3..5e0f91c4 100644 --- a/src/compiler/evm_frontend/evm_mir_compiler.h +++ b/src/compiler/evm_frontend/evm_mir_compiler.h @@ -1458,6 +1458,7 @@ class EVMMirBuilder final { uint64_t MStore8ExpandCount = 0; uint64_t MCopyExpandCount = 0; uint64_t MCopyGuaranteedElisionCount = 0; + uint64_t CopyGuaranteedElisionCount = 0; uint64_t MemoryDSEStoreCandidates = 0; uint64_t MemoryDSEEliminatedWrites = 0; uint64_t MemoryLoadForwardCandidates = 0; @@ -1883,7 +1884,7 @@ class EVMMirBuilder final { void reloadMemorySizeFromInstance(); void expandMemoryIR(MInstruction *RequiredSize, MInstruction *Overflow); void preExpandKeccakTwoWordMemory(Operand &OffsetComponents); - void preExpandCopyMemory(Operand &DestOffsetComponents, + bool preExpandCopyMemory(Operand &DestOffsetComponents, Operand &SizeComponents); void chargeWordCopyGasIR(MInstruction *Size); void chargeDynamicGasIR(MInstruction *GasCost); diff --git a/src/tests/evm_jit_frontend_tests.cpp b/src/tests/evm_jit_frontend_tests.cpp index f046c7ad..c61bf013 100644 --- a/src/tests/evm_jit_frontend_tests.cpp +++ b/src/tests/evm_jit_frontend_tests.cpp @@ -3,6 +3,7 @@ #include "action/evm_bytecode_visitor.h" #include "compiler/evm_frontend/evm_analyzer.h" +#include "compiler/evm_frontend/evm_imported.h" #include "compiler/evm_frontend/evm_memory_analysis.h" #include "compiler/evm_frontend/evm_memory_facts.h" #include "compiler/evm_frontend/evm_memory_grouping.h" @@ -32,6 +33,23 @@ using COMPILER::EVMValueRange; using zen::common::BinaryOperator; using zen::common::CompareOperator; +#ifdef ZEN_ENABLE_MULTIPASS_JIT_LOGGING +class CapturingLogger final : public zen::utils::ILogger { +public: + void trace(const std::string &, const char *, int, const char *) override {} + void debug(const std::string &Message, const char *, int, + const char *) override { + DebugMessages.push_back(Message); + } + void info(const std::string &, const char *, int, const char *) override {} + void warn(const std::string &, const char *, int, const char *) override {} + void error(const std::string &, const char *, int, const char *) override {} + void fatal(const std::string &, const char *, int, const char *) override {} + + std::vector DebugMessages; +}; +#endif + EVMAnalyzer analyzeBytecode(const std::vector &Bytecode) { EVMAnalyzer Analyzer(EVMC_CANCUN); const uint8_t *Data = Bytecode.empty() ? nullptr : Bytecode.data(); @@ -123,6 +141,51 @@ void expectPCList(const std::vector &Actual, } } +#if defined(ZEN_ENABLE_MULTIPASS_JIT_LOGGING) && \ + defined(ZEN_ENABLE_EVM_MEMORY_PLAN_FRAMEWORK) +size_t countRuntimeCalls(const COMPILER::MFunction &Func, uint64_t Address) { + std::string Mir; + llvm::raw_string_ostream OS(Mir); + Func.print(OS); + OS.flush(); + + const std::string Needle = + "target = const.i64 " + std::to_string(Address) + ", "; + size_t Count = 0; + for (size_t Pos = Mir.find(Needle); Pos != std::string::npos; + Pos = Mir.find(Needle, Pos + Needle.size())) { + ++Count; + } + return Count; +} + +std::optional +countExpandMemoryCallsForBytecode(const std::vector &Bytecode) { + COMPILER::EVMFrontendContext Ctx; + Ctx.setRevision(EVMC_CANCUN); + Ctx.setBytecode(reinterpret_cast(Bytecode.data()), + Bytecode.size()); + + COMPILER::MModule Mod(Ctx); + std::array ParamTypes = { + COMPILER::MPointerType::create(Ctx, Ctx.VoidType)}; + COMPILER::MFunctionType *FuncType = COMPILER::MFunctionType::create( + Ctx, Ctx.VoidType, llvm::ArrayRef(ParamTypes)); + Mod.addFuncType(FuncType); + + COMPILER::MFunction Func(Ctx, 0); + Func.setFunctionType(FuncType); + EVMMirBuilder Builder(Ctx, Func); + if (!Builder.compile(&Ctx)) { + return std::nullopt; + } + + const auto &RuntimeFunctions = COMPILER::getRuntimeFunctionTable(); + return countRuntimeCalls( + Func, COMPILER::getFunctionAddress(RuntimeFunctions.ExpandMemoryNoGas)); +} +#endif + void appendPushU64(std::vector &Bytecode, uint64_t Value) { if (Value == 0) { Bytecode.push_back(OP_PUSH0); @@ -408,6 +471,44 @@ TEST(EVMMemoryGuaranteedMinBytesAnalysisTest, IgnoresZeroLengthMCopy) { EXPECT_EQ(Guaranteed.getGuaranteedMinBytesBeforeOp(Facts.Ops[1].Id), 0u); } +TEST(EVMMemoryGuaranteedMinBytesAnalysisTest, + LearnsConstCallDataCopyDestinationEnd) { + const std::vector Bytecode = { + OP_PUSH1, 0x20, OP_PUSH1, 0x00, OP_PUSH1, 0x80, + OP_CALLDATACOPY, OP_PUSH1, 0x80, OP_MLOAD}; + + COMPILER::MemoryFacts Facts = collectMemoryFacts(Bytecode); + COMPILER::MemoryGuaranteedMinBytesAnalysis Guaranteed(Facts); + + ASSERT_EQ(Facts.Ops.size(), 2u); + EXPECT_EQ(Guaranteed.getGuaranteedMinBytesBeforeOp(Facts.Ops[1].Id), 0xa0u); +} + +TEST(EVMMemoryGuaranteedMinBytesAnalysisTest, + LearnsConstCodeCopyDestinationEnd) { + const std::vector Bytecode = { + OP_PUSH1, 0x20, OP_PUSH1, 0x00, OP_PUSH1, + 0x40, OP_CODECOPY, OP_PUSH1, 0x40, OP_MLOAD}; + + COMPILER::MemoryFacts Facts = collectMemoryFacts(Bytecode); + COMPILER::MemoryGuaranteedMinBytesAnalysis Guaranteed(Facts); + + ASSERT_EQ(Facts.Ops.size(), 2u); + EXPECT_EQ(Guaranteed.getGuaranteedMinBytesBeforeOp(Facts.Ops[1].Id), 0x60u); +} + +TEST(EVMMemoryGuaranteedMinBytesAnalysisTest, IgnoresZeroLengthCallDataCopy) { + const std::vector Bytecode = { + OP_PUSH1, 0x00, OP_PUSH1, 0x00, OP_PUSH1, 0x80, + OP_CALLDATACOPY, OP_PUSH1, 0x00, OP_MLOAD}; + + COMPILER::MemoryFacts Facts = collectMemoryFacts(Bytecode); + COMPILER::MemoryGuaranteedMinBytesAnalysis Guaranteed(Facts); + + ASSERT_EQ(Facts.Ops.size(), 2u); + EXPECT_EQ(Guaranteed.getGuaranteedMinBytesBeforeOp(Facts.Ops[1].Id), 0u); +} + TEST(EVMMemoryFactsBuilderTest, RecordsCopyAddressSpaces) { const std::vector Bytecode = {OP_PUSH1, 0x20, OP_PUSH1, 0x04, OP_PUSH1, 0x80, OP_CALLDATACOPY}; @@ -430,6 +531,150 @@ TEST(EVMMemoryFactsBuilderTest, RecordsCopyAddressSpaces) { EXPECT_EQ(Op.Writes[0].Size.Value, 0x20u); } +#if defined(ZEN_ENABLE_MULTIPASS_JIT_LOGGING) && \ + defined(ZEN_ENABLE_EVM_MEMORY_PLAN_FRAMEWORK) +TEST(EVMMirBuilderMemoryStatsTest, + ReusesCrossBlockGuaranteedSizeForCopyConsumers) { + const std::vector Bytecode = { + 0x60, 0x01, // PUSH1 value + 0x60, 0x80, // PUSH1 memory offset + 0x52, // MSTORE: guarantees memory through 0x9f + 0x60, 0x08, // PUSH1 successor block + 0x56, // JUMP + 0x5b, // JUMPDEST + 0x60, 0x20, // PUSH1 CALLDATACOPY size + 0x60, 0x00, // PUSH1 calldata source + 0x60, 0x80, // PUSH1 memory destination + 0x37, // CALLDATACOPY: covered by the predecessor proof + 0x60, 0x20, // PUSH1 CODECOPY size + 0x60, 0x00, // PUSH1 code source + 0x60, 0x40, // PUSH1 memory destination + 0x39, // CODECOPY: covered by the predecessor proof + 0x60, 0x20, // PUSH1 CODECOPY size + 0x60, 0x00, // PUSH1 code source + 0x60, 0xa0, // PUSH1 memory destination + 0x39, // CODECOPY: extends beyond the predecessor proof + 0x00 // STOP + }; + + COMPILER::EVMFrontendContext Ctx; + Ctx.setRevision(EVMC_CANCUN); + Ctx.setBytecode(reinterpret_cast(Bytecode.data()), + Bytecode.size()); + + COMPILER::MModule Mod(Ctx); + std::array ParamTypes = { + COMPILER::MPointerType::create(Ctx, Ctx.VoidType)}; + COMPILER::MFunctionType *FuncType = COMPILER::MFunctionType::create( + Ctx, Ctx.VoidType, llvm::ArrayRef(ParamTypes)); + Mod.addFuncType(FuncType); + + COMPILER::MFunction Func(Ctx, 0); + Func.setFunctionType(FuncType); + EVMMirBuilder Builder(Ctx, Func); + ASSERT_TRUE(Builder.compile(&Ctx)); + + auto Logger = std::make_shared(); + auto &Logging = zen::utils::Logging::getInstance(); + auto PreviousLogger = Logging.getLogger(); + Logging.setLogger(Logger); + Builder.dumpMemoryCompileStats(); + Logging.setLogger(std::move(PreviousLogger)); + + auto Summary = std::find_if( + Logger->DebugMessages.begin(), Logger->DebugMessages.end(), + [](const std::string &Message) { + return Message.find("[EVM-MEM-SUMMARY]") != std::string::npos; + }); + ASSERT_NE(Summary, Logger->DebugMessages.end()); + EXPECT_NE(Summary->find("copy_guaranteed_elision=2 "), std::string::npos) + << *Summary; + EXPECT_EQ(Summary->find("copy_guaranteed_elision=3 "), std::string::npos) + << *Summary; +} + +TEST(EVMMirBuilderMemoryStatsTest, + TracksCopyProofAtEachOpcodePCAndRetainsFallbackExpansion) { + const std::vector ProducerOnly = {0x60, 0x01, 0x60, 0x80, 0x52, + 0x60, 0x08, 0x56, 0x5b, 0x00}; + const std::vector WithUncoveredCopy = { + 0x60, 0x01, // PUSH1 value + 0x60, 0x80, // PUSH1 memory offset + 0x52, // MSTORE: guarantees memory through 0x9f + 0x60, 0x08, // PUSH1 successor block + 0x56, // JUMP + 0x5b, // JUMPDEST + 0x60, 0x20, // PUSH1 CODECOPY size + 0x60, 0x00, // PUSH1 code source + 0x60, 0xa0, // PUSH1 memory destination + 0x39, // CODECOPY: uncovered, grows memory through 0xbf + 0x00 // STOP + }; + const std::vector Bytecode = { + 0x60, 0x01, // PUSH1 value + 0x60, 0x80, // PUSH1 memory offset + 0x52, // MSTORE: guarantees memory through 0x9f + 0x60, 0x08, // PUSH1 successor block + 0x56, // JUMP + 0x5b, // JUMPDEST + 0x60, 0x20, // PUSH1 CODECOPY size + 0x60, 0x00, // PUSH1 code source + 0x60, 0xa0, // PUSH1 memory destination + 0x39, // CODECOPY: uncovered, grows memory through 0xbf + 0x60, 0x20, // PUSH1 CALLDATACOPY size + 0x60, 0x00, // PUSH1 calldata source + 0x60, 0xa0, // PUSH1 memory destination + 0x37, // CALLDATACOPY: covered by the preceding COPY proof + 0x00 // STOP + }; + + const std::optional ProducerExpansionCalls = + countExpandMemoryCallsForBytecode(ProducerOnly); + const std::optional FallbackExpansionCalls = + countExpandMemoryCallsForBytecode(WithUncoveredCopy); + const std::optional CoveredSuccessorExpansionCalls = + countExpandMemoryCallsForBytecode(Bytecode); + ASSERT_TRUE(ProducerExpansionCalls.has_value()); + ASSERT_TRUE(FallbackExpansionCalls.has_value()); + ASSERT_TRUE(CoveredSuccessorExpansionCalls.has_value()); + EXPECT_EQ(*FallbackExpansionCalls, *ProducerExpansionCalls + 1); + EXPECT_EQ(*CoveredSuccessorExpansionCalls, *FallbackExpansionCalls); + + COMPILER::EVMFrontendContext Ctx; + Ctx.setRevision(EVMC_CANCUN); + Ctx.setBytecode(reinterpret_cast(Bytecode.data()), + Bytecode.size()); + + COMPILER::MModule Mod(Ctx); + std::array ParamTypes = { + COMPILER::MPointerType::create(Ctx, Ctx.VoidType)}; + COMPILER::MFunctionType *FuncType = COMPILER::MFunctionType::create( + Ctx, Ctx.VoidType, llvm::ArrayRef(ParamTypes)); + Mod.addFuncType(FuncType); + + COMPILER::MFunction Func(Ctx, 0); + Func.setFunctionType(FuncType); + EVMMirBuilder Builder(Ctx, Func); + ASSERT_TRUE(Builder.compile(&Ctx)); + + auto Logger = std::make_shared(); + auto &Logging = zen::utils::Logging::getInstance(); + auto PreviousLogger = Logging.getLogger(); + Logging.setLogger(Logger); + Builder.dumpMemoryCompileStats(); + Logging.setLogger(std::move(PreviousLogger)); + + auto Summary = std::find_if( + Logger->DebugMessages.begin(), Logger->DebugMessages.end(), + [](const std::string &Message) { + return Message.find("[EVM-MEM-SUMMARY]") != std::string::npos; + }); + ASSERT_NE(Summary, Logger->DebugMessages.end()); + EXPECT_NE(Summary->find("copy_guaranteed_elision=1 "), std::string::npos) + << *Summary; +} +#endif + TEST(EVMMemoryAnalysisViewTest, ClassifiesBarriersFromMemoryOps) { const std::vector Bytecode = {OP_PUSH1, 0x00, OP_MLOAD, OP_PUSH1, 0x00, OP_MSIZE, OP_PUSH1, 0x00,