perf(evm): reuse CFG memory proofs for COPY expansion

src/compiler/evm_frontend/evm_memory_analysis.h

@@ -879,6 +879,9 @@ class MemoryGuaranteedMinBytesAnalysis {
     case MemoryOpKind::MStore:
     case MemoryOpKind::MStore8:
       return Op.Writes.size() == 1 && getIntervalEnd(Op.Writes[0], End);
+    case MemoryOpKind::CallDataCopy:
+    case MemoryOpKind::CodeCopy:
+      return Op.Writes.size() == 1 && getIntervalEnd(Op.Writes[0], End);
     case MemoryOpKind::MCopy: {
       if (Op.Reads.size() != 1 || Op.Writes.size() != 1) {
         return false;

src/compiler/evm_frontend/evm_mir_compiler.cpp

@@ -4740,7 +4740,11 @@ void EVMMirBuilder::handleCodeCopy(Operand DestOffsetComponents,
                                    Operand OffsetComponents,
                                    Operand SizeComponents) {
   const auto &RuntimeFunctions = getRuntimeFunctionTable();
-  preExpandCopyMemory(DestOffsetComponents, SizeComponents);
+  if (preExpandCopyMemory(DestOffsetComponents, SizeComponents)) {
+#ifdef ZEN_ENABLE_MULTIPASS_JIT_LOGGING
+    ++MemStats.CopyGuaranteedElisionCount;
+#endif
+  }
   MInstruction *Size = extractKnownU64LowOperand(SizeComponents);
   chargeWordCopyGasIR(Size);
   uint64_t Non64Value = std::numeric_limits<uint64_t>::max();
@@ -6789,9 +6793,20 @@ void EVMMirBuilder::preExpandKeccakTwoWordMemory(Operand &OffsetComponents) {
   expandMemoryIR(RequiredSize, Overflow);
 }
 
-void EVMMirBuilder::preExpandCopyMemory(Operand &DestOffsetComponents,
+bool EVMMirBuilder::preExpandCopyMemory(Operand &DestOffsetComponents,
                                         Operand &SizeComponents) {
+  const bool DestWasConstU64 = DestOffsetComponents.isConstU64();
+  const uint64_t ConstDest =
+      DestWasConstU64 ? DestOffsetComponents.getConstValue()[0] : 0;
+  const bool SizeWasConstU64 = SizeComponents.isConstU64();
+  const uint64_t ConstSize =
+      SizeWasConstU64 ? SizeComponents.getConstValue()[0] : 0;
+
   normalizeOffsetWithSize(DestOffsetComponents, SizeComponents);
+  if (DestWasConstU64 && SizeWasConstU64 &&
+      tryUseGuaranteedMinBytesExpansionElision(true, ConstDest, ConstSize)) {
+    return true;
+  }
 
   MType *I64Type = &Ctx.I64Type;
   MInstruction *DestOffset = extractKnownU64LowOperand(DestOffsetComponents);
@@ -6802,6 +6817,7 @@ void EVMMirBuilder::preExpandCopyMemory(Operand &DestOffsetComponents,
       false, CmpInstruction::Predicate::ICMP_ULT, I64Type, RequiredSize,
       DestOffset);
   expandMemoryIR(RequiredSize, Overflow);
+  return false;
 }
 
 void EVMMirBuilder::chargeWordCopyGasIR(MInstruction *Size) {
@@ -7214,7 +7230,11 @@ void EVMMirBuilder::handleCallDataCopy(Operand DestOffsetComponents,
                                        Operand SizeComponents) {
   const auto &RuntimeFunctions = getRuntimeFunctionTable();
   uint64_t Non64Value = std::numeric_limits<uint64_t>::max();
-  preExpandCopyMemory(DestOffsetComponents, SizeComponents);
+  if (preExpandCopyMemory(DestOffsetComponents, SizeComponents)) {
+#ifdef ZEN_ENABLE_MULTIPASS_JIT_LOGGING
+    ++MemStats.CopyGuaranteedElisionCount;
+#endif
+  }
   MInstruction *Size = extractKnownU64LowOperand(SizeComponents);
   chargeWordCopyGasIR(Size);
   normalizeOperandU64(OffsetComponents, &Non64Value);
@@ -7312,6 +7332,7 @@ typename EVMMirBuilder::Operand EVMMirBuilder::handleReturnDataSize() {
 bool EVMMirBuilder::hasMemoryCompileStats() const {
   return MemStats.MLoadExpandCount != 0 || MemStats.MStoreExpandCount != 0 ||
          MemStats.MStore8ExpandCount != 0 || MemStats.MCopyExpandCount != 0 ||
+         MemStats.CopyGuaranteedElisionCount != 0 ||
          MemStats.BlockConstPrecheckCount != 0 ||
          MemStats.MemoryExpansionPlanCount != 0 ||
          MemStats.LinearU64AddrFastPathCount != 0 ||
@@ -7674,7 +7695,8 @@ void EVMMirBuilder::dumpMemoryCompileStats() const {
   ZEN_LOG_DEBUG(
       "[EVM-MEM-SUMMARY] mload_expand=%llu mstore_expand=%llu "
       "mstore8_expand=%llu mcopy_expand=%llu "
-      "mcopy_guaranteed_elision=%llu memory_dse_candidates=%llu "
+      "mcopy_guaranteed_elision=%llu copy_guaranteed_elision=%llu "
+      "memory_dse_candidates=%llu "
       "memory_dse_eliminated_writes=%llu memory_load_forward_candidates=%llu "
       "memory_load_forwarded=%llu block_const_precheck=%llu "
       "block_linear_precheck=%llu prechecked_mload_ops=%llu "
@@ -7768,6 +7790,7 @@ void EVMMirBuilder::dumpMemoryCompileStats() const {
       static_cast<unsigned long long>(MemStats.MStore8ExpandCount),
       static_cast<unsigned long long>(MemStats.MCopyExpandCount),
       static_cast<unsigned long long>(MemStats.MCopyGuaranteedElisionCount),
+      static_cast<unsigned long long>(MemStats.CopyGuaranteedElisionCount),
       static_cast<unsigned long long>(MemStats.MemoryDSEStoreCandidates),
       static_cast<unsigned long long>(MemStats.MemoryDSEEliminatedWrites),
       static_cast<unsigned long long>(MemStats.MemoryLoadForwardCandidates),
@@ -8639,6 +8662,7 @@ void EVMMirBuilder::noteMemoryOpcodeInBlock(evmc_opcode Opcode, uint64_t PC) {
 }
 
 void EVMMirBuilder::noteHelperOpcodeInBlock(evmc_opcode Opcode, uint64_t PC) {
+  CurrentMemoryOpPC = PC;
   if (!CurBlockMemStats.Active) {
     return;
   }

src/compiler/evm_frontend/evm_mir_compiler.h

@@ -1458,6 +1458,7 @@ class EVMMirBuilder final {
     uint64_t MStore8ExpandCount = 0;
     uint64_t MCopyExpandCount = 0;
     uint64_t MCopyGuaranteedElisionCount = 0;
+    uint64_t CopyGuaranteedElisionCount = 0;
     uint64_t MemoryDSEStoreCandidates = 0;
     uint64_t MemoryDSEEliminatedWrites = 0;
     uint64_t MemoryLoadForwardCandidates = 0;
@@ -1883,7 +1884,7 @@ class EVMMirBuilder final {
   void reloadMemorySizeFromInstance();
   void expandMemoryIR(MInstruction *RequiredSize, MInstruction *Overflow);
   void preExpandKeccakTwoWordMemory(Operand &OffsetComponents);
-  void preExpandCopyMemory(Operand &DestOffsetComponents,
+  bool preExpandCopyMemory(Operand &DestOffsetComponents,
                            Operand &SizeComponents);
   void chargeWordCopyGasIR(MInstruction *Size);
   void chargeDynamicGasIR(MInstruction *GasCost);

src/tests/evm_jit_frontend_tests.cpp

@@ -3,6 +3,7 @@
 
 #include "action/evm_bytecode_visitor.h"
 #include "compiler/evm_frontend/evm_analyzer.h"
+#include "compiler/evm_frontend/evm_imported.h"
 #include "compiler/evm_frontend/evm_memory_analysis.h"
 #include "compiler/evm_frontend/evm_memory_facts.h"
 #include "compiler/evm_frontend/evm_memory_grouping.h"
@@ -32,6 +33,23 @@ using COMPILER::EVMValueRange;
 using zen::common::BinaryOperator;
 using zen::common::CompareOperator;
 
+#ifdef ZEN_ENABLE_MULTIPASS_JIT_LOGGING
+class CapturingLogger final : public zen::utils::ILogger {
+public:
+  void trace(const std::string &, const char *, int, const char *) override {}
+  void debug(const std::string &Message, const char *, int,
+             const char *) override {
+    DebugMessages.push_back(Message);
+  }
+  void info(const std::string &, const char *, int, const char *) override {}
+  void warn(const std::string &, const char *, int, const char *) override {}
+  void error(const std::string &, const char *, int, const char *) override {}
+  void fatal(const std::string &, const char *, int, const char *) override {}
+
+  std::vector<std::string> DebugMessages;
+};
+#endif
+
 EVMAnalyzer analyzeBytecode(const std::vector<uint8_t> &Bytecode) {
   EVMAnalyzer Analyzer(EVMC_CANCUN);
   const uint8_t *Data = Bytecode.empty() ? nullptr : Bytecode.data();
@@ -123,6 +141,51 @@ void expectPCList(const std::vector<uint64_t> &Actual,
   }
 }
 
+#if defined(ZEN_ENABLE_MULTIPASS_JIT_LOGGING) &&                               \
+    defined(ZEN_ENABLE_EVM_MEMORY_PLAN_FRAMEWORK)
+size_t countRuntimeCalls(const COMPILER::MFunction &Func, uint64_t Address) {
+  std::string Mir;
+  llvm::raw_string_ostream OS(Mir);
+  Func.print(OS);
+  OS.flush();
+
+  const std::string Needle =
+      "target = const.i64 " + std::to_string(Address) + ", ";
+  size_t Count = 0;
+  for (size_t Pos = Mir.find(Needle); Pos != std::string::npos;
+       Pos = Mir.find(Needle, Pos + Needle.size())) {
+    ++Count;
+  }
+  return Count;
+}
+
+std::optional<size_t>
+countExpandMemoryCallsForBytecode(const std::vector<uint8_t> &Bytecode) {
+  COMPILER::EVMFrontendContext Ctx;
+  Ctx.setRevision(EVMC_CANCUN);
+  Ctx.setBytecode(reinterpret_cast<const zen::common::Byte *>(Bytecode.data()),
+                  Bytecode.size());
+
+  COMPILER::MModule Mod(Ctx);
+  std::array<COMPILER::MType *, 1> ParamTypes = {
+      COMPILER::MPointerType::create(Ctx, Ctx.VoidType)};
+  COMPILER::MFunctionType *FuncType = COMPILER::MFunctionType::create(
+      Ctx, Ctx.VoidType, llvm::ArrayRef<COMPILER::MType *>(ParamTypes));
+  Mod.addFuncType(FuncType);
+
+  COMPILER::MFunction Func(Ctx, 0);
+  Func.setFunctionType(FuncType);
+  EVMMirBuilder Builder(Ctx, Func);
+  if (!Builder.compile(&Ctx)) {
+    return std::nullopt;
+  }
+
+  const auto &RuntimeFunctions = COMPILER::getRuntimeFunctionTable();
+  return countRuntimeCalls(
+      Func, COMPILER::getFunctionAddress(RuntimeFunctions.ExpandMemoryNoGas));
+}
+#endif
+
 void appendPushU64(std::vector<uint8_t> &Bytecode, uint64_t Value) {
   if (Value == 0) {
     Bytecode.push_back(OP_PUSH0);
@@ -408,6 +471,44 @@ TEST(EVMMemoryGuaranteedMinBytesAnalysisTest, IgnoresZeroLengthMCopy) {
   EXPECT_EQ(Guaranteed.getGuaranteedMinBytesBeforeOp(Facts.Ops[1].Id), 0u);
 }
 
+TEST(EVMMemoryGuaranteedMinBytesAnalysisTest,
+     LearnsConstCallDataCopyDestinationEnd) {
+  const std::vector<uint8_t> Bytecode = {
+      OP_PUSH1,        0x20,     OP_PUSH1, 0x00,    OP_PUSH1, 0x80,
+      OP_CALLDATACOPY, OP_PUSH1, 0x80,     OP_MLOAD};
+
+  COMPILER::MemoryFacts Facts = collectMemoryFacts(Bytecode);
+  COMPILER::MemoryGuaranteedMinBytesAnalysis Guaranteed(Facts);
+
+  ASSERT_EQ(Facts.Ops.size(), 2u);
+  EXPECT_EQ(Guaranteed.getGuaranteedMinBytesBeforeOp(Facts.Ops[1].Id), 0xa0u);
+}
+
+TEST(EVMMemoryGuaranteedMinBytesAnalysisTest,
+     LearnsConstCodeCopyDestinationEnd) {
+  const std::vector<uint8_t> Bytecode = {
+      OP_PUSH1, 0x20,        OP_PUSH1, 0x00, OP_PUSH1,
+      0x40,     OP_CODECOPY, OP_PUSH1, 0x40, OP_MLOAD};
+
+  COMPILER::MemoryFacts Facts = collectMemoryFacts(Bytecode);
+  COMPILER::MemoryGuaranteedMinBytesAnalysis Guaranteed(Facts);
+
+  ASSERT_EQ(Facts.Ops.size(), 2u);
+  EXPECT_EQ(Guaranteed.getGuaranteedMinBytesBeforeOp(Facts.Ops[1].Id), 0x60u);
+}
+
+TEST(EVMMemoryGuaranteedMinBytesAnalysisTest, IgnoresZeroLengthCallDataCopy) {
+  const std::vector<uint8_t> Bytecode = {
+      OP_PUSH1,        0x00,     OP_PUSH1, 0x00,    OP_PUSH1, 0x80,
+      OP_CALLDATACOPY, OP_PUSH1, 0x00,     OP_MLOAD};
+
+  COMPILER::MemoryFacts Facts = collectMemoryFacts(Bytecode);
+  COMPILER::MemoryGuaranteedMinBytesAnalysis Guaranteed(Facts);
+
+  ASSERT_EQ(Facts.Ops.size(), 2u);
+  EXPECT_EQ(Guaranteed.getGuaranteedMinBytesBeforeOp(Facts.Ops[1].Id), 0u);
+}
+
 TEST(EVMMemoryFactsBuilderTest, RecordsCopyAddressSpaces) {
   const std::vector<uint8_t> Bytecode = {OP_PUSH1, 0x20, OP_PUSH1,       0x04,
                                          OP_PUSH1, 0x80, OP_CALLDATACOPY};
@@ -430,6 +531,150 @@ TEST(EVMMemoryFactsBuilderTest, RecordsCopyAddressSpaces) {
   EXPECT_EQ(Op.Writes[0].Size.Value, 0x20u);
 }
 
+#if defined(ZEN_ENABLE_MULTIPASS_JIT_LOGGING) &&                               \
+    defined(ZEN_ENABLE_EVM_MEMORY_PLAN_FRAMEWORK)
+TEST(EVMMirBuilderMemoryStatsTest,
+     ReusesCrossBlockGuaranteedSizeForCopyConsumers) {
+  const std::vector<uint8_t> Bytecode = {
+      0x60, 0x01, // PUSH1 value
+      0x60, 0x80, // PUSH1 memory offset
+      0x52,       // MSTORE: guarantees memory through 0x9f
+      0x60, 0x08, // PUSH1 successor block
+      0x56,       // JUMP
+      0x5b,       // JUMPDEST
+      0x60, 0x20, // PUSH1 CALLDATACOPY size
+      0x60, 0x00, // PUSH1 calldata source
+      0x60, 0x80, // PUSH1 memory destination
+      0x37,       // CALLDATACOPY: covered by the predecessor proof
+      0x60, 0x20, // PUSH1 CODECOPY size
+      0x60, 0x00, // PUSH1 code source
+      0x60, 0x40, // PUSH1 memory destination
+      0x39,       // CODECOPY: covered by the predecessor proof
+      0x60, 0x20, // PUSH1 CODECOPY size
+      0x60, 0x00, // PUSH1 code source
+      0x60, 0xa0, // PUSH1 memory destination
+      0x39,       // CODECOPY: extends beyond the predecessor proof
+      0x00        // STOP
+  };
+
+  COMPILER::EVMFrontendContext Ctx;
+  Ctx.setRevision(EVMC_CANCUN);
+  Ctx.setBytecode(reinterpret_cast<const zen::common::Byte *>(Bytecode.data()),
+                  Bytecode.size());
+
+  COMPILER::MModule Mod(Ctx);
+  std::array<COMPILER::MType *, 1> ParamTypes = {
+      COMPILER::MPointerType::create(Ctx, Ctx.VoidType)};
+  COMPILER::MFunctionType *FuncType = COMPILER::MFunctionType::create(
+      Ctx, Ctx.VoidType, llvm::ArrayRef<COMPILER::MType *>(ParamTypes));
+  Mod.addFuncType(FuncType);
+
+  COMPILER::MFunction Func(Ctx, 0);
+  Func.setFunctionType(FuncType);
+  EVMMirBuilder Builder(Ctx, Func);
+  ASSERT_TRUE(Builder.compile(&Ctx));
+
+  auto Logger = std::make_shared<CapturingLogger>();
+  auto &Logging = zen::utils::Logging::getInstance();
+  auto PreviousLogger = Logging.getLogger();
+  Logging.setLogger(Logger);
+  Builder.dumpMemoryCompileStats();
+  Logging.setLogger(std::move(PreviousLogger));
+
+  auto Summary = std::find_if(
+      Logger->DebugMessages.begin(), Logger->DebugMessages.end(),
+      [](const std::string &Message) {
+        return Message.find("[EVM-MEM-SUMMARY]") != std::string::npos;
+      });
+  ASSERT_NE(Summary, Logger->DebugMessages.end());
+  EXPECT_NE(Summary->find("copy_guaranteed_elision=2 "), std::string::npos)
+      << *Summary;
+  EXPECT_EQ(Summary->find("copy_guaranteed_elision=3 "), std::string::npos)
+      << *Summary;
+}
+
+TEST(EVMMirBuilderMemoryStatsTest,
+     TracksCopyProofAtEachOpcodePCAndRetainsFallbackExpansion) {
+  const std::vector<uint8_t> ProducerOnly = {0x60, 0x01, 0x60, 0x80, 0x52,
+                                             0x60, 0x08, 0x56, 0x5b, 0x00};
+  const std::vector<uint8_t> WithUncoveredCopy = {
+      0x60, 0x01, // PUSH1 value
+      0x60, 0x80, // PUSH1 memory offset
+      0x52,       // MSTORE: guarantees memory through 0x9f
+      0x60, 0x08, // PUSH1 successor block
+      0x56,       // JUMP
+      0x5b,       // JUMPDEST
+      0x60, 0x20, // PUSH1 CODECOPY size
+      0x60, 0x00, // PUSH1 code source
+      0x60, 0xa0, // PUSH1 memory destination
+      0x39,       // CODECOPY: uncovered, grows memory through 0xbf
+      0x00        // STOP
+  };
+  const std::vector<uint8_t> Bytecode = {
+      0x60, 0x01, // PUSH1 value
+      0x60, 0x80, // PUSH1 memory offset
+      0x52,       // MSTORE: guarantees memory through 0x9f
+      0x60, 0x08, // PUSH1 successor block
+      0x56,       // JUMP
+      0x5b,       // JUMPDEST
+      0x60, 0x20, // PUSH1 CODECOPY size
+      0x60, 0x00, // PUSH1 code source
+      0x60, 0xa0, // PUSH1 memory destination
+      0x39,       // CODECOPY: uncovered, grows memory through 0xbf
+      0x60, 0x20, // PUSH1 CALLDATACOPY size
+      0x60, 0x00, // PUSH1 calldata source
+      0x60, 0xa0, // PUSH1 memory destination
+      0x37,       // CALLDATACOPY: covered by the preceding COPY proof
+      0x00        // STOP
+  };
+
+  const std::optional<size_t> ProducerExpansionCalls =
+      countExpandMemoryCallsForBytecode(ProducerOnly);
+  const std::optional<size_t> FallbackExpansionCalls =
+      countExpandMemoryCallsForBytecode(WithUncoveredCopy);
+  const std::optional<size_t> CoveredSuccessorExpansionCalls =
+      countExpandMemoryCallsForBytecode(Bytecode);
+  ASSERT_TRUE(ProducerExpansionCalls.has_value());
+  ASSERT_TRUE(FallbackExpansionCalls.has_value());
+  ASSERT_TRUE(CoveredSuccessorExpansionCalls.has_value());
+  EXPECT_EQ(*FallbackExpansionCalls, *ProducerExpansionCalls + 1);
+  EXPECT_EQ(*CoveredSuccessorExpansionCalls, *FallbackExpansionCalls);
+
+  COMPILER::EVMFrontendContext Ctx;
+  Ctx.setRevision(EVMC_CANCUN);
+  Ctx.setBytecode(reinterpret_cast<const zen::common::Byte *>(Bytecode.data()),
+                  Bytecode.size());
+
+  COMPILER::MModule Mod(Ctx);
+  std::array<COMPILER::MType *, 1> ParamTypes = {
+      COMPILER::MPointerType::create(Ctx, Ctx.VoidType)};
+  COMPILER::MFunctionType *FuncType = COMPILER::MFunctionType::create(
+      Ctx, Ctx.VoidType, llvm::ArrayRef<COMPILER::MType *>(ParamTypes));
+  Mod.addFuncType(FuncType);
+
+  COMPILER::MFunction Func(Ctx, 0);
+  Func.setFunctionType(FuncType);
+  EVMMirBuilder Builder(Ctx, Func);
+  ASSERT_TRUE(Builder.compile(&Ctx));
+
+  auto Logger = std::make_shared<CapturingLogger>();
+  auto &Logging = zen::utils::Logging::getInstance();
+  auto PreviousLogger = Logging.getLogger();
+  Logging.setLogger(Logger);
+  Builder.dumpMemoryCompileStats();
+  Logging.setLogger(std::move(PreviousLogger));
+
+  auto Summary = std::find_if(
+      Logger->DebugMessages.begin(), Logger->DebugMessages.end(),
+      [](const std::string &Message) {
+        return Message.find("[EVM-MEM-SUMMARY]") != std::string::npos;
+      });
+  ASSERT_NE(Summary, Logger->DebugMessages.end());
+  EXPECT_NE(Summary->find("copy_guaranteed_elision=1 "), std::string::npos)
+      << *Summary;
+}
+#endif
+
 TEST(EVMMemoryAnalysisViewTest, ClassifiesBarriersFromMemoryOps) {
   const std::vector<uint8_t> Bytecode = {OP_PUSH1, 0x00,     OP_MLOAD, OP_PUSH1,
                                          0x00,     OP_MSIZE, OP_PUSH1, 0x00,