diff --git a/.github/workflows/_build-binaries.yaml b/.github/workflows/_build-binaries.yaml index a7bf7e5f..4824d6f8 100644 --- a/.github/workflows/_build-binaries.yaml +++ b/.github/workflows/_build-binaries.yaml @@ -19,7 +19,7 @@ on: description: "Base Reth Node version to build" required: false type: string - default: "main" + default: "feature/load-test-benchmark" # Set minimal permissions for all jobs by default permissions: @@ -155,6 +155,8 @@ jobs: - name: Set up Rust uses: actions-rust-lang/setup-rust-toolchain@9399c7bb15d4c7d47b27263d024f0a4978346ba4 # v1.11.0 + with: + cache: false - name: Cache base-reth-node binaries uses: actions/cache@2f8e54208210a422b2efd51efaa6bd6d7ca8920f # v3.4.3 @@ -163,9 +165,10 @@ jobs: path: | ~/bin/base-reth-node ~/bin/builder - key: ${{ runner.os }}-base-reth-node-builder-${{ inputs.base_reth_node_version }} + ~/bin/base-load-test + key: ${{ runner.os }}-base-reth-node-builder-load-test-${{ inputs.base_reth_node_version }} - - name: Build base-reth-node and base-builder + - name: Build base-reth-node, base-builder, and base-load-test if: steps.cache-base-reth-node.outputs.cache-hit != 'true' run: | unset CI @@ -188,6 +191,13 @@ jobs: path: ~/bin/builder retention-days: 1 + - name: Upload base-load-test artifact + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 + with: + name: base-load-test + path: ~/bin/base-load-test + retention-days: 1 + build-op-program: runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 1a7c75eb..21823488 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -18,7 +18,6 @@ jobs: with: optimism_version: 3019251e80aa248e91743addd3e833190acb26f1 geth_version: 6cbfcd5161083bcd4052edc3022d9f99c6fe40e0 - base_reth_node_version: main example-benchmarks: runs-on: ubuntu-latest diff --git a/.github/workflows/load-test.yaml b/.github/workflows/load-test.yaml new file mode 100644 index 00000000..de9d06a7 --- /dev/null +++ b/.github/workflows/load-test.yaml @@ -0,0 +1,96 @@ +name: Load Test + +on: + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build-binaries: + name: Build binaries + uses: ./.github/workflows/_build-binaries.yaml + + load-test: + name: Run load test benchmark + runs-on: ubuntu-latest + needs: [build-binaries] + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1 + with: + egress-policy: audit + + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Set up Go + uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0 + + - name: Download base-reth-node + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: base-reth-node + path: ${{ runner.temp }}/bin/ + + - name: Download builder + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: builder + path: ${{ runner.temp }}/bin/ + + - name: Download base-load-test + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: base-load-test + path: ${{ runner.temp }}/bin/ + + - name: Make binaries executable + run: chmod +x ${{ runner.temp }}/bin/* + + - name: Run load test benchmark + run: | + mkdir -p ${{ runner.temp }}/data-dir + mkdir -p ${{ runner.temp }}/output + + go run benchmark/cmd/main.go \ + --log.level info \ + run \ + --config configs/examples/load-test.yml \ + --root-dir ${{ runner.temp }}/data-dir \ + --output-dir ${{ runner.temp }}/output \ + --builder-bin ${{ runner.temp }}/bin/builder \ + --base-reth-node-bin ${{ runner.temp }}/bin/base-reth-node \ + --load-test-bin ${{ runner.temp }}/bin/base-load-test + + - name: Setup Node.js + if: always() + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "20" + + - name: Build Report + if: always() + run: | + cp -r ${{ runner.temp }}/output/ ./output/ || true + pushd report + npm install + npm run build + popd + + - name: Upload Output + if: always() + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 + with: + name: load-test-output + path: ${{ runner.temp }}/output/ + retention-days: 7 + + - name: Upload Report + if: always() + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 + with: + name: load-test-report + path: report/dist/ + retention-days: 7 diff --git a/benchmark/flags/flags.go b/benchmark/flags/flags.go index 8f86508b..368ab569 100644 --- a/benchmark/flags/flags.go +++ b/benchmark/flags/flags.go @@ -15,22 +15,24 @@ func prefixEnvVars(name string) []string { } const ( - ConfigFlagName = "config" - RootDirFlagName = "root-dir" - OutputDirFlagName = "output-dir" - TxFuzzBinFlagName = "tx-fuzz-bin" - ProxyPortFlagName = "proxy-port" - BenchmarkRunIDFlagName = "benchmark-run-id" - MachineTypeFlagName = "machine-type" - MachineProviderFlagName = "machine-provider" - MachineRegionFlagName = "machine-region" - FileSystemFlagName = "file-system" + ConfigFlagName = "config" + RootDirFlagName = "root-dir" + OutputDirFlagName = "output-dir" + TxFuzzBinFlagName = "tx-fuzz-bin" + LoadTestBinFlagName = "load-test-bin" + ProxyPortFlagName = "proxy-port" + BenchmarkRunIDFlagName = "benchmark-run-id" + MachineTypeFlagName = "machine-type" + MachineProviderFlagName = "machine-provider" + MachineRegionFlagName = "machine-region" + FileSystemFlagName = "file-system" ParallelTxBatchesFlagName = "parallel-tx-batches" ) // TxFuzz defaults const ( - DefaultTxFuzzBin = "../tx-fuzz/cmd/livefuzzer/livefuzzer" + DefaultTxFuzzBin = "../tx-fuzz/cmd/livefuzzer/livefuzzer" + DefaultLoadTestBin = "./base-load-test" ) var ( @@ -62,6 +64,13 @@ var ( EnvVars: opservice.PrefixEnvVar(EnvVarPrefix, "TX_FUZZ_BIN"), } + LoadTestBinFlag = &cli.StringFlag{ + Name: LoadTestBinFlagName, + Usage: "Load test binary path", + Value: DefaultLoadTestBin, + EnvVars: opservice.PrefixEnvVar(EnvVarPrefix, "LOAD_TEST_BIN"), + } + ProxyPortFlag = &cli.IntFlag{ Name: "proxy-port", Usage: "Proxy port", @@ -116,6 +125,7 @@ var RunFlags = []cli.Flag{ RootDirFlag, OutputDirFlag, TxFuzzBinFlag, + LoadTestBinFlag, ProxyPortFlag, BenchmarkRunIDFlag, MachineTypeFlag, diff --git a/clients/build-base-reth-node.sh b/clients/build-base-reth-node.sh index 98d030bb..7341be5f 100755 --- a/clients/build-base-reth-node.sh +++ b/clients/build-base-reth-node.sh @@ -40,12 +40,13 @@ fi # Checkout specified version/commit echo "Checking out version: $BASE_RETH_NODE_VERSION" -git checkout -f "$BASE_RETH_NODE_VERSION" +git fetch origin "$BASE_RETH_NODE_VERSION" || true +git checkout -f "$BASE_RETH_NODE_VERSION" || git checkout -f "origin/$BASE_RETH_NODE_VERSION" # Build the binaries using cargo -echo "Building base-reth-node and base-builder with cargo..." -# Build with maxperf profile +echo "Building base-reth-node, base-builder, and base-load-test with cargo..." cargo build --bin base-reth-node --bin base-builder --profile maxperf +cargo build -p base-load-tests --bin base-load-test --profile maxperf # Copy binaries to output directory echo "Copying binaries to output directory..." @@ -74,4 +75,11 @@ else exit 1 fi -echo "base-reth-node and base-builder binaries built successfully and placed in $FINAL_OUTPUT_DIR/" +if [ -f "target/maxperf/base-load-test" ]; then + cp target/maxperf/base-load-test "$FINAL_OUTPUT_DIR/" +else + echo "No base-load-test binary found" + exit 1 +fi + +echo "Binaries built successfully and placed in $FINAL_OUTPUT_DIR/" diff --git a/clients/versions.env b/clients/versions.env index 536ac9a8..3427491a 100644 --- a/clients/versions.env +++ b/clients/versions.env @@ -12,7 +12,7 @@ GETH_VERSION="v1.101604.0" # Base Reth Node Configuration BASE_RETH_NODE_REPO="https://github.com/base/base" -BASE_RETH_NODE_VERSION="main" +BASE_RETH_NODE_VERSION="feature/load-test-benchmark" # Build Configuration # BUILD_DIR="./build" diff --git a/configs/examples/load-test.yml b/configs/examples/load-test.yml new file mode 100644 index 00000000..6743ca60 --- /dev/null +++ b/configs/examples/load-test.yml @@ -0,0 +1,29 @@ +name: Load test throughput test +description: Test builder throughput using base-load-test binary as transaction generator +payloads: + - name: Load Test + type: load-test + id: load-test + sender_count: 10 + transactions: + - weight: 70 + type: transfer + - weight: 20 + type: calldata + max_size: 256 + - weight: 10 + type: precompile + target: sha256 + +benchmarks: + - variables: + - type: payload + value: load-test + - type: node_type + value: builder + - type: validator_node_type + value: base-reth-node + - type: num_blocks + value: 10 + - type: gas_limit + value: 1000000000 diff --git a/runner/clients/common/proxy/proxy.go b/runner/clients/common/proxy/proxy.go index 3da48b7d..f261c39e 100644 --- a/runner/clients/common/proxy/proxy.go +++ b/runner/clients/common/proxy/proxy.go @@ -22,7 +22,6 @@ import ( "github.com/ethereum/go-ethereum/common" ethTypes "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" ) type ProxyServer struct { @@ -196,11 +195,13 @@ func (p *ProxyServer) OverrideRequest(method string, rawParams json.RawMessage) return false, nil, fmt.Errorf("failed to decode hex: %w", err) } - err = rlp.DecodeBytes(rawTxBytes, &tx) + // Use UnmarshalBinary to support both legacy and typed (EIP-2718) transactions. + // The previous rlp.DecodeBytes only handled legacy transactions. + err = tx.UnmarshalBinary(rawTxBytes) if err != nil { - p.log.Error("failed to decode RLP", "err", err) - return false, nil, fmt.Errorf("failed to decode RLP: %w", err) + p.log.Error("failed to decode transaction", "err", err) + return false, nil, fmt.Errorf("failed to decode transaction: %w", err) } p.pendingTxs = append(p.pendingTxs, &tx) diff --git a/runner/config/config.go b/runner/config/config.go index c2890c42..0386f86f 100644 --- a/runner/config/config.go +++ b/runner/config/config.go @@ -20,6 +20,7 @@ type Config interface { DataDir() string OutputDir() string TxFuzzBinary() string + LoadTestBinary() string ProxyPort() int BenchmarkRunID() string MachineType() string @@ -30,36 +31,38 @@ type Config interface { } type config struct { - logConfig oplog.CLIConfig - configPath string - dataDir string - outputDir string - clientOptions ClientOptions - txFuzzBinary string - proxyPort int - benchmarkRunID string - machineType string - machineProvider string - machineRegion string - fileSystem string + logConfig oplog.CLIConfig + configPath string + dataDir string + outputDir string + clientOptions ClientOptions + txFuzzBinary string + loadTestBinary string + proxyPort int + benchmarkRunID string + machineType string + machineProvider string + machineRegion string + fileSystem string parallelTxBatches int } func NewConfig(ctx *cli.Context) Config { return &config{ - logConfig: oplog.ReadCLIConfig(ctx), - configPath: ctx.String(appFlags.ConfigFlagName), - dataDir: ctx.String(appFlags.RootDirFlagName), - outputDir: ctx.String(appFlags.OutputDirFlagName), - txFuzzBinary: ctx.String(appFlags.TxFuzzBinFlagName), - proxyPort: ctx.Int(appFlags.ProxyPortFlagName), - benchmarkRunID: ctx.String(appFlags.BenchmarkRunIDFlagName), - machineType: ctx.String(appFlags.MachineTypeFlagName), - machineProvider: ctx.String(appFlags.MachineProviderFlagName), - machineRegion: ctx.String(appFlags.MachineRegionFlagName), - fileSystem: ctx.String(appFlags.FileSystemFlagName), + logConfig: oplog.ReadCLIConfig(ctx), + configPath: ctx.String(appFlags.ConfigFlagName), + dataDir: ctx.String(appFlags.RootDirFlagName), + outputDir: ctx.String(appFlags.OutputDirFlagName), + txFuzzBinary: ctx.String(appFlags.TxFuzzBinFlagName), + loadTestBinary: ctx.String(appFlags.LoadTestBinFlagName), + proxyPort: ctx.Int(appFlags.ProxyPortFlagName), + benchmarkRunID: ctx.String(appFlags.BenchmarkRunIDFlagName), + machineType: ctx.String(appFlags.MachineTypeFlagName), + machineProvider: ctx.String(appFlags.MachineProviderFlagName), + machineRegion: ctx.String(appFlags.MachineRegionFlagName), + fileSystem: ctx.String(appFlags.FileSystemFlagName), parallelTxBatches: ctx.Int(appFlags.ParallelTxBatchesFlagName), - clientOptions: ReadClientOptions(ctx), + clientOptions: ReadClientOptions(ctx), } } @@ -112,6 +115,10 @@ func (c *config) TxFuzzBinary() string { return c.txFuzzBinary } +func (c *config) LoadTestBinary() string { + return c.loadTestBinary +} + func (c *config) BenchmarkRunID() string { return c.benchmarkRunID } diff --git a/runner/payload/factory.go b/runner/payload/factory.go index dd1d3102..dd621a99 100644 --- a/runner/payload/factory.go +++ b/runner/payload/factory.go @@ -7,6 +7,7 @@ import ( clienttypes "github.com/base/base-bench/runner/clients/types" benchtypes "github.com/base/base-bench/runner/network/types" "github.com/base/base-bench/runner/payload/contract" + "github.com/base/base-bench/runner/payload/loadtest" "github.com/base/base-bench/runner/payload/simulator" "github.com/base/base-bench/runner/payload/transferonly" "github.com/base/base-bench/runner/payload/txfuzz" @@ -31,6 +32,13 @@ func NewPayloadWorker(ctx context.Context, log log.Logger, testConfig *benchtype case "tx-fuzz": worker, err = txfuzz.NewTxFuzzPayloadWorker( log, sequencerClient.ClientURL(), params, privateKey, amount, config.TxFuzzBinary(), genesis.Config.ChainID) + case "load-test": + def, _ := definition.Params.(*loadtest.LoadTestPayloadDefinition) + if def == nil { + def = &loadtest.LoadTestPayloadDefinition{} + } + worker, err = loadtest.NewLoadTestPayloadWorker( + log, sequencerClient.ClientURL(), params, privateKey, amount, config.LoadTestBinary(), genesis.Config.ChainID, *def) case "transfer-only": worker, err = transferonly.NewTransferPayloadWorker( ctx, log, sequencerClient.ClientURL(), params, privateKey, amount, &genesis, definition.Params) @@ -77,6 +85,8 @@ func (t *Definition) UnmarshalYAML(node *yaml.Node) error { params = &transferonly.TransferOnlyPayloadDefinition{} case "tx-fuzz": params = &txfuzz.TxFuzzPayloadDefinition{} + case "load-test": + params = &loadtest.LoadTestPayloadDefinition{} case "contract": params = &contract.ContractPayloadDefinition{} case "simulator": diff --git a/runner/payload/loadtest/load_test_worker.go b/runner/payload/loadtest/load_test_worker.go new file mode 100644 index 00000000..5ea9f1cf --- /dev/null +++ b/runner/payload/loadtest/load_test_worker.go @@ -0,0 +1,247 @@ +package loadtest + +import ( + "context" + "crypto/ecdsa" + cryptorand "crypto/rand" + "encoding/hex" + "fmt" + "math/big" + "os" + "os/exec" + + "github.com/base/base-bench/runner/clients/common/proxy" + "github.com/base/base-bench/runner/network/mempool" + "github.com/base/base-bench/runner/network/types" + "github.com/base/base-bench/runner/payload/worker" + "github.com/ethereum/go-ethereum/log" + "github.com/pkg/errors" + "gopkg.in/yaml.v3" +) + +const proxyPort = 8545 + +// LoadTestPayloadDefinition is the YAML payload params for the load-test type. +// Fields map directly to the Rust base-load-test config format. +// The `transactions` field is passed through as raw YAML to support the full +// Rust config schema (transfer, calldata, precompile, erc20, etc.). +type LoadTestPayloadDefinition struct { + SenderCount uint64 `yaml:"sender_count"` + FundingAmount string `yaml:"funding_amount"` + Transactions yaml.Node `yaml:"transactions"` +} + +// loadTestConfig is the YAML config written to a temp file for the load-test binary. +type loadTestConfig struct { + RPC string `yaml:"rpc"` + SenderCount uint64 `yaml:"sender_count"` + TargetGPS uint64 `yaml:"target_gps"` + Duration string `yaml:"duration"` + Seed uint64 `yaml:"seed"` + FundingAmount string `yaml:"funding_amount"` + Transactions yaml.Node `yaml:"transactions"` +} + +type loadTestPayloadWorker struct { + log log.Logger + prefundSK string + loadTestBin string + elRPCURL string + gasLimit uint64 + blockTimeSec uint64 + params LoadTestPayloadDefinition + mempool *mempool.StaticWorkloadMempool + proxyServer *proxy.ProxyServer + cmd *exec.Cmd + configFilePath string +} + +// NewLoadTestPayloadWorker creates a worker that runs the base-load-test binary +// as an external transaction generator, capturing transactions via a proxy server. +func NewLoadTestPayloadWorker( + log log.Logger, + elRPCURL string, + params types.RunParams, + prefundedPrivateKey ecdsa.PrivateKey, + prefundAmount *big.Int, + loadTestBin string, + chainID *big.Int, + definition LoadTestPayloadDefinition, +) (worker.Worker, error) { + mp := mempool.NewStaticWorkloadMempool(log, chainID) + ps := proxy.NewProxyServer(elRPCURL, log, proxyPort, mp) + + blockTimeSec := uint64(params.BlockTime.Seconds()) + if blockTimeSec == 0 { + blockTimeSec = 1 + } + + w := &loadTestPayloadWorker{ + log: log, + prefundSK: hex.EncodeToString(prefundedPrivateKey.D.Bytes()), + loadTestBin: loadTestBin, + elRPCURL: elRPCURL, + gasLimit: params.GasLimit, + blockTimeSec: blockTimeSec, + params: definition, + mempool: mp, + proxyServer: ps, + } + + return w, nil +} + +func (w *loadTestPayloadWorker) Mempool() mempool.FakeMempool { + return w.mempool +} + +func (w *loadTestPayloadWorker) Setup(ctx context.Context) error { + if err := w.proxyServer.Run(ctx); err != nil { + return errors.Wrap(err, "failed to run proxy server") + } + + configPath, err := w.writeConfig() + if err != nil { + return errors.Wrap(err, "failed to write load-test config") + } + w.configFilePath = configPath + + w.log.Info("Starting load test", "binary", w.loadTestBin, "config", configPath) + + cmd := exec.CommandContext(ctx, w.loadTestBin, configPath) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stdout + cmd.Env = append(os.Environ(), fmt.Sprintf("FUNDER_KEY=%s", w.prefundSK)) + + if err := cmd.Start(); err != nil { + return errors.Wrap(err, "failed to start load test binary") + } + w.cmd = cmd + + return nil +} + +func (w *loadTestPayloadWorker) Stop(ctx context.Context) error { + if w.cmd != nil && w.cmd.Process != nil { + w.log.Info("Stopping load test process", "pid", w.cmd.Process.Pid) + if err := w.cmd.Process.Kill(); err != nil { + w.log.Warn("failed to kill load test process", "err", err) + } else { + // Reap the process to avoid zombies. + _, _ = w.cmd.Process.Wait() + } + } + + w.proxyServer.Stop() + + if w.configFilePath != "" { + if err := os.Remove(w.configFilePath); err != nil { + w.log.Warn("failed to remove load-test config", "path", w.configFilePath, "err", err) + } + } + + return nil +} + +func (w *loadTestPayloadWorker) SendTxs(ctx context.Context) error { + w.log.Info("Collecting txs from load test") + pendingTxs := w.proxyServer.PendingTxs() + w.proxyServer.ClearPendingTxs() + + w.mempool.AddTransactions(pendingTxs) + return nil +} + +// defaultTransactions returns the default transaction mix as a yaml.Node. +func defaultTransactions() yaml.Node { + var node yaml.Node + // Default: 70% transfer, 20% calldata, 10% precompile + defaultYAML := ` +- weight: 70 + type: transfer +- weight: 20 + type: calldata + max_size: 256 +- weight: 10 + type: precompile + target: sha256 +` + if err := yaml.Unmarshal([]byte(defaultYAML), &node); err != nil { + panic(fmt.Sprintf("failed to parse default transactions YAML: %v", err)) + } + // yaml.Unmarshal wraps in a document node; return the inner sequence + if node.Kind == yaml.DocumentNode && len(node.Content) > 0 { + return *node.Content[0] + } + return node +} + +// randomSeed returns a cryptographically random uint64 seed. +func randomSeed() uint64 { + var b [8]byte + if _, err := cryptorand.Read(b[:]); err != nil { + return 42 + } + return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | + uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 +} + +// writeConfig generates a temporary YAML config file for the load-test binary +// with the RPC URL pointing to the proxy server. +func (w *loadTestPayloadWorker) writeConfig() (string, error) { + senderCount := w.params.SenderCount + if senderCount == 0 { + senderCount = 10 + } + + fundingAmount := w.params.FundingAmount + if fundingAmount == "" { + fundingAmount = "10000000000000000000" + } + + // Compute target GPS from gas limit and block time + targetGPS := w.gasLimit / w.blockTimeSec + + transactions := w.params.Transactions + if transactions.Kind == 0 { + transactions = defaultTransactions() + } + + config := loadTestConfig{ + RPC: fmt.Sprintf("http://localhost:%d", proxyPort), + SenderCount: senderCount, + TargetGPS: targetGPS, + Duration: "99999s", + Seed: randomSeed(), + FundingAmount: fundingAmount, + Transactions: transactions, + } + + data, err := yaml.Marshal(&config) + if err != nil { + return "", errors.Wrap(err, "failed to marshal load-test config") + } + + tmpFile, err := os.CreateTemp("", "load-test-config-*.yaml") + if err != nil { + return "", errors.Wrap(err, "failed to create temp config file") + } + + if _, err := tmpFile.Write(data); err != nil { + _ = tmpFile.Close() + return "", errors.Wrap(err, "failed to write temp config file") + } + + if err := tmpFile.Close(); err != nil { + return "", errors.Wrap(err, "failed to close temp config file") + } + + w.log.Info("Generated load-test config", + "sender_count", senderCount, + "target_gps", targetGPS, + "gas_limit", w.gasLimit, + "block_time_sec", w.blockTimeSec, + ) + + return tmpFile.Name(), nil +}