[#140] Add initial JMH benchmark baseline for agentscope-core

agentscope-core/pom.xml

@@ -34,16 +34,22 @@
         <java.version>17</java.version>
         <maven.compiler.source>17</maven.compiler.source>
         <maven.compiler.target>17</maven.compiler.target>
+        <jmh.version>1.37</jmh.version>
         <maven-gpg-plugin.version>3.2.8</maven-gpg-plugin.version>
         <maven-site-plugin.version>4.0.0-M13</maven-site-plugin.version>
         <maven-compiler-plugin.version>3.11.0</maven-compiler-plugin.version>
+        <exec-maven-plugin.version>3.5.0</exec-maven-plugin.version>
         <maven-surefire-plugin.version>3.1.2</maven-surefire-plugin.version>
         <maven-deploy-plugin.version>3.1.1</maven-deploy-plugin.version>
         <maven-source-plugin.version>3.3.1</maven-source-plugin.version>
         <flatten-maven-plugin.version>1.5.0</flatten-maven-plugin.version>
         <maven-javadoc-plugin.version>3.5.0</maven-javadoc-plugin.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <argLine>-Xms512m -Xmx1024m</argLine>
+        <benchmark.include>.*Benchmark.*</benchmark.include>
+        <benchmark.warmupIterations>2</benchmark.warmupIterations>
+        <benchmark.measurementIterations>3</benchmark.measurementIterations>
+        <benchmark.forks>1</benchmark.forks>
 
         <central.publishing.maven.version>0.7.0</central.publishing.maven.version>
     </properties>
@@ -140,5 +146,69 @@
             <groupId>com.networknt</groupId>
             <artifactId>json-schema-validator</artifactId>
         </dependency>
+
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-core</artifactId>
+            <version>${jmh.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-generator-annprocess</artifactId>
+            <version>${jmh.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <testAnnotationProcessorPaths>
+                        <path>
+                            <groupId>org.openjdk.jmh</groupId>
+                            <artifactId>jmh-generator-annprocess</artifactId>
+                            <version>${jmh.version}</version>
+                        </path>
+                    </testAnnotationProcessorPaths>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>exec-maven-plugin</artifactId>
+                <version>${exec-maven-plugin.version}</version>
+                <configuration>
+                    <classpathScope>test</classpathScope>
+                    <longClasspath>true</longClasspath>
+                    <executable>java</executable>
+                    <arguments>
+                        <argument>-Dbenchmark.include=${benchmark.include}</argument>
+                        <argument>-Dbenchmark.warmupIterations=${benchmark.warmupIterations}</argument>
+                        <argument>-Dbenchmark.measurementIterations=${benchmark.measurementIterations}</argument>
+                        <argument>-Dbenchmark.forks=${benchmark.forks}</argument>
+                        <argument>-cp</argument>
+                        <classpath/>
+                        <argument>io.agentscope.core.benchmark.BenchmarkLauncher</argument>
+                    </arguments>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>run-benchmarks</id>
+                        <goals>
+                            <goal>exec</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
 </project>
+
+
+
+
+

agentscope-core/src/test/java/io/agentscope/core/benchmark/BenchmarkLauncher.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2024-2026 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.agentscope.core.benchmark;
+
+import org.openjdk.jmh.results.format.ResultFormatType;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+/** Launcher for running the core JMH benchmark suite from Maven test classpath. */
+public final class BenchmarkLauncher {
+
+    private BenchmarkLauncher() {}
+
+    public static void main(String[] args) throws RunnerException {
+        String includePattern = System.getProperty("benchmark.include", ".*Benchmark.*");
+        int warmupIterations = Integer.getInteger("benchmark.warmupIterations", 2);
+        int measurementIterations = Integer.getInteger("benchmark.measurementIterations", 3);
+        int forks = Integer.getInteger("benchmark.forks", 1);
+
+        Options options =
+                new OptionsBuilder()
+                        .include(includePattern)
+                        .warmupIterations(warmupIterations)
+                        .measurementIterations(measurementIterations)
+                        .forks(forks)
+                        .resultFormat(ResultFormatType.JSON)
+                        .result("target/jmh-result.json")
+                        .build();
+
+        new Runner(options).run();
+    }
+}

agentscope-core/src/test/java/io/agentscope/core/benchmark/BenchmarkSupport.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2024-2026 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.agentscope.core.benchmark;
+
+import io.agentscope.core.ReActAgent;
+import io.agentscope.core.agent.AgentBase;
+import io.agentscope.core.agent.test.MockModel;
+import io.agentscope.core.agent.test.TestUtils;
+import io.agentscope.core.memory.InMemoryMemory;
+import io.agentscope.core.message.Msg;
+import io.agentscope.core.pipeline.MsgHub;
+import io.agentscope.core.pipeline.SequentialPipeline;
+import io.agentscope.core.tool.Toolkit;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+
+/** Shared fixtures and helpers for JMH benchmarks. */
+public final class BenchmarkSupport {
+
+    public static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(5);
+
+    private BenchmarkSupport() {}
+
+    public static ReActAgent createAgent(String name, String responseText) {
+        return ReActAgent.builder()
+                .name(name)
+                .sysPrompt("You are a benchmark agent.")
+                .model(new MockModel(responseText))
+                .toolkit(new Toolkit())
+                .memory(new InMemoryMemory())
+                .build();
+    }
+
+    public static Msg createInputMessage(String text) {
+        return TestUtils.createUserMessage("benchmark-user", text);
+    }
+
+    public static SequentialPipeline createSequentialPipeline(int agentCount) {
+        List<AgentBase> agents = new ArrayList<>();
+        for (int index = 0; index < agentCount; index++) {
+            agents.add(createAgent("PipelineAgent" + index, "pipeline-response-" + index));
+        }
+        return new SequentialPipeline(agents);
+    }
+
+    public static MsgHub createEnteredHub(int participantCount) {
+        List<AgentBase> participants = new ArrayList<>();
+        for (int index = 0; index < participantCount; index++) {
+            participants.add(createAgent("HubAgent" + index, "hub-response-" + index));
+        }
+
+        MsgHub hub = MsgHub.builder().name("BenchmarkHub").participants(participants).build();
+        hub.enter().block(DEFAULT_TIMEOUT);
+        return hub;
+    }
+}

agentscope-core/src/test/java/io/agentscope/core/benchmark/MsgHubBenchmark.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2024-2026 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.agentscope.core.benchmark;
+
+import io.agentscope.core.message.Msg;
+import io.agentscope.core.pipeline.MsgHub;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+/** Benchmarks for MsgHub lifecycle and single-message distribution. */
+public class MsgHubBenchmark {
+
+    @BenchmarkMode(Mode.SampleTime)
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    @Warmup(iterations = 2, time = 1)
+    @Measurement(iterations = 3, time = 1)
+    @State(Scope.Thread)
+    public static class LifecycleState {
+
+        @Benchmark
+        public MsgHub enterAndExitLifecycle() {
+            MsgHub hub = BenchmarkSupport.createEnteredHub(3);
+            hub.exit().block(BenchmarkSupport.DEFAULT_TIMEOUT);
+            return hub;
+        }
+    }
+
+    @BenchmarkMode({Mode.SampleTime, Mode.Throughput})
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    @Warmup(iterations = 2, time = 1)
+    @Measurement(iterations = 3, time = 1)
+    @State(Scope.Thread)
+    public static class BroadcastState {
+
+        private MsgHub hub;
+        private Msg message;
+
+        @Setup(Level.Iteration)
+        public void setUp() {
+            hub = BenchmarkSupport.createEnteredHub(3);
+            message = BenchmarkSupport.createInputMessage("Broadcast benchmark message.");
+        }
+
+        @TearDown(Level.Iteration)
+        public void tearDown() {
+            if (hub != null) {
+                hub.close();
+            }
+        }
+
+        @Benchmark
+        public Void singleMessageBroadcast() {
+            return hub.broadcast(message).block(BenchmarkSupport.DEFAULT_TIMEOUT);
+        }
+    }
+}

agentscope-core/src/test/java/io/agentscope/core/benchmark/README.md

@@ -0,0 +1,40 @@
+# AgentScope Core Benchmarks
+
+This directory contains the first-phase JMH benchmark harness for `agentscope-core`.
+
+## Included benchmarks
+
+- `ReActAgentBenchmark` �single-agent calls with a mock model
+- `SequentialPipelineBenchmark` �sequential pipeline orchestration over three agents
+- `MsgHubBenchmark` �MsgHub lifecycle and single-message distribution
+
+## Compile benchmark sources
+
+```bash
+mvn -pl agentscope-core -am -DskipTests test-compile
+```
+
+## Run the full benchmark suite
+
+```bash
+mvn -pl agentscope-core -am -DskipTests test-compile exec:java \
+  -Dexec.classpathScope=test \
+  -Dexec.mainClass=io.agentscope.core.benchmark.BenchmarkLauncher
+```
+
+## Run a focused benchmark quickly
+
+```bash
+mvn -pl agentscope-core -am -DskipTests test-compile exec:java \
+  -Dexec.classpathScope=test \
+  -Dexec.mainClass=io.agentscope.core.benchmark.BenchmarkLauncher \
+  -Dbenchmark.include=.*ReActAgentBenchmark.* \
+  -Dbenchmark.warmupIterations=1 \
+  -Dbenchmark.measurementIterations=1 \
+  -Dbenchmark.forks=1
+```
+
+Results are written to `agentscope-core/target/jmh-result.json`.
+
+
+

agentscope-core/src/test/java/io/agentscope/core/benchmark/ReActAgentBenchmark.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2024-2026 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.agentscope.core.benchmark;
+
+import io.agentscope.core.ReActAgent;
+import io.agentscope.core.message.Msg;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+/** Benchmarks for single-agent execution with a mock model. */
+@BenchmarkMode({Mode.SampleTime, Mode.Throughput})
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 2, time = 1)
+@Measurement(iterations = 3, time = 1)
+@State(Scope.Thread)
+public class ReActAgentBenchmark {
+
+    private ReActAgent agent;
+    private Msg input;
+
+    @Setup(Level.Iteration)
+    public void setUp() {
+        agent = BenchmarkSupport.createAgent("BenchmarkAgent", "agent-benchmark-response");
+        input = BenchmarkSupport.createInputMessage("Summarize the benchmark request.");
+    }
+
+    @Benchmark
+    public Msg singleAgentCall() {
+        return agent.call(input).block(BenchmarkSupport.DEFAULT_TIMEOUT);
+    }
+}