diff --git a/.github/workflows/performance.yaml b/.github/workflows/performance.yaml new file mode 100644 index 0000000..e99b303 --- /dev/null +++ b/.github/workflows/performance.yaml @@ -0,0 +1,30 @@ +name: Manual Performance Benchmarks + +on: + workflow_dispatch: + inputs: + benchmark: + description: "JMH benchmark regex" + required: true + default: ".*" + +jobs: + jmh: + runs-on: + - self-hosted + - mapsmith-perf + + steps: + - uses: actions/checkout@v6 + + - name: Run JMH + run: | + mkdir -p mapsmith-benchmarks/build/benchmarks + ./gradlew :mapsmith-benchmarks:jmh \ + --args="${{ inputs.benchmark }} -rf json -rff $GITHUB_WORKSPACE/mapsmith-benchmarks/build/benchmarks/jmh-results.json" + + - name: Upload JMH results + uses: actions/upload-artifact@v4 + with: + name: jmh-results + path: mapsmith-benchmarks/build/benchmarks/jmh-results.json diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index c091803..6ad9b1c 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -10,6 +10,7 @@ junit = "6.0.0" ktfmt = "0.61" maven-publish = "0.36.0" pmd = "7.25.0" +picocli = "4.7.7" spotbugs-plugin = "6.5.6" spotless = "8.6.0" @@ -18,6 +19,7 @@ assertj-core = { module = "org.assertj:assertj-core", version.ref = "assertj" } errorprone-core = { module = "com.google.errorprone:error_prone_core", version.ref = "errorprone" } jmh-core = { module = "org.openjdk.jmh:jmh-core", version.ref = "jmh" } jmh-generator-annprocess = { module = "org.openjdk.jmh:jmh-generator-annprocess", version.ref = "jmh" } +picocli = { module = "info.picocli:picocli", version.ref = "picocli" } jqwik = { module = "net.jqwik:jqwik", version.ref = "jqwik" } junit-bom = { module = "org.junit:junit-bom", version.ref = "junit" } junit-jupiter = { module = "org.junit.jupiter:junit-jupiter" } diff --git a/infra/perf-runner/.gitignore b/infra/perf-runner/.gitignore new file mode 100644 index 0000000..8a0c824 --- /dev/null +++ b/infra/perf-runner/.gitignore @@ -0,0 +1,6 @@ +.terraform/ +.terraform.lock.hcl +*.tfstate +*.tfstate.* +crash.log +crash.*.log diff --git a/infra/perf-runner/README.md b/infra/perf-runner/README.md new file mode 100644 index 0000000..bf4a81d --- /dev/null +++ b/infra/perf-runner/README.md @@ -0,0 +1,56 @@ +# Mapsmith performance runner + +This Terraform configuration creates a `c7i.xlarge` one-time Spot instance +(default region: `eu-west-2`) and configures it as a GitHub Actions +self-hosted runner for `mrk-andreev/mapsmith`, labelled `mapsmith-perf`. + +Prerequisites: + +- Terraform 1.5 or newer and AWS credentials for the selected region. +- GitHub CLI authenticated locally with permission to administer Actions + runners for the repository (`gh auth login`). +- A subnet with outbound HTTPS access. The default configuration uses a public + IP; set `associate_public_ip_address = false` when using a private subnet + with NAT egress. + +Run it from this directory: + +```sh +terraform init +terraform apply +``` + +Specify network placement when the account does not have a suitable default VPC: + +```sh +terraform apply -var='subnet_id=subnet-0123456789abcdef0' +``` + +Override the default `c7i.xlarge` instance type when needed: + +```sh +terraform apply -var='instance_type=c7i.2xlarge' +``` + +Override the default `eu-west-2` region when needed: + +```sh +terraform apply -var='aws_region=eu-central-1' +``` + +Destroy the instance when benchmarks are finished: + +```sh +terraform destroy +``` + +To recreate a runner deliberately, request replacement so Terraform obtains a +fresh registration token: + +```sh +terraform apply -replace=aws_instance.perf_runner +``` + +The registration token is deliberately obtained through `gh` at apply time. +Terraform records the resulting user data in its local state, so keep the +state file private and do not commit it. diff --git a/infra/perf-runner/github-runner-token.sh b/infra/perf-runner/github-runner-token.sh new file mode 100644 index 0000000..b4fb012 --- /dev/null +++ b/infra/perf-runner/github-runner-token.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +# The external provider supplies the JSON query on stdin. +repository="$(jq -r '.repository' < /dev/stdin)" + +if [[ -z "$repository" || "$repository" == "null" ]]; then + echo "repository is required" >&2 + exit 1 +fi + +# gh must already be authenticated to this repository with Actions administration access. +token="$(gh api --method POST "repos/${repository}/actions/runners/registration-token" --jq '.token')" +printf '{"token":"%s"}\n' "$token" diff --git a/infra/perf-runner/main.tf b/infra/perf-runner/main.tf new file mode 100644 index 0000000..971be7f --- /dev/null +++ b/infra/perf-runner/main.tf @@ -0,0 +1,90 @@ +terraform { + required_version = ">= 1.5.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + external = { + source = "hashicorp/external" + version = "~> 2.3" + } + } +} + +provider "aws" { + region = var.aws_region +} + +data "aws_ami" "ubuntu" { + most_recent = true + owners = ["099720109477"] # Canonical + + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*"] + } + + filter { + name = "architecture" + values = ["x86_64"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } +} + +data "external" "github_runner_token" { + program = ["bash", "${path.module}/github-runner-token.sh"] + + query = { + repository = var.repository + } +} + +resource "aws_instance" "perf_runner" { + ami = data.aws_ami.ubuntu.id + instance_type = var.instance_type + subnet_id = var.subnet_id + vpc_security_group_ids = var.security_group_ids + associate_public_ip_address = var.associate_public_ip_address + + instance_market_options { + market_type = "spot" + + spot_options { + spot_instance_type = "one-time" + instance_interruption_behavior = "terminate" + } + } + + root_block_device { + encrypted = true + volume_size = 30 + volume_type = "gp3" + } + + metadata_options { + http_endpoint = "enabled" + http_tokens = "required" + } + + user_data_replace_on_change = true + user_data = templatefile("${path.module}/user-data.sh.tftpl", { + repository = var.repository + runner_token = sensitive(data.external.github_runner_token.result.token) + }) + + # A registration token is newly generated on every plan. It is needed only + # during the first boot, so it must not cause a running runner to be replaced. + lifecycle { + ignore_changes = [user_data] + } + + tags = merge(var.tags, { + Name = "mapsmith-perf-runner" + }) +} diff --git a/infra/perf-runner/outputs.tf b/infra/perf-runner/outputs.tf new file mode 100644 index 0000000..6a7ed0d --- /dev/null +++ b/infra/perf-runner/outputs.tf @@ -0,0 +1,9 @@ +output "instance_id" { + description = "ID of the Spot-backed performance runner instance." + value = aws_instance.perf_runner.id +} + +output "public_ip" { + description = "Public IPv4 address, if one was assigned." + value = aws_instance.perf_runner.public_ip +} diff --git a/infra/perf-runner/user-data.sh.tftpl b/infra/perf-runner/user-data.sh.tftpl new file mode 100644 index 0000000..55531b8 --- /dev/null +++ b/infra/perf-runner/user-data.sh.tftpl @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get install -y --no-install-recommends ca-certificates curl git jq openjdk-21-jdk-headless + +id -u runner >/dev/null 2>&1 || useradd --create-home --shell /bin/bash runner + +install -d -o runner -g runner /opt/actions-runner +cd /opt/actions-runner + +runner_version="$$(curl --fail --silent --show-error --location \ + https://api.github.com/repos/actions/runner/releases/latest | jq -r '.tag_name')" +curl --fail --silent --show-error --location \ + --output actions-runner.tar.gz \ + "https://github.com/actions/runner/releases/download/$${runner_version}/actions-runner-linux-x64-$${runner_version#v}.tar.gz" +tar xzf actions-runner.tar.gz +rm actions-runner.tar.gz +chown -R runner:runner /opt/actions-runner +./bin/installdependencies.sh + +runuser -u runner -- ./config.sh --unattended \ + --url "https://github.com/${repository}" \ + --token '${runner_token}' \ + --name "$$(hostname)" \ + --labels "mapsmith-perf" \ + --replace + +./svc.sh install runner +./svc.sh start diff --git a/infra/perf-runner/variables.tf b/infra/perf-runner/variables.tf new file mode 100644 index 0000000..12558b3 --- /dev/null +++ b/infra/perf-runner/variables.tf @@ -0,0 +1,41 @@ +variable "repository" { + description = "GitHub repository that owns the self-hosted runner." + type = string + default = "mrk-andreev/mapsmith" +} + +variable "aws_region" { + description = "AWS region in which to create the performance runner." + type = string + default = "eu-west-2" +} + +variable "instance_type" { + description = "EC2 instance type for the performance runner." + type = string + default = "c7i.xlarge" +} + +variable "subnet_id" { + description = "Subnet in the selected AWS region in which to create the runner. Leave null to use the account default VPC subnet." + type = string + default = null +} + +variable "security_group_ids" { + description = "Security groups for the runner. No inbound rules are required; it needs outbound HTTPS access." + type = list(string) + default = null +} + +variable "associate_public_ip_address" { + description = "Whether the runner receives a public IPv4 address. Set false when the subnet has NAT egress." + type = bool + default = true +} + +variable "tags" { + description = "Additional AWS tags." + type = map(string) + default = {} +} diff --git a/mapsmith-benchmarks/build.gradle.kts b/mapsmith-benchmarks/build.gradle.kts index 738a1ec..7322d2e 100644 --- a/mapsmith-benchmarks/build.gradle.kts +++ b/mapsmith-benchmarks/build.gradle.kts @@ -19,6 +19,7 @@ dependencies { add(jmh.implementationConfigurationName, project(":mapsmith-core")) add(jmh.implementationConfigurationName, libs.jmh.core) + add(jmh.implementationConfigurationName, libs.picocli) add(jmh.annotationProcessorConfigurationName, libs.jmh.generator.annprocess) } diff --git a/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/BenchmarkRunner.java b/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/BenchmarkRunner.java new file mode 100644 index 0000000..43b8e55 --- /dev/null +++ b/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/BenchmarkRunner.java @@ -0,0 +1,82 @@ +package name.mrkandreev.mapsmith.benchmarks; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import org.openjdk.jmh.results.format.ResultFormatType; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import picocli.CommandLine; + +public enum BenchmarkRunner { + ; + + public static void run(Class cls, String[] args) throws RunnerException { + var command = new RunBenchmarkCommand(cls); + var commandLine = new CommandLine(command); + var parseResult = commandLine.parseArgs(args); + if (!CommandLine.printHelpIfRequested(parseResult)) { + command.run(); + } + } + + @CommandLine.Command( + name = "benchmark", + mixinStandardHelpOptions = true, + description = "Runs a JMH benchmark and writes its results as JSON.") + private static final class RunBenchmarkCommand { + private final Class benchmarkClass; + + @CommandLine.Option( + names = {"-o", "--output-dir"}, + defaultValue = "build/benchmarks", + paramLabel = "", + description = "Directory for JSON results (default: ${DEFAULT-VALUE}).") + private Path outputDirectory; + + @CommandLine.Option( + names = {"-p", "--profiler"}, + paramLabel = "", + description = + "JMH profiler, optionally followed by its parameters; may be specified more than once.") + private final List profilerSpecifications = new ArrayList<>(); + + private RunBenchmarkCommand(Class benchmarkClass) { + this.benchmarkClass = benchmarkClass; + } + + private void run() throws RunnerException { + try { + Files.createDirectories(outputDirectory); + } catch (IOException exception) { + throw new RunnerException( + "Could not create benchmark output directory: " + outputDirectory, exception); + } + + var optionsBuilder = new OptionsBuilder(); + optionsBuilder + .include(benchmarkClass.getSimpleName()) + .result(outputDirectory.resolve(benchmarkClass.getSimpleName() + ".json").toString()) + .resultFormat(ResultFormatType.JSON); + profilerSpecifications.forEach(specification -> addProfiler(optionsBuilder, specification)); + + Options options = optionsBuilder.build(); + new Runner(options).run(); + } + + private static void addProfiler(OptionsBuilder optionsBuilder, String specification) { + int parameterSeparator = specification.indexOf(':'); + if (parameterSeparator < 0) { + optionsBuilder.addProfiler(specification); + } else { + optionsBuilder.addProfiler( + specification.substring(0, parameterSeparator), + specification.substring(parameterSeparator + 1)); + } + } + } +} diff --git a/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/LongLongMapBenchmark.java b/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/LongLongMapBenchmark.java index 5583e35..82bd71c 100644 --- a/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/LongLongMapBenchmark.java +++ b/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/LongLongMapBenchmark.java @@ -21,6 +21,7 @@ import org.openjdk.jmh.annotations.State; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.RunnerException; @BenchmarkMode(Mode.Throughput) @OutputTimeUnit(TimeUnit.SECONDS) @@ -28,6 +29,10 @@ @Measurement(iterations = 5, time = 1) @Fork(2) public class LongLongMapBenchmark { + public static void main(String[] args) throws RunnerException { + BenchmarkRunner.run(LongLongMapBenchmark.class, args); + } + @Benchmark public void getExisting(PopulatedMaps maps, Blackhole blackhole) { blackhole.consume(getExisting(maps.map, maps.lookupKeys)); diff --git a/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/LongLongRangeMapBenchmark.java b/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/LongLongRangeMapBenchmark.java index ad5d3dc..f522b91 100644 --- a/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/LongLongRangeMapBenchmark.java +++ b/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/LongLongRangeMapBenchmark.java @@ -22,10 +22,7 @@ import org.openjdk.jmh.annotations.Threads; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; -import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; -import org.openjdk.jmh.runner.options.Options; -import org.openjdk.jmh.runner.options.OptionsBuilder; @BenchmarkMode(Mode.Throughput) @OutputTimeUnit(TimeUnit.SECONDS) @@ -34,15 +31,13 @@ @Fork(2) @Threads(1) public class LongLongRangeMapBenchmark { - private static final long RANGE_WIDTH = 16L; - private static final long RANGE_STRIDE = 32L; - public static void main(String[] args) throws RunnerException { - Options options = - new OptionsBuilder().include(LongLongRangeMapBenchmark.class.getSimpleName()).build(); - new Runner(options).run(); + BenchmarkRunner.run(LongLongRangeMapBenchmark.class, args); } + private static final long RANGE_WIDTH = 16L; + private static final long RANGE_STRIDE = 32L; + @Benchmark public void putAll(Ranges ranges, Blackhole blackhole) { blackhole.consume(putAll(ranges)); diff --git a/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/LongLongRankingMapBenchmark.java b/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/LongLongRankingMapBenchmark.java index a8d18af..486f44e 100644 --- a/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/LongLongRankingMapBenchmark.java +++ b/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/LongLongRankingMapBenchmark.java @@ -20,10 +20,7 @@ import org.openjdk.jmh.annotations.Threads; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; -import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; -import org.openjdk.jmh.runner.options.Options; -import org.openjdk.jmh.runner.options.OptionsBuilder; @BenchmarkMode(Mode.Throughput) @OutputTimeUnit(TimeUnit.SECONDS) @@ -33,9 +30,7 @@ @Threads(1) public class LongLongRankingMapBenchmark { public static void main(String[] args) throws RunnerException { - Options options = - new OptionsBuilder().include(LongLongRankingMapBenchmark.class.getSimpleName()).build(); - new Runner(options).run(); + BenchmarkRunner.run(LongLongRankingMapBenchmark.class, args); } @Benchmark diff --git a/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/LongObjectMapBenchmark.java b/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/LongObjectMapBenchmark.java index 125597a..048c143 100644 --- a/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/LongObjectMapBenchmark.java +++ b/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/LongObjectMapBenchmark.java @@ -21,6 +21,7 @@ import org.openjdk.jmh.annotations.State; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.RunnerException; @BenchmarkMode(Mode.Throughput) @OutputTimeUnit(TimeUnit.SECONDS) @@ -28,6 +29,10 @@ @Measurement(iterations = 5, time = 1) @Fork(2) public class LongObjectMapBenchmark { + public static void main(String[] args) throws RunnerException { + BenchmarkRunner.run(LongObjectMapBenchmark.class, args); + } + @Benchmark public void getExisting(PopulatedMaps maps, Blackhole blackhole) { blackhole.consume(getExisting(maps.map, maps.lookupKeys)); diff --git a/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/LongObjectRangeMapBenchmark.java b/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/LongObjectRangeMapBenchmark.java index 6b2f9ce..0ddaa65 100644 --- a/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/LongObjectRangeMapBenchmark.java +++ b/mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/LongObjectRangeMapBenchmark.java @@ -18,6 +18,7 @@ import org.openjdk.jmh.annotations.Threads; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.RunnerException; @BenchmarkMode(Mode.Throughput) @OutputTimeUnit(TimeUnit.SECONDS) @@ -26,6 +27,10 @@ @Fork(2) @Threads(1) public class LongObjectRangeMapBenchmark { + public static void main(String[] args) throws RunnerException { + BenchmarkRunner.run(LongObjectRangeMapBenchmark.class, args); + } + private static final long RANGE_WIDTH = 16L; private static final long RANGE_STRIDE = 32L;