Add performance test workflow

.github/workflows/performance.yaml

@@ -0,0 +1,30 @@
+name: Manual Performance Benchmarks
+
+on:
+  workflow_dispatch:
+    inputs:
+      benchmark:
+        description: "JMH benchmark regex"
+        required: true
+        default: ".*"
+
+jobs:
+  jmh:
+    runs-on:
+      - self-hosted
+      - mapsmith-perf
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Run JMH
+        run: |
+          mkdir -p mapsmith-benchmarks/build/benchmarks
+          ./gradlew :mapsmith-benchmarks:jmh \
+            --args="${{ inputs.benchmark }} -rf json -rff $GITHUB_WORKSPACE/mapsmith-benchmarks/build/benchmarks/jmh-results.json"
+
+      - name: Upload JMH results
+        uses: actions/upload-artifact@v4
+        with:
+          name: jmh-results
+          path: mapsmith-benchmarks/build/benchmarks/jmh-results.json

gradle/libs.versions.toml

@@ -10,6 +10,7 @@ junit = "6.0.0"
 ktfmt = "0.61"
 maven-publish = "0.36.0"
 pmd = "7.25.0"
+picocli = "4.7.7"
 spotbugs-plugin = "6.5.6"
 spotless = "8.6.0"
 
@@ -18,6 +19,7 @@ assertj-core = { module = "org.assertj:assertj-core", version.ref = "assertj" }
 errorprone-core = { module = "com.google.errorprone:error_prone_core", version.ref = "errorprone" }
 jmh-core = { module = "org.openjdk.jmh:jmh-core", version.ref = "jmh" }
 jmh-generator-annprocess = { module = "org.openjdk.jmh:jmh-generator-annprocess", version.ref = "jmh" }
+picocli = { module = "info.picocli:picocli", version.ref = "picocli" }
 jqwik = { module = "net.jqwik:jqwik", version.ref = "jqwik" }
 junit-bom = { module = "org.junit:junit-bom", version.ref = "junit" }
 junit-jupiter = { module = "org.junit.jupiter:junit-jupiter" }

infra/perf-runner/.gitignore

@@ -0,0 +1,6 @@
+.terraform/
+.terraform.lock.hcl
+*.tfstate
+*.tfstate.*
+crash.log
+crash.*.log

infra/perf-runner/README.md

@@ -0,0 +1,56 @@
+# Mapsmith performance runner
+
+This Terraform configuration creates a `c7i.xlarge` one-time Spot instance
+(default region: `eu-west-2`) and configures it as a GitHub Actions
+self-hosted runner for `mrk-andreev/mapsmith`, labelled `mapsmith-perf`.
+
+Prerequisites:
+
+- Terraform 1.5 or newer and AWS credentials for the selected region.
+- GitHub CLI authenticated locally with permission to administer Actions
+  runners for the repository (`gh auth login`).
+- A subnet with outbound HTTPS access. The default configuration uses a public
+  IP; set `associate_public_ip_address = false` when using a private subnet
+  with NAT egress.
+
+Run it from this directory:
+
+```sh
+terraform init
+terraform apply
+```
+
+Specify network placement when the account does not have a suitable default VPC:
+
+```sh
+terraform apply -var='subnet_id=subnet-0123456789abcdef0'
+```
+
+Override the default `c7i.xlarge` instance type when needed:
+
+```sh
+terraform apply -var='instance_type=c7i.2xlarge'
+```
+
+Override the default `eu-west-2` region when needed:
+
+```sh
+terraform apply -var='aws_region=eu-central-1'
+```
+
+Destroy the instance when benchmarks are finished:
+
+```sh
+terraform destroy
+```
+
+To recreate a runner deliberately, request replacement so Terraform obtains a
+fresh registration token:
+
+```sh
+terraform apply -replace=aws_instance.perf_runner
+```
+
+The registration token is deliberately obtained through `gh` at apply time.
+Terraform records the resulting user data in its local state, so keep the
+state file private and do not commit it.

infra/perf-runner/github-runner-token.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# The external provider supplies the JSON query on stdin.
+repository="$(jq -r '.repository' < /dev/stdin)"
+
+if [[ -z "$repository" || "$repository" == "null" ]]; then
+  echo "repository is required" >&2
+  exit 1
+fi
+
+# gh must already be authenticated to this repository with Actions administration access.
+token="$(gh api --method POST "repos/${repository}/actions/runners/registration-token" --jq '.token')"
+printf '{"token":"%s"}\n' "$token"

infra/perf-runner/main.tf

@@ -0,0 +1,90 @@
+terraform {
+  required_version = ">= 1.5.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 6.0"
+    }
+    external = {
+      source  = "hashicorp/external"
+      version = "~> 2.3"
+    }
+  }
+}
+
+provider "aws" {
+  region = var.aws_region
+}
+
+data "aws_ami" "ubuntu" {
+  most_recent = true
+  owners      = ["099720109477"] # Canonical
+
+  filter {
+    name   = "name"
+    values = ["ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*"]
+  }
+
+  filter {
+    name   = "architecture"
+    values = ["x86_64"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+}
+
+data "external" "github_runner_token" {
+  program = ["bash", "${path.module}/github-runner-token.sh"]
+
+  query = {
+    repository = var.repository
+  }
+}
+
+resource "aws_instance" "perf_runner" {
+  ami                         = data.aws_ami.ubuntu.id
+  instance_type               = var.instance_type
+  subnet_id                   = var.subnet_id
+  vpc_security_group_ids      = var.security_group_ids
+  associate_public_ip_address = var.associate_public_ip_address
+
+  instance_market_options {
+    market_type = "spot"
+
+    spot_options {
+      spot_instance_type             = "one-time"
+      instance_interruption_behavior = "terminate"
+    }
+  }
+
+  root_block_device {
+    encrypted   = true
+    volume_size = 30
+    volume_type = "gp3"
+  }
+
+  metadata_options {
+    http_endpoint = "enabled"
+    http_tokens   = "required"
+  }
+
+  user_data_replace_on_change = true
+  user_data = templatefile("${path.module}/user-data.sh.tftpl", {
+    repository   = var.repository
+    runner_token = sensitive(data.external.github_runner_token.result.token)
+  })
+
+  # A registration token is newly generated on every plan. It is needed only
+  # during the first boot, so it must not cause a running runner to be replaced.
+  lifecycle {
+    ignore_changes = [user_data]
+  }
+
+  tags = merge(var.tags, {
+    Name = "mapsmith-perf-runner"
+  })
+}

infra/perf-runner/outputs.tf

@@ -0,0 +1,9 @@
+output "instance_id" {
+  description = "ID of the Spot-backed performance runner instance."
+  value       = aws_instance.perf_runner.id
+}
+
+output "public_ip" {
+  description = "Public IPv4 address, if one was assigned."
+  value       = aws_instance.perf_runner.public_ip
+}

infra/perf-runner/user-data.sh.tftpl

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get update
+apt-get install -y --no-install-recommends ca-certificates curl git jq openjdk-21-jdk-headless
+
+id -u runner >/dev/null 2>&1 || useradd --create-home --shell /bin/bash runner
+
+install -d -o runner -g runner /opt/actions-runner
+cd /opt/actions-runner
+
+runner_version="$$(curl --fail --silent --show-error --location \
+  https://api.github.com/repos/actions/runner/releases/latest | jq -r '.tag_name')"
+curl --fail --silent --show-error --location \
+  --output actions-runner.tar.gz \
+  "https://github.com/actions/runner/releases/download/$${runner_version}/actions-runner-linux-x64-$${runner_version#v}.tar.gz"
+tar xzf actions-runner.tar.gz
+rm actions-runner.tar.gz
+chown -R runner:runner /opt/actions-runner
+./bin/installdependencies.sh
+
+runuser -u runner -- ./config.sh --unattended \
+  --url "https://github.com/${repository}" \
+  --token '${runner_token}' \
+  --name "$$(hostname)" \
+  --labels "mapsmith-perf" \
+  --replace
+
+./svc.sh install runner
+./svc.sh start

infra/perf-runner/variables.tf

@@ -0,0 +1,41 @@
+variable "repository" {
+  description = "GitHub repository that owns the self-hosted runner."
+  type        = string
+  default     = "mrk-andreev/mapsmith"
+}
+
+variable "aws_region" {
+  description = "AWS region in which to create the performance runner."
+  type        = string
+  default     = "eu-west-2"
+}
+
+variable "instance_type" {
+  description = "EC2 instance type for the performance runner."
+  type        = string
+  default     = "c7i.xlarge"
+}
+
+variable "subnet_id" {
+  description = "Subnet in the selected AWS region in which to create the runner. Leave null to use the account default VPC subnet."
+  type        = string
+  default     = null
+}
+
+variable "security_group_ids" {
+  description = "Security groups for the runner. No inbound rules are required; it needs outbound HTTPS access."
+  type        = list(string)
+  default     = null
+}
+
+variable "associate_public_ip_address" {
+  description = "Whether the runner receives a public IPv4 address. Set false when the subnet has NAT egress."
+  type        = bool
+  default     = true
+}
+
+variable "tags" {
+  description = "Additional AWS tags."
+  type        = map(string)
+  default     = {}
+}

mapsmith-benchmarks/build.gradle.kts

@@ -19,6 +19,7 @@ dependencies {
 
   add(jmh.implementationConfigurationName, project(":mapsmith-core"))
   add(jmh.implementationConfigurationName, libs.jmh.core)
+  add(jmh.implementationConfigurationName, libs.picocli)
   add(jmh.annotationProcessorConfigurationName, libs.jmh.generator.annprocess)
 }
 

mapsmith-benchmarks/src/jmh/java/name/mrkandreev/mapsmith/benchmarks/BenchmarkRunner.java

@@ -0,0 +1,82 @@
+package name.mrkandreev.mapsmith.benchmarks;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import org.openjdk.jmh.results.format.ResultFormatType;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import picocli.CommandLine;
+
+public enum BenchmarkRunner {
+  ;
+
+  public static void run(Class<?> cls, String[] args) throws RunnerException {
+    var command = new RunBenchmarkCommand(cls);
+    var commandLine = new CommandLine(command);
+    var parseResult = commandLine.parseArgs(args);
+    if (!CommandLine.printHelpIfRequested(parseResult)) {
+      command.run();
+    }
+  }
+
+  @CommandLine.Command(
+      name = "benchmark",
+      mixinStandardHelpOptions = true,
+      description = "Runs a JMH benchmark and writes its results as JSON.")
+  private static final class RunBenchmarkCommand {
+    private final Class<?> benchmarkClass;
+
+    @CommandLine.Option(
+        names = {"-o", "--output-dir"},
+        defaultValue = "build/benchmarks",
+        paramLabel = "<directory>",
+        description = "Directory for JSON results (default: ${DEFAULT-VALUE}).")
+    private Path outputDirectory;
+
+    @CommandLine.Option(
+        names = {"-p", "--profiler"},
+        paramLabel = "<name[:parameters]>",
+        description =
+            "JMH profiler, optionally followed by its parameters; may be specified more than once.")
+    private final List<String> profilerSpecifications = new ArrayList<>();
+
+    private RunBenchmarkCommand(Class<?> benchmarkClass) {
+      this.benchmarkClass = benchmarkClass;
+    }
+
+    private void run() throws RunnerException {
+      try {
+        Files.createDirectories(outputDirectory);
+      } catch (IOException exception) {
+        throw new RunnerException(
+            "Could not create benchmark output directory: " + outputDirectory, exception);
+      }
+
+      var optionsBuilder = new OptionsBuilder();
+      optionsBuilder
+          .include(benchmarkClass.getSimpleName())
+          .result(outputDirectory.resolve(benchmarkClass.getSimpleName() + ".json").toString())
+          .resultFormat(ResultFormatType.JSON);
+      profilerSpecifications.forEach(specification -> addProfiler(optionsBuilder, specification));
+
+      Options options = optionsBuilder.build();
+      new Runner(options).run();
+    }
+
+    private static void addProfiler(OptionsBuilder optionsBuilder, String specification) {
+      int parameterSeparator = specification.indexOf(':');
+      if (parameterSeparator < 0) {
+        optionsBuilder.addProfiler(specification);
+      } else {
+        optionsBuilder.addProfiler(
+            specification.substring(0, parameterSeparator),
+            specification.substring(parameterSeparator + 1));
+      }
+    }
+  }
+}