From 670a7d807b3d0a22e9449ce716a8fdccc41fc50d Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Wed, 19 Aug 2026 12:11:47 +0800 Subject: [PATCH 1/3] ci: add LLGo runtime performance job --- .../workflows/llgo-runtime-performance.yml | 114 ++++++++++++++++++ ci/llgo-performance/README.md | 18 +++ ci/llgo-performance/report.sh | 52 ++++++++ cmd/bent/bent.go | 1 + .../configs/benchmarks-llgo-performance.toml | 65 ++++++++++ .../configurations-llgo-performance.toml | 24 ++++ 6 files changed, 274 insertions(+) create mode 100644 .github/workflows/llgo-runtime-performance.yml create mode 100644 ci/llgo-performance/README.md create mode 100755 ci/llgo-performance/report.sh create mode 100644 cmd/bent/configs/benchmarks-llgo-performance.toml create mode 100644 cmd/bent/configs/configurations-llgo-performance.toml diff --git a/.github/workflows/llgo-runtime-performance.yml b/.github/workflows/llgo-runtime-performance.yml new file mode 100644 index 0000000..5442dd4 --- /dev/null +++ b/.github/workflows/llgo-runtime-performance.yml @@ -0,0 +1,114 @@ +name: LLGo runtime performance + +on: + repository_dispatch: + types: [llgo-tag-released] + +permissions: + contents: read + +concurrency: + group: llgo-runtime-performance + cancel-in-progress: false + +jobs: + benchmark: + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + + - name: Validate the LLGo release notification + env: + DISPATCH_SOURCE_REPOSITORY: ${{ github.event.client_payload.source_repository }} + DISPATCH_LLGO_REPOSITORY: ${{ github.event.client_payload.llgo_repository }} + DISPATCH_LLGO_COMMIT: ${{ github.event.client_payload.llgo_commit }} + DISPATCH_LLGO_TAG: ${{ github.event.client_payload.llgo_tag }} + run: | + set -euo pipefail + set -a + source ci/llgo-size/llgo-version.env + set +a + + if [[ "$DISPATCH_SOURCE_REPOSITORY" != "xgo-dev/llgo" || \ + "$DISPATCH_LLGO_REPOSITORY" != "xgo-dev/llgo" ]]; then + echo "refusing LLGo release notification from ${DISPATCH_SOURCE_REPOSITORY:-unknown}" >&2 + exit 1 + fi + if [[ ! "$DISPATCH_LLGO_COMMIT" =~ ^[0-9a-f]{40}$ ]]; then + echo "invalid released LLGo commit: ${DISPATCH_LLGO_COMMIT:-missing}" >&2 + exit 1 + fi + if ! git check-ref-format "refs/tags/$DISPATCH_LLGO_TAG"; then + echo "invalid released LLGo tag: ${DISPATCH_LLGO_TAG:-missing}" >&2 + exit 1 + fi + + printf 'LLGO_REPOSITORY=%s\n' "$DISPATCH_LLGO_REPOSITORY" >> "$GITHUB_ENV" + printf 'LLGO_COMMIT=%s\n' "$DISPATCH_LLGO_COMMIT" >> "$GITHUB_ENV" + printf 'LLGO_TAG=%s\n' "$DISPATCH_LLGO_TAG" >> "$GITHUB_ENV" + printf 'LLVM_VERSION=%s\n' "$LLVM_VERSION" >> "$GITHUB_ENV" + mkdir -p .ci + printf '%s\n' "$GO_VERSION" > .ci/go-version + + - uses: actions/setup-go@v5 + with: + go-version-file: .ci/go-version + + - name: Install LLVM dependencies + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y clang-${LLVM_VERSION} lld-${LLVM_VERSION} llvm-${LLVM_VERSION}-dev \ + libuv1-dev libgc-dev libffi-dev + + - name: Build the released LLGo compiler + run: | + set -euo pipefail + git clone --filter=blob:none --no-checkout "https://github.com/${LLGO_REPOSITORY}.git" .ci/llgo + git -C .ci/llgo fetch --no-tags --depth=1 origin \ + "refs/tags/$LLGO_TAG:refs/tags/$LLGO_TAG" + tag_commit=$(git -C .ci/llgo rev-parse "refs/tags/$LLGO_TAG^{commit}") + if [[ "$tag_commit" != "$LLGO_COMMIT" ]]; then + echo "LLGo tag $LLGO_TAG points to $tag_commit, not dispatched $LLGO_COMMIT" >&2 + exit 1 + fi + git -C .ci/llgo checkout --detach "$LLGO_COMMIT" + export GOROOT="$(go env GOROOT)" + export PATH="$GOROOT/bin:/usr/lib/llvm-${LLVM_VERSION}/bin:$PATH" + export LLGO_ROOT="$GITHUB_WORKSPACE/.ci/llgo" + (cd "$LLGO_ROOT" && go build -tags=dev -o "$LLGO_ROOT/llgo" ./cmd/llgo) + + - name: Run the selected Bent comparison + env: + LLGO_BIN: ${{ github.workspace }}/.ci/llgo/llgo + LLGO_ROOT: ${{ github.workspace }}/.ci/llgo + run: | + set -euo pipefail + export GOROOT="$(go env GOROOT)" + export PATH="$GOROOT/bin:/usr/lib/llvm-${LLVM_VERSION}/bin:$PATH" + go build -o .ci/bent ./cmd/bent + GOBIN="$GITHUB_WORKSPACE/.ci/bin" \ + go install golang.org/x/perf/cmd/benchstat@v0.0.0-20260813145340-fd4a688df892 + mkdir -p .ci/bent-performance + ( + cd .ci/bent-performance + "$GITHUB_WORKSPACE/.ci/bent" -I + "$GITHUB_WORKSPACE/.ci/bent" \ + -B=benchmarks-llgo-performance.toml \ + -C=configurations-llgo-performance.toml \ + -N=10 -G + ) + ci/llgo-performance/report.sh \ + .ci/bent-performance \ + .ci/bin/benchstat | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Upload Bent runtime results + if: always() + uses: actions/upload-artifact@v4 + with: + name: llgo-runtime-performance + path: | + .ci/bent-performance/bench + .ci/bent-performance/benchstat.txt + if-no-files-found: warn diff --git a/ci/llgo-performance/README.md b/ci/llgo-performance/README.md new file mode 100644 index 0000000..993c092 --- /dev/null +++ b/ci/llgo-performance/README.md @@ -0,0 +1,18 @@ +# LLGo runtime performance + +This job compares the pinned Go and LLGo toolchains with Bent. It runs the +checked-in cases ten times each, grouped by benchmark to reduce time-local +runner noise, and publishes both the raw Bent output and a benchstat report. + +It runs only after the `xgo-dev/llgo` release workflow publishes a new tag and +dispatches the exact tag and commit. Ordinary pushes and pull requests do not +start a performance run. + +The selected set contains cases where LLGo or LLGo full LTO was faster in a +local `benchmarks-100.toml` survey, plus stable same-range cases that are useful +for tracking future optimization. The workflow checks sample completeness but +does not fail on a performance percentage because hosted-runner measurements +are not a reliable hard regression gate. + +The suite versions come from `cmd/bent/configs/suites.toml`. Keep those versions +pinned for comparable history; update them deliberately in a reviewed change. diff --git a/ci/llgo-performance/report.sh b/ci/llgo-performance/report.sh new file mode 100755 index 0000000..e9ea3b5 --- /dev/null +++ b/ci/llgo-performance/report.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euo pipefail + +run_dir=$1 +benchstat_bin=$2 +bench_dir="$run_dir/bench" +benchmark_file="$run_dir/benchmarks-llgo-performance.toml" +report="$run_dir/benchstat.txt" +repetitions=10 + +benchmarks=() +while IFS= read -r benchmark; do + benchmarks+=("$benchmark") +done < <(awk -F'"' '/^ Name = / { print $2 }' "$benchmark_file") +configs=(Go LLGo LLGoFullLTO) + +if ((${#benchmarks[@]} == 0)); then + echo "no LLGo performance benchmarks found in $benchmark_file" >&2 + exit 1 +fi + +stdout_files=() +for config in "${configs[@]}"; do + matches=() + while IFS= read -r match; do + matches+=("$match") + done < <(find "$bench_dir" -maxdepth 1 -type f -name "*.$config.stdout" -print) + if ((${#matches[@]} != 1)); then + echo "expected one $config stdout file in $bench_dir, found ${#matches[@]}" >&2 + exit 1 + fi + stdout_file=${matches[0]} + stdout_files+=("$stdout_file") + + for benchmark in "${benchmarks[@]}"; do + count=$(grep -Fxc "shortname: $benchmark" "$stdout_file" || true) + if ((count != repetitions)); then + echo "$config produced $count/$repetitions samples for $benchmark" >&2 + exit 1 + fi + done + + pass_count=$(grep -c '^PASS$' "$stdout_file" || true) + expected_passes=$((${#benchmarks[@]} * repetitions)) + if ((pass_count != expected_passes)); then + echo "$config produced $pass_count/$expected_passes passing samples" >&2 + exit 1 + fi +done + +"$benchstat_bin" -table shortname -col toolchain -row .name "${stdout_files[@]}" > "$report" +cat "$report" diff --git a/cmd/bent/bent.go b/cmd/bent/bent.go index f04a4e9..c9120f2 100644 --- a/cmd/bent/bent.go +++ b/cmd/bent/bent.go @@ -169,6 +169,7 @@ var copyConfigs = []string{ "configurations-sample.toml", "configurations-gollvm.toml", "configurations-cronjob.toml", "configurations-cmpjob.toml", "configurations-pgo.toml", "configurations-random.toml", "suites.toml", "configurations-llgo-size.toml", "benchmarks-llgo-size.toml", + "configurations-llgo-performance.toml", "benchmarks-llgo-performance.toml", } var defaultEnv []string diff --git a/cmd/bent/configs/benchmarks-llgo-performance.toml b/cmd/bent/configs/benchmarks-llgo-performance.toml new file mode 100644 index 0000000..5869fc8 --- /dev/null +++ b/cmd/bent/configs/benchmarks-llgo-performance.toml @@ -0,0 +1,65 @@ +# Stable cases selected from a benchmarks-100 run with LLGo. The first group +# showed an LLGo or LLGo full-LTO speedup; the second group remains in the same +# performance range and provides useful optimization coverage. + +# LLGo advantages. +[[Benchmarks]] + Name = "workiva_avl-ImmutableInsert" + Suite = "workiva_avl" + Benchmarks = "^BenchmarkImmutableInsert$" + +[[Benchmarks]] + Name = "gonum_mat-CholeskyInverseTo%2Fn%3D100" + Suite = "gonum_mat" + Benchmarks = "^BenchmarkCholeskyInverseTo$/^n=100$" + +[[Benchmarks]] + Name = "uber_tally-TimerReport" + Suite = "uber_tally" + Benchmarks = "^BenchmarkTimerReport$" + +[[Benchmarks]] + Name = "bloom_bits-BitSetExtractDeposit%2Fsize%3D64%2Ffn%3DDepositTo" + Suite = "bloom_bits" + Benchmarks = "^BenchmarkBitSetExtractDeposit$/^size=64$/^fn=DepositTo$" + +[[Benchmarks]] + Name = "workiva_triey-Iterator" + Suite = "workiva_triey" + Benchmarks = "^BenchmarkIterator$" + +[[Benchmarks]] + Name = "workiva_rangetree-ImmutableInsertSecondDimension" + Suite = "workiva_rangetree" + Benchmarks = "^BenchmarkImmutableInsertSecondDimension$" + +# Same-range optimization coverage. +[[Benchmarks]] + Name = "kanzi-BWTSmallBlock" + Suite = "kanzi" + Benchmarks = "^BenchmarkBWTSmallBlock$" + +[[Benchmarks]] + Name = "kanzi-ANS0" + Suite = "kanzi" + Benchmarks = "^BenchmarkANS0$" + +[[Benchmarks]] + Name = "kanzi-Huffman" + Suite = "kanzi" + Benchmarks = "^BenchmarkHuffman$" + +[[Benchmarks]] + Name = "gonum_community-NewDirectedLayers%2FgraphSize%3D10%2CnumGraphs%3D1" + Suite = "gonum_community" + Benchmarks = "^BenchmarkNewDirectedLayers$/^graphSize=10,numGraphs=1$" + +[[Benchmarks]] + Name = "gproto_reflect-Value%2FInterface" + Suite = "gproto_reflect" + Benchmarks = "^BenchmarkValue$/^Interface$" + +[[Benchmarks]] + Name = "gonum_community-NewDirectedLayers%2FgraphSize%3D1000%2CnumGraphs%3D1" + Suite = "gonum_community" + Benchmarks = "^BenchmarkNewDirectedLayers$/^graphSize=1000,numGraphs=1$" diff --git a/cmd/bent/configs/configurations-llgo-performance.toml b/cmd/bent/configs/configurations-llgo-performance.toml new file mode 100644 index 0000000..c18e5de --- /dev/null +++ b/cmd/bent/configs/configurations-llgo-performance.toml @@ -0,0 +1,24 @@ +# Runtime comparison used by ci/llgo-performance. + +[[Configurations]] + Name = "Go" + Root = "$GOROOT" + UseBuildCache = true + RunEnv = ["GOMAXPROCS=1"] + +[[Configurations]] + Name = "LLGo" + Compiler = "$LLGO_BIN" + Root = "$GOROOT" + OmitVetFlag = true + UseBuildCache = true + RunEnv = ["GOMAXPROCS=1"] + +[[Configurations]] + Name = "LLGoFullLTO" + Compiler = "$LLGO_BIN" + Root = "$GOROOT" + OmitVetFlag = true + UseBuildCache = true + BuildFlags = ["-lto=full"] + RunEnv = ["GOMAXPROCS=1"] From c0af02ce00e8fa8ebb70402b82ac0e36c6991602 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Wed, 19 Aug 2026 17:39:14 +0800 Subject: [PATCH 2/3] ci: publish runtime performance history --- .../workflows/llgo-runtime-performance.yml | 69 ++++++- ci/llgo-performance/README.md | 9 +- ci/llgo-performance/publish.sh | 102 ++++++++++ ci/llgo-performance/report.py | 167 ++++++++++++++++ ci/llgo-performance/report.sh | 6 + ci/llgo-size/publish-site.sh | 2 +- ci/llgo-size/publish.sh | 7 +- ci/llgo-size/site/index.html | 4 + ci/llgo-size/site/performance.html | 60 ++++++ ci/llgo-size/site/performance.js | 180 ++++++++++++++++++ ci/llgo-size/site/style.css | 32 +++- 11 files changed, 629 insertions(+), 9 deletions(-) create mode 100755 ci/llgo-performance/publish.sh create mode 100755 ci/llgo-performance/report.py create mode 100644 ci/llgo-size/site/performance.html create mode 100644 ci/llgo-size/site/performance.js diff --git a/.github/workflows/llgo-runtime-performance.yml b/.github/workflows/llgo-runtime-performance.yml index 5442dd4..2b17990 100644 --- a/.github/workflows/llgo-runtime-performance.yml +++ b/.github/workflows/llgo-runtime-performance.yml @@ -15,6 +15,8 @@ jobs: benchmark: runs-on: ubuntu-24.04 timeout-minutes: 90 + permissions: + contents: write steps: - uses: actions/checkout@v4 @@ -47,7 +49,9 @@ jobs: printf 'LLGO_REPOSITORY=%s\n' "$DISPATCH_LLGO_REPOSITORY" >> "$GITHUB_ENV" printf 'LLGO_COMMIT=%s\n' "$DISPATCH_LLGO_COMMIT" >> "$GITHUB_ENV" printf 'LLGO_TAG=%s\n' "$DISPATCH_LLGO_TAG" >> "$GITHUB_ENV" + printf 'GO_VERSION=%s\n' "$GO_VERSION" >> "$GITHUB_ENV" printf 'LLVM_VERSION=%s\n' "$LLVM_VERSION" >> "$GITHUB_ENV" + printf 'LLGO_PERFORMANCE_REPETITIONS=10\n' >> "$GITHUB_ENV" mkdir -p .ci printf '%s\n' "$GO_VERSION" > .ci/go-version @@ -58,8 +62,22 @@ jobs: - name: Install LLVM dependencies run: | set -euo pipefail - sudo apt-get update - sudo apt-get install -y clang-${LLVM_VERSION} lld-${LLVM_VERSION} llvm-${LLVM_VERSION}-dev \ + ubuntu_sources=/etc/apt/sources.list.d/ubuntu.sources + if [[ -f "$ubuntu_sources" ]]; then + sudo sed -i \ + -e 's|mirror+file:/etc/apt/apt-mirrors.txt|https://archive.ubuntu.com/ubuntu|g' \ + -e 's|http://azure.archive.ubuntu.com/ubuntu|https://archive.ubuntu.com/ubuntu|g' \ + "$ubuntu_sources" + fi + apt_options=( + -o Acquire::Retries=3 + -o Acquire::http::Timeout=30 + -o Acquire::https::Timeout=30 + -o DPkg::Lock::Timeout=60 + ) + sudo timeout --kill-after=30s 5m apt-get "${apt_options[@]}" update + sudo timeout --kill-after=30s 15m apt-get "${apt_options[@]}" install -y \ + clang-${LLVM_VERSION} lld-${LLVM_VERSION} llvm-${LLVM_VERSION}-dev \ libuv1-dev libgc-dev libffi-dev - name: Build the released LLGo compiler @@ -111,4 +129,51 @@ jobs: path: | .ci/bent-performance/bench .ci/bent-performance/benchstat.txt + .ci/bent-performance/benchstat.csv + .ci/bent-performance/results.json if-no-files-found: warn + + - name: Publish runtime performance history to Pages + env: + PAGES_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + pages_dir="$GITHUB_WORKSPACE/.ci/pages" + bash ci/llgo-size/prepare-pages-branch.sh \ + "$pages_dir" \ + "https://x-access-token:${PAGES_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + ci/llgo-performance/publish.sh \ + .ci/bent-performance \ + "$pages_dir" \ + ci/llgo-size/site + + deploy-pages: + needs: benchmark + runs-on: ubuntu-24.04 + permissions: + contents: read + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Check out Pages source + uses: actions/checkout@v4 + with: + ref: pages + path: pages-source + - name: Configure GitHub Pages + uses: actions/configure-pages@v5 + - name: Build Pages with Jekyll + uses: actions/jekyll-build-pages@v1 + with: + source: ./pages-source + destination: ./_site + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v4 + with: + path: ./_site + - name: Deploy GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/ci/llgo-performance/README.md b/ci/llgo-performance/README.md index 993c092..5898a40 100644 --- a/ci/llgo-performance/README.md +++ b/ci/llgo-performance/README.md @@ -2,7 +2,8 @@ This job compares the pinned Go and LLGo toolchains with Bent. It runs the checked-in cases ten times each, grouped by benchmark to reduce time-local -runner noise, and publishes both the raw Bent output and a benchstat report. +runner noise, and publishes the raw Bent output, native benchstat text/CSV, and +a structured result used by the runtime-performance Pages table. It runs only after the `xgo-dev/llgo` release workflow publishes a new tag and dispatches the exact tag and commit. Ordinary pushes and pull requests do not @@ -14,5 +15,11 @@ for tracking future optimization. The workflow checks sample completeness but does not fail on a performance percentage because hosted-runner measurements are not a reliable hard regression gate. +Each successful release run is stored under `data/performance/runs//` on +the `pages` branch. The lightweight `performance.html` view follows benchstat's +layout: Go is the baseline, with LLGo and LLGo full-LTO values, confidence +ranges, deltas, and p-values shown side by side. The existing `index.html` +continues to show only binary size and build time. + The suite versions come from `cmd/bent/configs/suites.toml`. Keep those versions pinned for comparable history; update them deliberately in a reviewed change. diff --git a/ci/llgo-performance/publish.sh b/ci/llgo-performance/publish.sh new file mode 100755 index 0000000..27bfa10 --- /dev/null +++ b/ci/llgo-performance/publish.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -euo pipefail + +run_dir=$1 +pages_dir=$2 +site_dir=$3 +if [[ -z "$run_dir" || -z "$pages_dir" || -z "$site_dir" ]]; then + echo "usage: publish.sh RUN_DIR PAGES_DIR SITE_DIR" >&2 + exit 2 +fi + +result_json="$run_dir/results.json" +for file in "$result_json" "$run_dir/benchstat.txt" "$run_dir/benchstat.csv"; do + if [[ ! -s "$file" ]]; then + echo "missing performance result: $file" >&2 + exit 1 + fi +done + +run_key=$(python3 - "$result_json" <<'PY' +import json +import re +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + run = json.load(source)["run"] +key = str(run.get("llgoTag") or run.get("llgoCommit") or run.get("id") or "manual") +if not re.fullmatch(r"[A-Za-z0-9._-]+", key): + raise SystemExit("invalid performance run key: " + repr(key)) +print(key) +PY +) + +performance_dir="$pages_dir/data/performance" +published_run_dir="$performance_dir/runs/$run_key" +mkdir -p "$published_run_dir" +cp "$result_json" "$published_run_dir/results.json" +cp "$run_dir/benchstat.txt" "$published_run_dir/benchstat.txt" +cp "$run_dir/benchstat.csv" "$published_run_dir/benchstat.csv" +if [[ -d "$published_run_dir/raw" ]]; then + rm -r "$published_run_dir/raw" +fi +mkdir -p "$published_run_dir/raw" +if compgen -G "$run_dir/bench/*.stdout" >/dev/null; then + cp "$run_dir/bench/"*.stdout "$published_run_dir/raw/" +fi + +python3 - "$performance_dir" <<'PY' +import glob +import json +import os +import sys +from datetime import datetime, timezone + +data_dir = sys.argv[1] +runs = [] +for path in glob.glob(os.path.join(data_dir, "runs", "*", "results.json")): + with open(path, encoding="utf-8") as source: + document = json.load(source) + run = document.get("run", {}) + key = os.path.basename(os.path.dirname(path)) + runs.append({ + "key": key, + "id": run.get("id", ""), + "attempt": run.get("attempt"), + "createdAt": run.get("createdAt", ""), + "sourceCommit": run.get("sourceCommit", ""), + "llgoRepository": run.get("llgoRepository", ""), + "llgoCommit": run.get("llgoCommit", ""), + "llgoTag": run.get("llgoTag", ""), + "goVersion": run.get("goVersion", ""), + "llvmVersion": run.get("llvmVersion", ""), + "workflowUrl": run.get("workflowUrl", ""), + "path": "performance/runs/" + key + "/results.json", + }) +runs.sort(key=lambda item: item["createdAt"], reverse=True) + +index = { + "schemaVersion": 1, + "generatedAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "runs": runs, +} +os.makedirs(data_dir, exist_ok=True) +with open(os.path.join(data_dir, "index.json"), "w", encoding="utf-8") as destination: + json.dump(index, destination, indent=2) + destination.write("\n") +PY + +for file in index.html app.js performance.html performance.js style.css _config.yml; do + cp "$site_dir/$file" "$pages_dir/$file" +done +rm -f "$pages_dir/.nojekyll" + +git -C "$pages_dir" config user.name "github-actions[bot]" +git -C "$pages_dir" config user.email "41898282+github-actions[bot]@users.noreply.github.com" +git -C "$pages_dir" add -A +if git -C "$pages_dir" diff --cached --quiet; then + echo "Performance history is already up to date" +else + git -C "$pages_dir" commit -m "ci: publish LLGo performance run $run_key" + git -C "$pages_dir" push origin HEAD:pages +fi diff --git a/ci/llgo-performance/report.py b/ci/llgo-performance/report.py new file mode 100755 index 0000000..314eb6c --- /dev/null +++ b/ci/llgo-performance/report.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 + +import csv +import json +import os +import re +import sys +from datetime import datetime, timezone +from urllib.parse import unquote + + +CONFIGS = ("Go", "LLGo", "LLGoFullLTO") + + +def env(name, fallback=""): + return os.environ.get(name) or os.environ.get(fallback, "") + + +def number(name, fallback=""): + try: + return int(env(name, fallback)) + except ValueError: + return None + + +def benchmark_environment(path): + values = {} + with open(path, encoding="utf-8") as source: + for line in source: + if line.startswith("Benchmark"): + break + key, separator, value = line.rstrip("\n").partition(": ") + if separator and key in {"goos", "goarch", "cpu"}: + values[key] = value + return values + + +def parse_delta(value): + match = re.fullmatch(r"([+-]?\d+(?:\.\d+)?)%", value) + return float(match.group(1)) if match else None + + +def split_shortname(shortname): + suite, separator, encoded_case = shortname.partition("-") + case = unquote(encoded_case) if separator else shortname + return suite if separator else "", case + + +def parse_benchstat(path): + with open(path, newline="", encoding="utf-8") as source: + rows = list(csv.reader(source)) + + benchmarks = [] + index = 0 + while index < len(rows): + row = rows[index] + if not row or not row[0].startswith("shortname: "): + index += 1 + continue + + shortname = row[0].removeprefix("shortname: ") + index += 1 + while index < len(rows) and not rows[index]: + index += 1 + if index + 1 >= len(rows): + raise ValueError(f"missing benchstat headers for {shortname}") + + config_row = rows[index] + metric_row = rows[index + 1] + columns = {} + for config in CONFIGS: + try: + columns[config] = config_row.index(config) + except ValueError as error: + raise ValueError(f"missing {config} column for {shortname}") from error + if any(column >= len(metric_row) or metric_row[column] != "sec/op" for column in columns.values()): + raise ValueError(f"expected sec/op table for {shortname}") + + index += 2 + suite, case = split_shortname(shortname) + while index < len(rows) and rows[index] and not rows[index][0].startswith("shortname: "): + values = rows[index] + index += 1 + if not values or values[0] == "geomean": + continue + + measurements = {} + for config in CONFIGS: + column = columns[config] + try: + measurement = { + "value": float(values[column]), + "ci": values[column + 1], + } + if config != "Go": + measurement["delta"] = values[column + 2] + measurement["deltaPercent"] = parse_delta(values[column + 2]) + measurement["p"] = values[column + 3] + except (IndexError, ValueError) as error: + raise ValueError(f"invalid {config} result for {shortname}") from error + measurements[config] = measurement + + benchmarks.append({ + "shortname": shortname, + "suite": suite, + "case": case, + "benchmark": values[0], + "values": measurements, + }) + + if not benchmarks: + raise ValueError("benchstat CSV contains no sec/op benchmarks") + return benchmarks + + +def main(): + if len(sys.argv) != 4: + raise SystemExit("usage: report.py BENCHSTAT_CSV OUTPUT_JSON GO_STDOUT") + + csv_path, output_path, go_stdout = sys.argv[1:] + repository = env("LLGO_PERFORMANCE_REPOSITORY", "GITHUB_REPOSITORY") + run_id = env("LLGO_PERFORMANCE_RUN_ID", "GITHUB_RUN_ID") + workflow_url = env("LLGO_PERFORMANCE_WORKFLOW_URL") + if not workflow_url and repository and run_id: + workflow_url = f"https://github.com/{repository}/actions/runs/{run_id}" + + document = { + "schemaVersion": 1, + "format": "benchstat", + "run": { + "id": run_id, + "attempt": number("LLGO_PERFORMANCE_RUN_ATTEMPT", "GITHUB_RUN_ATTEMPT"), + "number": number("LLGO_PERFORMANCE_RUN_NUMBER", "GITHUB_RUN_NUMBER"), + "createdAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "repository": repository, + "sourceCommit": env("LLGO_PERFORMANCE_SOURCE_COMMIT", "GITHUB_SHA"), + "ref": env("LLGO_PERFORMANCE_REF", "GITHUB_REF_NAME"), + "llgoRepository": env("LLGO_REPOSITORY"), + "llgoCommit": env("LLGO_COMMIT"), + "llgoTag": env("LLGO_TAG"), + "goVersion": env("GO_VERSION"), + "llvmVersion": env("LLVM_VERSION"), + "event": env("LLGO_PERFORMANCE_EVENT", "GITHUB_EVENT_NAME"), + "workflowUrl": workflow_url, + "runnerOS": env("RUNNER_OS"), + "runnerArch": env("RUNNER_ARCH"), + "runnerImage": env("ImageOS"), + "repetitions": number("LLGO_PERFORMANCE_REPETITIONS"), + "benchmarkEnvironment": benchmark_environment(go_stdout), + }, + "configs": list(CONFIGS), + "metric": "sec/op", + "benchmarks": parse_benchstat(csv_path), + "native": { + "text": "benchstat.txt", + "csv": "benchstat.csv", + "rawDir": "raw", + }, + } + + with open(output_path, "w", encoding="utf-8") as destination: + json.dump(document, destination, indent=2) + destination.write("\n") + + +if __name__ == "__main__": + main() diff --git a/ci/llgo-performance/report.sh b/ci/llgo-performance/report.sh index e9ea3b5..598967a 100755 --- a/ci/llgo-performance/report.sh +++ b/ci/llgo-performance/report.sh @@ -6,6 +6,8 @@ benchstat_bin=$2 bench_dir="$run_dir/bench" benchmark_file="$run_dir/benchmarks-llgo-performance.toml" report="$run_dir/benchstat.txt" +csv_report="$run_dir/benchstat.csv" +json_report="$run_dir/results.json" repetitions=10 benchmarks=() @@ -49,4 +51,8 @@ for config in "${configs[@]}"; do done "$benchstat_bin" -table shortname -col toolchain -row .name "${stdout_files[@]}" > "$report" +"$benchstat_bin" -format csv -filter '.unit:sec/op' \ + -table shortname -col toolchain -row .name "${stdout_files[@]}" > "$csv_report" +LLGO_PERFORMANCE_REPETITIONS=$repetitions \ + python3 "$(dirname "$0")/report.py" "$csv_report" "$json_report" "${stdout_files[0]}" cat "$report" diff --git a/ci/llgo-size/publish-site.sh b/ci/llgo-size/publish-site.sh index 00a57b5..05dd549 100755 --- a/ci/llgo-size/publish-site.sh +++ b/ci/llgo-size/publish-site.sh @@ -9,7 +9,7 @@ if [[ -z "$pages_dir" || -z "$site_dir" ]]; then exit 2 fi -for file in index.html app.js style.css _config.yml; do +for file in index.html app.js performance.html performance.js style.css _config.yml; do cp "$site_dir/$file" "$pages_dir/$file" done rm -f "$pages_dir/.nojekyll" diff --git a/ci/llgo-size/publish.sh b/ci/llgo-size/publish.sh index 464695f..90dc01c 100755 --- a/ci/llgo-size/publish.sh +++ b/ci/llgo-size/publish.sh @@ -64,10 +64,9 @@ for source in sorted(os.listdir(runs_dir)): continue os.rename(source_dir, target_dir) PY -cp "$site_dir/index.html" "$pages_dir/index.html" -cp "$site_dir/app.js" "$pages_dir/app.js" -cp "$site_dir/style.css" "$pages_dir/style.css" -cp "$site_dir/_config.yml" "$pages_dir/_config.yml" +for file in index.html app.js performance.html performance.js style.css _config.yml; do + cp "$site_dir/$file" "$pages_dir/$file" +done rm -f "$pages_dir/.nojekyll" run_key=$(python3 - "$result_json" <<'PY' diff --git a/ci/llgo-size/site/index.html b/ci/llgo-size/site/index.html index 84e66a3..3bbcf6e 100644 --- a/ci/llgo-size/site/index.html +++ b/ci/llgo-size/site/index.html @@ -8,6 +8,10 @@
+

LLGo compiler benchmarks

diff --git a/ci/llgo-size/site/performance.html b/ci/llgo-size/site/performance.html new file mode 100644 index 0000000..a5eac9a --- /dev/null +++ b/ci/llgo-size/site/performance.html @@ -0,0 +1,60 @@ + + + + + + LLGo runtime performance history + + + +
+ + +
+
+

LLGo compiler benchmarks

+

Runtime performance

+

Benchstat comparisons for selected Go benchmarks. Go is the baseline; negative deltas mean LLGo is faster.

+
+

Loading results…

+
+ +
+
LLGo release—
+
Go baseline—
+
Environment—
+
Protocol—
+
+ +
+ +

+
+ Workflow run + benchstat text + benchstat CSV +
+
+ +
+
+

benchstat comparison

Execution time

+

Median and confidence interval; LLGo deltas are relative to Go.

+
+
+
+
+
+ + +
+ + + diff --git a/ci/llgo-size/site/performance.js b/ci/llgo-size/site/performance.js new file mode 100644 index 0000000..df71295 --- /dev/null +++ b/ci/llgo-size/site/performance.js @@ -0,0 +1,180 @@ +const performanceState = { + index: null, + runs: new Map(), +}; + +const performanceDom = { + status: document.querySelector("#performance-status"), + release: document.querySelector("#performance-release"), + go: document.querySelector("#performance-go"), + environment: document.querySelector("#performance-environment"), + protocol: document.querySelector("#performance-protocol"), + run: document.querySelector("#performance-run"), + meta: document.querySelector("#performance-meta"), + workflow: document.querySelector("#performance-workflow"), + text: document.querySelector("#performance-text"), + csv: document.querySelector("#performance-csv"), + table: document.querySelector("#performance-table"), +}; + +function performanceEscape(value) { + return String(value == null ? "" : value).replace(/[&<>"']/g, function (character) { + return { "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[character]; + }); +} + +function performanceDate(value) { + if (!value) return "unknown time"; + return new Date(value).toLocaleString(undefined, { dateStyle: "medium", timeStyle: "short" }); +} + +function performanceSeconds(value) { + const seconds = Number(value); + if (!Number.isFinite(seconds)) return "—"; + const absolute = Math.abs(seconds); + if (absolute < 1e-6) return (seconds * 1e9).toFixed(2).replace(/\.00$/, "") + " ns"; + if (absolute < 1e-3) return (seconds * 1e6).toFixed(2).replace(/\.00$/, "") + " µs"; + if (absolute < 1) return (seconds * 1e3).toFixed(2).replace(/\.00$/, "") + " ms"; + return seconds.toFixed(3).replace(/\.000$/, "") + " s"; +} + +function performanceDeltaClass(measurement) { + const delta = Number(measurement && measurement.deltaPercent); + if (!Number.isFinite(delta)) return "flat"; + return delta < 0 ? "good" : delta > 0 ? "bad" : "flat"; +} + +function performanceMeasurement(measurement) { + if (!measurement) return '—'; + return "" + performanceEscape(performanceSeconds(measurement.value)) + "" + + '± ' + performanceEscape(measurement.ci || "—") + ""; +} + +function performanceDelta(measurement) { + if (!measurement) return '—'; + return '' + + performanceEscape(measurement.delta || "—") + "" + + '' + performanceEscape(measurement.p || "") + ""; +} + +function performanceSafeUrl(value) { + const url = String(value || ""); + return /^https:\/\/github\.com\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\/(?:actions\/runs\/[0-9]+|releases\/tag\/.+|commit\/[0-9a-fA-F]{40})$/.test(url) ? url : ""; +} + +function performanceSetLink(element, href) { + if (!href) { + element.hidden = true; + element.removeAttribute("href"); + return; + } + element.hidden = false; + element.href = href; +} + +function performanceReleaseLink(run) { + const repository = String(run.llgoRepository || "xgo-dev/llgo"); + const tag = String(run.llgoTag || ""); + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository) || !tag) return ""; + return "https://github.com/" + repository + "/releases/tag/" + encodeURIComponent(tag); +} + +function performanceRunUrl(meta, path) { + const version = [meta.id || meta.key, meta.attempt || 1, meta.createdAt || ""].join("-"); + return "data/" + path + "?v=" + encodeURIComponent(version); +} + +async function performanceLoadRun(meta) { + if (!performanceState.runs.has(meta.key)) { + performanceState.runs.set(meta.key, (async function () { + const response = await fetch(performanceRunUrl(meta, meta.path)); + if (!response.ok) throw new Error("Cannot load " + meta.path); + return response.json(); + })()); + } + return performanceState.runs.get(meta.key); +} + +function performanceRenderTable(document) { + const rows = (document.benchmarks || []).map(function (entry) { + const values = entry.values || {}; + return "" + + '' + performanceEscape(entry.suite || entry.benchmark) + + '' + performanceEscape(entry.case || entry.benchmark) + "" + + "" + performanceMeasurement(values.Go) + "" + + "" + performanceMeasurement(values.LLGo) + "" + + '' + performanceDelta(values.LLGo) + "" + + "" + performanceMeasurement(values.LLGoFullLTO) + "" + + '' + performanceDelta(values.LLGoFullLTO) + "" + + ""; + }).join(""); + + performanceDom.table.innerHTML = + "" + + 'Benchmark' + + 'Go
sec/op' + + 'LLGo' + + 'LLGo full LTO' + + "" + + "sec/opvs Gosec/opvs Go" + + "" + rows + ""; +} + +async function performanceRender(meta) { + performanceDom.status.classList.remove("error"); + performanceDom.status.textContent = "Loading " + (meta.llgoTag || meta.key) + "…"; + const document = await performanceLoadRun(meta); + const run = document.run || {}; + const environment = run.benchmarkEnvironment || {}; + const tag = run.llgoTag || meta.llgoTag || "—"; + const releaseUrl = performanceSafeUrl(performanceReleaseLink(run)); + performanceDom.release.innerHTML = releaseUrl + ? '' + performanceEscape(tag) + "" + : performanceEscape(tag); + performanceDom.go.textContent = run.goVersion ? "Go " + run.goVersion : "—"; + performanceDom.environment.textContent = [environment.cpu, [environment.goos, environment.goarch].filter(Boolean).join("/")].filter(Boolean).join(" · ") || "—"; + performanceDom.protocol.textContent = (run.repetitions || 10) + " samples per configuration"; + performanceDom.meta.textContent = "Published " + performanceDate(run.createdAt); + + const basePath = meta.path.slice(0, meta.path.lastIndexOf("/") + 1); + performanceSetLink(performanceDom.workflow, performanceSafeUrl(run.workflowUrl)); + performanceSetLink(performanceDom.text, performanceRunUrl(meta, basePath + "benchstat.txt")); + performanceSetLink(performanceDom.csv, performanceRunUrl(meta, basePath + "benchstat.csv")); + performanceRenderTable(document); + performanceDom.status.textContent = (document.benchmarks || []).length + " benchmarks · " + tag; +} + +async function performanceStart() { + try { + const response = await fetch("data/performance/index.json", { cache: "no-store" }); + if (response.status === 404) { + performanceDom.status.textContent = "No published results yet"; + performanceDom.run.innerHTML = ''; + performanceDom.run.disabled = true; + performanceDom.table.innerHTML = 'The table will appear after the next tagged LLGo performance run.'; + return; + } + if (!response.ok) throw new Error("Cannot load performance history"); + performanceState.index = await response.json(); + const runs = performanceState.index.runs || []; + if (!runs.length) throw new Error("Performance history is empty"); + performanceDom.run.innerHTML = runs.map(function (run) { + const label = (run.llgoTag || run.key) + " · " + performanceDate(run.createdAt); + return '"; + }).join(""); + performanceDom.run.addEventListener("change", function () { + const selected = runs.find(function (run) { return run.key === performanceDom.run.value; }); + performanceRender(selected).catch(performanceFail); + }); + await performanceRender(runs[0]); + } catch (error) { + performanceFail(error); + } +} + +function performanceFail(error) { + performanceDom.status.classList.add("error"); + performanceDom.status.textContent = error && error.message ? error.message : "Cannot load performance results"; +} + +performanceStart(); diff --git a/ci/llgo-size/site/style.css b/ci/llgo-size/site/style.css index b916209..dadc300 100644 --- a/ci/llgo-size/site/style.css +++ b/ci/llgo-size/site/style.css @@ -27,6 +27,11 @@ button, input, select { font: inherit; } button { cursor: pointer; } .shell { max-width: 1640px; margin: 0 auto; padding: 24px 22px 52px; } +.site-nav { display: flex; gap: 5px; margin-bottom: 11px; padding: 4px; border: 1px solid #d9e3f1; border-radius: 10px; background: #e9eef6; width: fit-content; } +.site-nav a { padding: 7px 12px; border-radius: 7px; color: #4a5f7e; font-size: 11px; font-weight: 800; text-decoration: none; } +.site-nav a:hover { background: rgba(255, 255, 255, .65); color: #24469f; } +.site-nav a.active { background: #fff; color: var(--accent); box-shadow: 0 2px 8px rgba(19, 42, 79, .09); } + .hero { display: flex; justify-content: space-between; gap: 24px; align-items: end; margin-bottom: 12px; padding: 22px 26px; border: 1px solid #d9e3f7; border-radius: 15px; background: #fff; box-shadow: var(--shadow); } .eyebrow, .label, .control-label { color: var(--accent); font-size: 9px; font-weight: 850; letter-spacing: .12em; text-transform: uppercase; } h1, h2, p { margin-top: 0; } @@ -37,7 +42,7 @@ h2 { margin-bottom: 0; font-size: 19px; letter-spacing: -.02em; } .status::before { width: 6px; height: 6px; border-radius: 50%; background: currentColor; content: ""; } .status.error { background: #fff0f0; color: var(--bad); } -.environment-bar, .commit-toolbar, .card, .compare-hint, .comparison-strip { border: 1px solid var(--line); background: var(--panel); box-shadow: 0 6px 22px rgba(19, 42, 79, .04); } +.environment-bar, .commit-toolbar, .performance-toolbar, .card, .compare-hint, .comparison-strip { border: 1px solid var(--line); background: var(--panel); box-shadow: 0 6px 22px rgba(19, 42, 79, .04); } .environment-bar { display: grid; grid-template-columns: 1.15fr 1.15fr 1fr 1fr; margin-bottom: 11px; border-color: #b9cbea; border-radius: 12px; overflow: hidden; } .environment-bar > div { display: grid; gap: 3px; min-width: 0; padding: 11px 14px; border-right: 1px solid var(--line); } .environment-bar > div:last-child { border-right: 0; } @@ -64,6 +69,25 @@ select:focus, input:focus, button:focus-visible, a:focus-visible { border-color: .comparison-strip strong { font-size: 14px; } .comparison-note { color: var(--muted); font-size: 10px; } +.performance-toolbar { display: grid; grid-template-columns: minmax(220px, 360px) 1fr auto; gap: 14px; align-items: end; margin-bottom: 11px; padding: 10px 13px; border-radius: 12px; } +.performance-meta { align-self: center; margin: 0; color: var(--muted); font-size: 11px; font-weight: 700; } +.result-links { display: flex; gap: 10px; align-self: center; font-size: 11px; font-weight: 750; } +.performance-note { margin: 0; color: var(--muted); font-size: 10px; } +.performance-table-wrap { overflow-x: auto; background: #fff; } +.performance-table { width: 100%; min-width: 880px; border-collapse: separate; border-spacing: 0; font-variant-numeric: tabular-nums; } +.performance-table th, .performance-table td { padding: 8px 10px; border-right: 1px solid #e3e9f1; border-bottom: 1px solid var(--line); text-align: right; white-space: nowrap; } +.performance-table tr > *:last-child { border-right: 0; } +.performance-table tbody tr:last-child > * { border-bottom: 0; } +.performance-table thead th { background: #f7f9fd; color: #526783; font-size: 10px; font-weight: 850; text-align: center; } +.performance-table thead th span { color: var(--muted); font-size: 8px; } +.performance-table tbody tr:hover > * { background: #f8faff; } +.performance-table td strong { display: block; color: #173354; font-size: 11px; } +.performance-table .performance-delta strong.good { color: var(--good); } +.performance-table .performance-delta strong.bad { color: var(--bad); } +.performance-table .performance-delta strong.flat { color: var(--muted); } +.performance-benchmark-cell { position: sticky; left: 0; z-index: 2; width: 36%; min-width: 280px; background: #fff; text-align: left !important; white-space: normal !important; } +.performance-table thead .performance-benchmark-cell { z-index: 3; background: #f7f9fd; } + .card { margin-bottom: 14px; border-radius: 13px; overflow: hidden; } .card-heading { display: flex; justify-content: space-between; gap: 20px; align-items: center; padding: 13px 16px 11px; border-bottom: 1px solid var(--line); background: #fff; } .card-heading p { margin-bottom: 2px; } @@ -155,6 +179,8 @@ footer { display: flex; gap: 9px; flex-wrap: wrap; padding: 4px 2px; color: var( .comparison-strip { grid-template-columns: auto 1fr 1fr auto; } .comparison-note { grid-column: 1 / -1; } .history-controls { grid-template-columns: 1fr; } + .performance-toolbar { grid-template-columns: minmax(220px, 1fr) 1fr; } + .result-links { grid-column: 1 / -1; } } @media (max-width: 640px) { @@ -168,4 +194,8 @@ footer { display: flex; gap: 9px; flex-wrap: wrap; padding: 4px 2px; color: var( .comparison-strip button { justify-self: start; } .card-heading { align-items: start; } .rank-legend { display: none; } + .site-nav { display: grid; width: 100%; grid-template-columns: 1fr 1fr; text-align: center; } + .performance-toolbar { grid-template-columns: 1fr; } + .result-links { grid-column: auto; flex-wrap: wrap; } + .performance-benchmark-cell { min-width: 220px; } } From 65e251ca4ed838adebf3c637d163ce1203fc3491 Mon Sep 17 00:00:00 2001 From: ZhouGuangyuan Date: Wed, 19 Aug 2026 18:08:58 +0800 Subject: [PATCH 3/3] ci: reduce runtime performance samples --- .github/workflows/llgo-runtime-performance.yml | 4 ++-- ci/llgo-performance/README.md | 6 ++++-- ci/llgo-performance/report.sh | 8 +++++--- ci/llgo-size/site/performance.html | 2 +- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/llgo-runtime-performance.yml b/.github/workflows/llgo-runtime-performance.yml index 2b17990..57c74b1 100644 --- a/.github/workflows/llgo-runtime-performance.yml +++ b/.github/workflows/llgo-runtime-performance.yml @@ -51,7 +51,7 @@ jobs: printf 'LLGO_TAG=%s\n' "$DISPATCH_LLGO_TAG" >> "$GITHUB_ENV" printf 'GO_VERSION=%s\n' "$GO_VERSION" >> "$GITHUB_ENV" printf 'LLVM_VERSION=%s\n' "$LLVM_VERSION" >> "$GITHUB_ENV" - printf 'LLGO_PERFORMANCE_REPETITIONS=10\n' >> "$GITHUB_ENV" + printf 'LLGO_PERFORMANCE_REPETITIONS=5\n' >> "$GITHUB_ENV" mkdir -p .ci printf '%s\n' "$GO_VERSION" > .ci/go-version @@ -115,7 +115,7 @@ jobs: "$GITHUB_WORKSPACE/.ci/bent" \ -B=benchmarks-llgo-performance.toml \ -C=configurations-llgo-performance.toml \ - -N=10 -G + -N=5 -G ) ci/llgo-performance/report.sh \ .ci/bent-performance \ diff --git a/ci/llgo-performance/README.md b/ci/llgo-performance/README.md index 5898a40..19e8108 100644 --- a/ci/llgo-performance/README.md +++ b/ci/llgo-performance/README.md @@ -1,9 +1,11 @@ # LLGo runtime performance This job compares the pinned Go and LLGo toolchains with Bent. It runs the -checked-in cases ten times each, grouped by benchmark to reduce time-local +checked-in cases five times each, grouped by benchmark to reduce time-local runner noise, and publishes the raw Bent output, native benchstat text/CSV, and -a structured result used by the runtime-performance Pages table. +a structured result used by the runtime-performance Pages table. Benchstat uses +a 90% confidence interval because its default 95% interval requires at least +six samples. It runs only after the `xgo-dev/llgo` release workflow publishes a new tag and dispatches the exact tag and commit. Ordinary pushes and pull requests do not diff --git a/ci/llgo-performance/report.sh b/ci/llgo-performance/report.sh index 598967a..9f20d0e 100755 --- a/ci/llgo-performance/report.sh +++ b/ci/llgo-performance/report.sh @@ -8,7 +8,8 @@ benchmark_file="$run_dir/benchmarks-llgo-performance.toml" report="$run_dir/benchstat.txt" csv_report="$run_dir/benchstat.csv" json_report="$run_dir/results.json" -repetitions=10 +repetitions=5 +confidence=0.90 benchmarks=() while IFS= read -r benchmark; do @@ -50,8 +51,9 @@ for config in "${configs[@]}"; do fi done -"$benchstat_bin" -table shortname -col toolchain -row .name "${stdout_files[@]}" > "$report" -"$benchstat_bin" -format csv -filter '.unit:sec/op' \ +"$benchstat_bin" -confidence "$confidence" \ + -table shortname -col toolchain -row .name "${stdout_files[@]}" > "$report" +"$benchstat_bin" -confidence "$confidence" -format csv -filter '.unit:sec/op' \ -table shortname -col toolchain -row .name "${stdout_files[@]}" > "$csv_report" LLGO_PERFORMANCE_REPETITIONS=$repetitions \ python3 "$(dirname "$0")/report.py" "$csv_report" "$json_report" "${stdout_files[0]}" diff --git a/ci/llgo-size/site/performance.html b/ci/llgo-size/site/performance.html index a5eac9a..f2579af 100644 --- a/ci/llgo-size/site/performance.html +++ b/ci/llgo-size/site/performance.html @@ -42,7 +42,7 @@

Runtime performance

benchstat comparison

Execution time

-

Median and confidence interval; LLGo deltas are relative to Go.

+

Median and 90% confidence interval; LLGo deltas are relative to Go.