From 8e820e0d27563b2212686b99c52f9f27a241c272 Mon Sep 17 00:00:00 2001 From: Patrick O'Reilly Date: Mon, 14 Sep 2026 23:43:49 -0700 Subject: [PATCH 1/8] docker: build and publish a multi-arch image (#548) The published image was amd64-only, so every Apple Silicon user -- most of the audience the MCP work targets -- got a platform-mismatch warning on the very first command in the README. #550 established the code builds and passes its full test suite on aarch64, which is what makes this possible. Native runner per architecture, not buildx+QEMU. This image compiles the whole source tree: 1753 build jobs, ~10 minutes natively. Emulated that is hours and would blow the 90-minute timeout. `ubuntu-24.04-arm` builds it natively in about the same wall time as x86, in parallel. That choice pays a second dividend: every gate already in this workflow -- the MCP smoke against the containerised server, the in-container orlyc compile, the REPL entrypoint mode -- now runs ON the arm image. The published arm64 artifact is therefore proven to work, not merely to compile, which emulated cross-builds would not have given us. Publishing is two stages: each leg pushes an arch-suffixed tag, then a manifest job stitches them with `imagetools create`. Arch tags are left pullable on purpose -- they are the manifest's inputs and the thing you want when only one architecture misbehaves. The manifest job runs only if BOTH legs passed their gates; a manifest pointing at an image that failed its smoke would be worse than no manifest. It then verifies the result rather than assuming it: the published manifest must advertise both amd64 and arm64, which is the actual claim being made. Note this does not change the published image by itself -- that needs a dispatch or a v* tag. README still says the published image is amd64, which stays true until the first multi-arch publish lands; updating it before then would be describing something that has not happened. --- .github/workflows/docker.yml | 104 +++++++++++++++++++++++++---- changelog.d/548-multiarch-image.md | 1 + 2 files changed, 91 insertions(+), 14 deletions(-) create mode 100644 changelog.d/548-multiarch-image.md diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b491ed0d..235f3e90 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,9 +1,18 @@ # Build (and on demand, publish) the Orly runtime image (#530). # -# Pull requests touching the docker files build the image and gate it with -# the MCP smoke pointed at the CONTAINERIZED server (the same smoke.mjs that -# gates clients/mcp against a host orlyi) -- no push. A version tag or a -# manual dispatch additionally pushes to ghcr.io/orlyatomics/orly. +# Pull requests touching the docker files build the image on BOTH supported +# architectures and gate each one with the MCP smoke pointed at the +# CONTAINERIZED server (the same smoke.mjs that gates clients/mcp against a +# host orlyi) -- no push. A version tag or a manual dispatch additionally +# pushes each arch and then stitches them into one multi-arch manifest. +# +# Why two native runners rather than buildx+QEMU (#548): the arm64 half would +# be emulated, and this image compiles the whole source tree -- 1753 build jobs, +# ~10 minutes natively. Under qemu that is hours and would blow the timeout. +# `ubuntu-24.04-arm` builds it natively in about the same time as x86, in +# parallel, and has the side benefit that every gate below (MCP smoke, +# in-container orlyc compile, REPL mode) runs ON the arm image -- so the +# published arm64 image is proven to work, not merely to build. name: docker on: @@ -19,21 +28,40 @@ on: jobs: image: - name: build image + in-container smoke - runs-on: ubuntu-24.04 + name: build image + in-container smoke (${{ matrix.arch }}) + strategy: + # Report both architectures even when one fails -- "arm is broken, x86 is + # fine" is the useful signal, and fail-fast would hide it. + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} timeout-minutes: 90 permissions: contents: read packages: write + env: + # Arch-suffixed so the two matrix legs cannot collide in the registry, + # and so a single-arch problem can be pulled and poked at directly. + IMAGE: ghcr.io/orlyatomics/orly:ci-${{ matrix.arch }} steps: - uses: actions/checkout@v6 + - name: Report runner architecture + run: uname -m + + # Plain `docker build`, not buildx: each runner is native for its own + # architecture, so there is nothing to cross-compile or emulate. - name: Build image - run: docker build -t ghcr.io/orlyatomics/orly:ci . + run: docker build -t "$IMAGE" . - name: Start container run: | - docker run -d --name orly -p 8082:8082 ghcr.io/orlyatomics/orly:ci + docker run -d --name orly -p 8082:8082 "$IMAGE" for _ in $(seq 1 60); do if ss -tln | grep -q ':8082'; then exit 0; fi sleep 1 @@ -74,7 +102,7 @@ jobs: - name: REPL mode against the image run: | printf '40 + 2;\n:quit\n' \ - | docker run -i --rm ghcr.io/orlyatomics/orly:ci repl \ + | docker run -i --rm "$IMAGE" repl \ | tee /tmp/repl-out grep -qE '(^|orly> )42$' /tmp/repl-out @@ -82,14 +110,62 @@ jobs: if: always() run: docker logs orly 2>&1 | tail -60 - - name: Push to ghcr.io + # Each leg pushes its own arch-tagged image; the manifest job below + # stitches them together. Pushing per-arch tags (rather than the + # push-by-digest dance) keeps this a plain `docker push` and leaves a + # pullable single-arch image behind for debugging. + - name: Push arch image to ghcr.io if: github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v') run: | echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - docker tag ghcr.io/orlyatomics/orly:ci ghcr.io/orlyatomics/orly:latest - docker push ghcr.io/orlyatomics/orly:latest + docker tag "$IMAGE" "ghcr.io/orlyatomics/orly:latest-${{ matrix.arch }}" + docker push "ghcr.io/orlyatomics/orly:latest-${{ matrix.arch }}" if [[ "${GITHUB_REF}" == refs/tags/v* ]]; then ver="${GITHUB_REF#refs/tags/}" - docker tag ghcr.io/orlyatomics/orly:ci "ghcr.io/orlyatomics/orly:${ver}" - docker push "ghcr.io/orlyatomics/orly:${ver}" + docker tag "$IMAGE" "ghcr.io/orlyatomics/orly:${ver}-${{ matrix.arch }}" + docker push "ghcr.io/orlyatomics/orly:${ver}-${{ matrix.arch }}" fi + + manifest: + name: publish multi-arch manifest + needs: image + # Only on a real publish, and only if BOTH arches built and passed their + # gates -- a manifest listing an image that failed its smoke would be worse + # than no manifest at all. + if: github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v') + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + packages: write + steps: + - name: Log in to ghcr.io + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + + # `imagetools create` builds a manifest list from the per-arch tags, so + # `docker pull ghcr.io/orlyatomics/orly` resolves to the right one and + # Apple Silicon stops printing a platform-mismatch warning on the first + # command in the README (#548). + - name: Stitch :latest + run: | + docker buildx imagetools create -t ghcr.io/orlyatomics/orly:latest \ + ghcr.io/orlyatomics/orly:latest-amd64 \ + ghcr.io/orlyatomics/orly:latest-arm64 + + - name: Stitch the version tag + if: startsWith(github.ref, 'refs/tags/v') + run: | + ver="${GITHUB_REF#refs/tags/}" + docker buildx imagetools create -t "ghcr.io/orlyatomics/orly:${ver}" \ + "ghcr.io/orlyatomics/orly:${ver}-amd64" \ + "ghcr.io/orlyatomics/orly:${ver}-arm64" + + # Assert the thing this whole change exists for, rather than assuming the + # stitch worked: the published manifest must advertise both platforms. + - name: Verify the manifest lists both platforms + run: | + got=$(docker buildx imagetools inspect --raw ghcr.io/orlyatomics/orly:latest \ + | grep -oE '"architecture":"[a-z0-9]+"' | sort -u | tr '\n' ' ') + echo "manifest advertises: $got" + echo "$got" | grep -q '"architecture":"amd64"' + echo "$got" | grep -q '"architecture":"arm64"' diff --git a/changelog.d/548-multiarch-image.md b/changelog.d/548-multiarch-image.md new file mode 100644 index 00000000..09b99868 --- /dev/null +++ b/changelog.d/548-multiarch-image.md @@ -0,0 +1 @@ +- **Changed**: the docker workflow builds the image on both `linux/amd64` and `linux/arm64` and, on a publish, stitches them into one multi-arch manifest — so `docker run ghcr.io/orlyatomics/orly` resolves natively on Apple Silicon instead of printing a platform-mismatch warning on the first command in the README. Each architecture is built on its own **native** runner (`ubuntu-24.04` / `ubuntu-24.04-arm`) rather than via buildx+QEMU: the image compiles the whole source tree (1753 build jobs, ~10 min natively), which under emulation would be hours and exceed the job timeout. The side benefit is that every existing gate — MCP smoke against the containerised server, in-container `orlyc` compile, REPL entrypoint mode — now runs **on the arm image too**, so the published arm64 artifact is proven to work rather than merely to build. The manifest job runs only when both legs pass, and asserts the published manifest advertises both platforms rather than assuming the stitch worked (#548). From eb197b1cfa39deee7c46038e152219c6e9748fc1 Mon Sep 17 00:00:00 2001 From: Patrick O'Reilly Date: Tue, 15 Sep 2026 01:16:58 -0700 Subject: [PATCH 2/8] docker: report runner capacity, and tune the arm build per-arch (#548) The arm64 leg was killed at the 90-minute cap, still inside `jhm -c release`. So it is >90 minutes at 4 workers, and how much over is unknown. The same runner class does `make debug` in 14 minutes (ci.yml, merged in #550), so aarch64 is not simply slow. The difference in the Dockerfile is `-c release` -- an LTO build. LTO is memory-hungry and largely serial at link time, which makes the leading hypothesis memory pressure rather than CPU: if the arm runner has less RAM per core, four concurrent LTO jobs thrash and FEWER workers is faster. Hence per-arch matrix values: 4 workers / 90 min on x86 (unchanged, it builds in ~10), 2 workers / 180 min on arm. The generous timeout is so a slow-but-working build is not killed while we are still learning what it actually costs. That is a hypothesis, not a diagnosis, and the reason the first failure taught us so little is that the job reported only `uname -m`. It now reports nproc and free -g as well. "How many cores and how much RAM did that runner have" is the first question on any slow leg and the one that cannot be answered after the fact. If the capacity report shows ample free memory, the hypothesis is wrong and the next thing to look at is aarch64 LTO codegen time directly. --- .github/workflows/docker.yml | 29 +++++++++++++++++++++++++---- changelog.d/548-multiarch-image.md | 1 + 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 235f3e90..c71ae5e6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -37,10 +37,23 @@ jobs: include: - arch: amd64 runner: ubuntu-24.04 + # Matches the Dockerfile default; the x86 leg builds in ~10 min. + workers: 4 + timeout: 90 - arch: arm64 runner: ubuntu-24.04-arm + # The arm leg ran >79 min on `jhm -c release` at 4 workers and was + # killed by a 90-minute cap, while the SAME runner class does + # `make debug` in 14 min (ci.yml). The difference is LTO: it is + # memory-hungry and largely serial at link time, so if the arm + # runner has less RAM per core, 4 concurrent LTO jobs thrash and + # FEWER workers is faster. 2 is the first thing to try; the + # capacity report above is what tells us whether that was the + # right diagnosis. + workers: 2 + timeout: 180 runs-on: ${{ matrix.runner }} - timeout-minutes: 90 + timeout-minutes: ${{ matrix.timeout }} permissions: contents: read packages: write @@ -51,13 +64,21 @@ jobs: steps: - uses: actions/checkout@v6 - - name: Report runner architecture - run: uname -m + # Cores and RAM, not just the arch: the release build in the Dockerfile is + # LTO, which is memory-hungry and largely serial at link time. When a leg + # is slow, "how many cores and how much RAM did it have" is the first + # question and the one that is impossible to answer after the fact. + - name: Report runner capacity + run: | + uname -m + echo "nproc: $(nproc)" + free -g || true + echo "JHM_WORKER_COUNT will be: ${{ matrix.workers }}" # Plain `docker build`, not buildx: each runner is native for its own # architecture, so there is nothing to cross-compile or emulate. - name: Build image - run: docker build -t "$IMAGE" . + run: docker build --build-arg JHM_WORKER_COUNT=${{ matrix.workers }} -t "$IMAGE" . - name: Start container run: | diff --git a/changelog.d/548-multiarch-image.md b/changelog.d/548-multiarch-image.md index 09b99868..c43e213d 100644 --- a/changelog.d/548-multiarch-image.md +++ b/changelog.d/548-multiarch-image.md @@ -1 +1,2 @@ - **Changed**: the docker workflow builds the image on both `linux/amd64` and `linux/arm64` and, on a publish, stitches them into one multi-arch manifest — so `docker run ghcr.io/orlyatomics/orly` resolves natively on Apple Silicon instead of printing a platform-mismatch warning on the first command in the README. Each architecture is built on its own **native** runner (`ubuntu-24.04` / `ubuntu-24.04-arm`) rather than via buildx+QEMU: the image compiles the whole source tree (1753 build jobs, ~10 min natively), which under emulation would be hours and exceed the job timeout. The side benefit is that every existing gate — MCP smoke against the containerised server, in-container `orlyc` compile, REPL entrypoint mode — now runs **on the arm image too**, so the published arm64 artifact is proven to work rather than merely to build. The manifest job runs only when both legs pass, and asserts the published manifest advertises both platforms rather than assuming the stitch worked (#548). +- **Changed**: the docker image job reports `nproc` and `free -g` alongside the architecture, and takes its build parallelism and timeout from the matrix — 4 workers / 90 min on x86, 2 workers / 180 min on arm64. The first arm attempt was killed at the 90-minute cap still inside `jhm -c release`, while the same runner class completes `make debug` in 14 minutes (ci.yml). The difference is LTO, which is memory-hungry and largely serial at link time, so if the arm runner has less RAM per core then *fewer* concurrent LTO jobs is faster rather than slower. That is a hypothesis; the capacity report is what confirms or kills it, and not having it is why the first failure was uninformative (#548). From 95491d939d926d1f81ea3a9b831b40a7344b16c6 Mon Sep 17 00:00:00 2001 From: Patrick O'Reilly Date: Tue, 15 Sep 2026 14:06:15 -0700 Subject: [PATCH 3/8] ci: one-shot profile of where the aarch64 release build spends its time (#548) The multi-arch image is blocked on an arm leg that never finishes `jhm -c release`: >90 min at 4 workers, >180 min at 2, against 10m13s for the entire amd64 job. What is already ruled out: resource starvation. Capacity is identical on both runners (4 cores, 15GB, zero swap used), and halving the workers made it WORSE rather than better, which is the opposite of what memory thrash predicts. The same runner class does `make debug` -- no LTO -- in 14m7s, slightly FASTER than x86, so aarch64 compiles this tree fine. That leaves one question worth answering before choosing a fix: is the cost spread across ~1753 compile targets, or concentrated in the LTO link? Dropping LTO for the image and chasing one pathological translation unit are very different pieces of work, and picking between them by intuition is how the last two rounds were wasted. Two instruments, both cheap: - jhm's progress output timestamped. It prints [done/total], so the timestamps show directly whether the counter crawls (compile-bound) or races to the end and then stalls (link-bound). - a 30s process sampler. If lto1 holds a core for an hour, that settles it. Runs on BOTH arches so the comparison is like-for-like rather than against a number remembered from another workflow, uploads both logs as artifacts, and prints the progress curve even when killed -- a partial run still shows the shape. workflow_dispatch only: this is expensive and diagnostic, not a gate. --- .github/workflows/arm-lto-profile.yml | 109 ++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 .github/workflows/arm-lto-profile.yml diff --git a/.github/workflows/arm-lto-profile.yml b/.github/workflows/arm-lto-profile.yml new file mode 100644 index 00000000..ed90be09 --- /dev/null +++ b/.github/workflows/arm-lto-profile.yml @@ -0,0 +1,109 @@ +# One-shot diagnostic for #548: where does the aarch64 release build spend its +# time? +# +# The multi-arch image (#552) is blocked because the arm leg never finishes +# `jhm -c release` -- >90 min at 4 workers, >180 min at 2, against 10m13s for +# the entire amd64 job. Runner capacity is identical (4 cores, 15GB, no swap +# used), so it is not resource starvation, and halving the workers made it +# WORSE, which kills the memory-thrash theory outright. The same runner class +# does `make debug` -- no LTO -- in 14m7s, slightly FASTER than x86. +# +# So the question is narrow: is the arm cost spread across compiling ~1753 +# targets, or concentrated in the LTO link? Those have entirely different +# answers -- drop LTO for the image, versus chase one pathological unit -- and +# nobody should choose between them by guessing. +# +# Deliberately workflow_dispatch only: expensive, diagnostic, not a gate. +name: arm-lto-profile + +on: + workflow_dispatch: + +jobs: + profile: + name: profile jhm -c release (${{ matrix.arch }}) + strategy: + # Both arches, so the comparison is like-for-like rather than against a + # remembered number from a different workflow. + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + # Long enough to actually finish if it can. If it cannot, the timestamped + # progress still shows the SHAPE of the curve, which is the whole point. + timeout-minutes: 330 + steps: + - uses: actions/checkout@v6 + + - name: Capacity + run: | + uname -m + echo "nproc: $(nproc)" + free -g + + - name: Install system dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq \ + build-essential gcc g++ uuid-dev libgmp-dev libaio-dev \ + libsnappy-dev libreadline-dev libboost-system-dev zlib1g-dev \ + bison flex valgrind moreutils + + - name: Bootstrap + run: make bootstrap version + + # Sample what is actually running, every 30s. If this shows `lto1` holding + # a core for an hour, the argument is over. + - name: Start process sampler + run: | + ( while true; do + echo "--- $(date -u +%H:%M:%S)" + ps -eo comm,etimes,pcpu,rss --sort=-pcpu | head -4 + sleep 30 + done ) > /tmp/sampler.log 2>&1 & + echo $! > /tmp/sampler.pid + + # Timestamp every jhm line. jhm prints `[done/total] waiting: n`, so the + # timestamps say directly whether the counter crawls (compile-bound) or + # races to the end and then stalls (link-bound). + - name: jhm -c release, timestamped + run: | + set -o pipefail + PATH="$PWD/tools:$PATH" \ + jhm -c release --worker-count 4 orly/server/orlyi orly/orlyc \ + 2>&1 | ts '[%H:%M:%S]' | tee /tmp/build.log | tail -40 + + - name: Stop sampler + if: always() + run: kill "$(cat /tmp/sampler.pid)" 2>/dev/null || true + + # The two things worth reading even if the job was killed mid-build. + - name: Progress curve + if: always() + run: | + echo "=== first 10 timestamped lines:" + head -10 /tmp/build.log || true + echo "=== last 20:" + tail -20 /tmp/build.log || true + echo + echo "=== when the progress counter moved (dedup'd):" + grep -oE '^\[[0-9:]+\] \[[0-9]+/[0-9]+\]' /tmp/build.log 2>/dev/null \ + | awk '{print $2, $1}' | uniq -f1 | tail -40 || true + + - name: Sampler tail + if: always() + run: tail -80 /tmp/sampler.log || true + + - name: Upload logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: lto-profile-${{ matrix.arch }} + path: | + /tmp/build.log + /tmp/sampler.log + if-no-files-found: warn From c5a2265b7ff0cf385cf496b418eca07fead391fc Mon Sep 17 00:00:00 2001 From: Patrick O'Reilly Date: Tue, 15 Sep 2026 14:06:47 -0700 Subject: [PATCH 4/8] ci: let the profile workflow run from this branch (#548) workflow_dispatch is only dispatchable once the workflow exists on the DEFAULT branch, and this diagnostic has to run before anything merges. Branch-scoped push trigger, to be removed with the rest of this file once the arm question is answered. --- .github/workflows/arm-lto-profile.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/arm-lto-profile.yml b/.github/workflows/arm-lto-profile.yml index ed90be09..756a3689 100644 --- a/.github/workflows/arm-lto-profile.yml +++ b/.github/workflows/arm-lto-profile.yml @@ -18,6 +18,12 @@ name: arm-lto-profile on: workflow_dispatch: + # TEMPORARY, and the reason is a GitHub constraint rather than a preference: + # `workflow_dispatch` is only dispatchable once the workflow exists on the + # DEFAULT branch, and this diagnostic needs to run before anything merges. + # Drop this trigger (or the whole file) once #548's arm question is answered. + push: + branches: [feat/548-multiarch-image] jobs: profile: From 2be3618edfbdc601bb7b964a7439b4c6db48182f Mon Sep 17 00:00:00 2001 From: Patrick O'Reilly Date: Tue, 15 Sep 2026 14:16:44 -0700 Subject: [PATCH 5/8] docker: pin the build platform, and find out if the arm image build was emulated (#548) The profiling run reframes this entirely. `jhm -c release` on the BARE arm runner completes in 8m06s -- 884 targets, steady progress, no stall, and faster than the x86 leg. The same command inside `docker build` on the same runner class never finished in 180 minutes. So neither of my two previous explanations survives. It is not memory pressure (identical 4-core/15GB runners, zero swap, and halving the workers made it worse) and it is not aarch64 LTO codegen (8 minutes, natively, at 4 workers). The build is healthy on arm. What is pathological is the build INSIDE docker on arm, and a >20x gap on identical hardware is the signature of qemu rather than of slow codegen. Hence: pass --platform explicitly rather than trusting the daemon default, log docker's client/server arch, the registered binfmt handlers, and what ubuntu:24.04 actually resolves to on that runner, and assert after the build that the image really is the architecture we asked for. If the default was resolving to amd64, every RUN line in the Dockerfile was going through emulation, which would explain everything. If it was already correct, pinning costs nothing and the diagnostics say where to look next. Workers on arm go back to 4, since the reason they were lowered has been refuted. The timeout stays generous until a native arm image build has actually been observed to finish. --- .github/workflows/docker.yml | 47 ++++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c71ae5e6..52e4a7a7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -42,15 +42,14 @@ jobs: timeout: 90 - arch: arm64 runner: ubuntu-24.04-arm - # The arm leg ran >79 min on `jhm -c release` at 4 workers and was - # killed by a 90-minute cap, while the SAME runner class does - # `make debug` in 14 min (ci.yml). The difference is LTO: it is - # memory-hungry and largely serial at link time, so if the arm - # runner has less RAM per core, 4 concurrent LTO jobs thrash and - # FEWER workers is faster. 2 is the first thing to try; the - # capacity report above is what tells us whether that was the - # right diagnosis. - workers: 2 + # Back to 4: the memory-thrash theory was tested and refuted -- + # both runners are 4-core/15GB with zero swap used, and HALVING the + # workers made it worse (>90 min at 4, >180 min at 2). The profiling + # run then showed `jhm -c release` completing on the bare arm runner + # in 8m06s at 4 workers, so the build is not the problem. Timeout + # stays generous until a native arm image build has actually been + # observed to finish. + workers: 4 timeout: 180 runs-on: ${{ matrix.runner }} timeout-minutes: ${{ matrix.timeout }} @@ -77,8 +76,36 @@ jobs: # Plain `docker build`, not buildx: each runner is native for its own # architecture, so there is nothing to cross-compile or emulate. + # Is this build NATIVE, or is it being emulated? That is the whole question + # after the profiling run (#548): `jhm -c release` on the bare arm runner + # finishes in 8m06s -- faster than x86 -- while the SAME command inside + # `docker build` on the same runner class never finished in 180 minutes. A + # >20x gap on identical hardware is the signature of qemu, not of slow + # codegen. + - name: Docker and emulation state + run: | + docker version --format 'client={{.Client.Arch}} server={{.Server.Arch}}' || true + echo "--- registered binfmt handlers (any qemu-* here means emulation is available):" + ls /proc/sys/fs/binfmt_misc/ 2>/dev/null | grep -i qemu || echo " none" + echo "--- what arch does the base image resolve to here:" + docker pull -q ubuntu:24.04 >/dev/null 2>&1 || true + docker image inspect ubuntu:24.04 --format ' ubuntu:24.04 -> {{.Architecture}}' || true + + # --platform pinned explicitly rather than relying on the daemon's default. + # If the default was resolving to amd64 on the arm runner, every RUN line + # was going through qemu, which explains the 20x. Pinning it costs nothing + # if the default was already correct. - name: Build image - run: docker build --build-arg JHM_WORKER_COUNT=${{ matrix.workers }} -t "$IMAGE" . + run: | + docker build --platform "linux/${{ matrix.arch }}" \ + --build-arg JHM_WORKER_COUNT=${{ matrix.workers }} \ + -t "$IMAGE" . + + - name: Confirm the built image is the intended architecture + run: | + got=$(docker image inspect "$IMAGE" --format '{{.Architecture}}') + echo "built image architecture: $got (wanted ${{ matrix.arch }})" + test "$got" = "${{ matrix.arch }}" - name: Start container run: | From a1bd72d167a852de46aeb57be60dbe169e7f2fa9 Mon Sep 17 00:00:00 2001 From: Patrick O'Reilly Date: Tue, 15 Sep 2026 15:06:45 -0700 Subject: [PATCH 6/8] ci: minimal repro for the orlyc hang on aarch64 (#554) The profiling round answered #548's question and produced a better one. `jhm -c release` is fine on arm: 482s inside docker, 8m06s on the bare runner, both faster than amd64's 11m05s. What hangs is the NEXT docker step -- compiling the example packages with the just-built orlyc, which printed its header and then nothing for 37 minutes before the job was cancelled. Per the Dockerfile's own comment, each orlyc run stands up and tears down an embedded mem-sim server, so that is the suspect. This reproduces it in ~9 minutes rather than a 45-minute docker build: build orlyc alone, run it on one package with a 300s deadline, and if it is still alive, interrogate the live process rather than merely noting that it never returned -- process state, kernel wait channel per thread, and all-thread gdb backtraces. amd64 runs as the control, since orlyc demonstrably works there. Replaces the LTO profiler, which has served its purpose. --- .github/workflows/arm-lto-profile.yml | 115 ----------------------- .github/workflows/orlyc-arm-repro.yml | 130 ++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 115 deletions(-) delete mode 100644 .github/workflows/arm-lto-profile.yml create mode 100644 .github/workflows/orlyc-arm-repro.yml diff --git a/.github/workflows/arm-lto-profile.yml b/.github/workflows/arm-lto-profile.yml deleted file mode 100644 index 756a3689..00000000 --- a/.github/workflows/arm-lto-profile.yml +++ /dev/null @@ -1,115 +0,0 @@ -# One-shot diagnostic for #548: where does the aarch64 release build spend its -# time? -# -# The multi-arch image (#552) is blocked because the arm leg never finishes -# `jhm -c release` -- >90 min at 4 workers, >180 min at 2, against 10m13s for -# the entire amd64 job. Runner capacity is identical (4 cores, 15GB, no swap -# used), so it is not resource starvation, and halving the workers made it -# WORSE, which kills the memory-thrash theory outright. The same runner class -# does `make debug` -- no LTO -- in 14m7s, slightly FASTER than x86. -# -# So the question is narrow: is the arm cost spread across compiling ~1753 -# targets, or concentrated in the LTO link? Those have entirely different -# answers -- drop LTO for the image, versus chase one pathological unit -- and -# nobody should choose between them by guessing. -# -# Deliberately workflow_dispatch only: expensive, diagnostic, not a gate. -name: arm-lto-profile - -on: - workflow_dispatch: - # TEMPORARY, and the reason is a GitHub constraint rather than a preference: - # `workflow_dispatch` is only dispatchable once the workflow exists on the - # DEFAULT branch, and this diagnostic needs to run before anything merges. - # Drop this trigger (or the whole file) once #548's arm question is answered. - push: - branches: [feat/548-multiarch-image] - -jobs: - profile: - name: profile jhm -c release (${{ matrix.arch }}) - strategy: - # Both arches, so the comparison is like-for-like rather than against a - # remembered number from a different workflow. - fail-fast: false - matrix: - include: - - arch: amd64 - runner: ubuntu-24.04 - - arch: arm64 - runner: ubuntu-24.04-arm - runs-on: ${{ matrix.runner }} - # Long enough to actually finish if it can. If it cannot, the timestamped - # progress still shows the SHAPE of the curve, which is the whole point. - timeout-minutes: 330 - steps: - - uses: actions/checkout@v6 - - - name: Capacity - run: | - uname -m - echo "nproc: $(nproc)" - free -g - - - name: Install system dependencies - run: | - sudo apt-get update -qq - sudo apt-get install -y -qq \ - build-essential gcc g++ uuid-dev libgmp-dev libaio-dev \ - libsnappy-dev libreadline-dev libboost-system-dev zlib1g-dev \ - bison flex valgrind moreutils - - - name: Bootstrap - run: make bootstrap version - - # Sample what is actually running, every 30s. If this shows `lto1` holding - # a core for an hour, the argument is over. - - name: Start process sampler - run: | - ( while true; do - echo "--- $(date -u +%H:%M:%S)" - ps -eo comm,etimes,pcpu,rss --sort=-pcpu | head -4 - sleep 30 - done ) > /tmp/sampler.log 2>&1 & - echo $! > /tmp/sampler.pid - - # Timestamp every jhm line. jhm prints `[done/total] waiting: n`, so the - # timestamps say directly whether the counter crawls (compile-bound) or - # races to the end and then stalls (link-bound). - - name: jhm -c release, timestamped - run: | - set -o pipefail - PATH="$PWD/tools:$PATH" \ - jhm -c release --worker-count 4 orly/server/orlyi orly/orlyc \ - 2>&1 | ts '[%H:%M:%S]' | tee /tmp/build.log | tail -40 - - - name: Stop sampler - if: always() - run: kill "$(cat /tmp/sampler.pid)" 2>/dev/null || true - - # The two things worth reading even if the job was killed mid-build. - - name: Progress curve - if: always() - run: | - echo "=== first 10 timestamped lines:" - head -10 /tmp/build.log || true - echo "=== last 20:" - tail -20 /tmp/build.log || true - echo - echo "=== when the progress counter moved (dedup'd):" - grep -oE '^\[[0-9:]+\] \[[0-9]+/[0-9]+\]' /tmp/build.log 2>/dev/null \ - | awk '{print $2, $1}' | uniq -f1 | tail -40 || true - - - name: Sampler tail - if: always() - run: tail -80 /tmp/sampler.log || true - - - name: Upload logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: lto-profile-${{ matrix.arch }} - path: | - /tmp/build.log - /tmp/sampler.log - if-no-files-found: warn diff --git a/.github/workflows/orlyc-arm-repro.yml b/.github/workflows/orlyc-arm-repro.yml new file mode 100644 index 00000000..b000416d --- /dev/null +++ b/.github/workflows/orlyc-arm-repro.yml @@ -0,0 +1,130 @@ +# One-shot diagnostic for #554: why does `orlyc` hang on aarch64? +# +# The profiling round answered #548's original question and produced a better +# one. `jhm -c release` is FINE on arm -- 482s inside docker, 8m06s on the bare +# runner, both faster than the amd64 leg's 11m05s. What hangs is the NEXT docker +# step: compiling the example packages with the freshly built orlyc, which +# emitted its header line and then nothing for 37 minutes. +# +# Per the Dockerfile's own comment, "each orlyc run stands up and tears down its +# embedded mem-sim server" -- so the suspicion is that server never completes its +# handshake or teardown on aarch64. +# +# `make test` does not exercise the orlyc shell-out path and lang_test.py is not +# in the arm job, which is why #550 could be green while this was broken. +# +# This reproduces it in ~9 minutes instead of a 45-minute docker build, and if it +# hangs, says WHERE: thread backtraces, kernel wait channel, and process state. +# amd64 runs as the control -- orlyc demonstrably works there. +# +# Deliberately not a gate. +name: orlyc-arm-repro + +on: + workflow_dispatch: + # TEMPORARY, and the reason is a GitHub constraint rather than a preference: + # `workflow_dispatch` is only dispatchable once the workflow exists on the + # DEFAULT branch, and this diagnostic needs to run before anything merges. + # Drop this trigger (or the whole file) once #548's arm question is answered. + push: + branches: [feat/548-multiarch-image] + +jobs: + repro: + name: orlyc repro (${{ matrix.arch }}) + strategy: + # Both arches, so the comparison is like-for-like rather than against a + # remembered number from a different workflow. + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + # Long enough to actually finish if it can. If it cannot, the timestamped + # progress still shows the SHAPE of the curve, which is the whole point. + timeout-minutes: 60 + steps: + - uses: actions/checkout@v6 + + - name: Capacity + run: | + uname -m + echo "nproc: $(nproc)" + free -g + + - name: Install system dependencies + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq \ + build-essential gcc g++ uuid-dev libgmp-dev libaio-dev \ + libsnappy-dev libreadline-dev libboost-system-dev zlib1g-dev \ + bison flex valgrind moreutils + + - name: Bootstrap + run: make bootstrap version + + - name: Build orlyc only + run: | + PATH="$PWD/tools:$PATH" \ + jhm -c release --worker-count 4 orly/orlyc + ls -la ../out_orly/release/orly/orlyc + + # The actual repro. Backgrounded so that if it hangs we can interrogate the + # live process rather than just observing that it never returned. + - name: Run orlyc on one package, with a deadline + id: repro + run: | + set +e + mkdir -p /tmp/pkgout && cd /tmp/pkgout + ORLYC=$GITHUB_WORKSPACE/../out_orly/release/orly/orlyc + "$ORLYC" -o /tmp/pkgout "$GITHUB_WORKSPACE/orly/clients/mcp/smoke/sample.orly" \ + > /tmp/orlyc.out 2>&1 & + PID=$! + echo "orlyc pid=$PID" + for i in $(seq 1 60); do + kill -0 "$PID" 2>/dev/null || break + sleep 5 + done + if kill -0 "$PID" 2>/dev/null; then + echo "HUNG=1" >> "$GITHUB_OUTPUT" + echo "=== still alive after 300s -- interrogating pid $PID" + echo "--- /proc/$PID/status:"; grep -E "State|Threads" /proc/$PID/status + echo "--- wchan (kernel function it is blocked in):" + cat /proc/$PID/wchan 2>/dev/null; echo + echo "--- per-thread state and wchan:" + for t in /proc/$PID/task/*; do + printf " tid %s state=%s wchan=%s\n" "$(basename $t)" \ + "$(awk '/^State:/{print $2}' $t/status 2>/dev/null)" \ + "$(cat $t/wchan 2>/dev/null)" + done + echo "--- all-thread backtraces:" + sudo gdb -p "$PID" -batch \ + -ex "set pagination off" -ex "thread apply all bt" 2>&1 | tail -80 + kill -9 "$PID" 2>/dev/null + else + wait "$PID"; rc=$? + echo "orlyc exited rc=$rc" + echo "HUNG=0" >> "$GITHUB_OUTPUT" + ls -la /tmp/pkgout + fi + echo "=== orlyc output:"; cat /tmp/orlyc.out || true + + - name: Verdict + if: always() + run: | + if [ "${{ steps.repro.outputs.HUNG }}" = "1" ]; then + echo "orlyc HUNG on ${{ matrix.arch }} -- see backtraces above (#554)" + else + echo "orlyc completed on ${{ matrix.arch }}" + fi + + - name: Upload output + if: always() + uses: actions/upload-artifact@v4 + with: + name: orlyc-repro-${{ matrix.arch }} + path: /tmp/orlyc.out + if-no-files-found: warn From 028bfe4f9bb77e7848e08965742cfd88a2e59bca Mon Sep 17 00:00:00 2001 From: Patrick O'Reilly Date: Tue, 15 Sep 2026 15:16:09 -0700 Subject: [PATCH 7/8] ci: fix the repro's package path, and assert its inputs exist (#554) The first run proved nothing. $GITHUB_WORKSPACE already ends in /orly (it is the repo root, the Dockerfile's WORKDIR /src/orly), and I appended another, so orlyc got a doubled path, exited 1 in two seconds on BOTH arches, and the job reported green. That is worse than a failed repro: a diagnostic that exits fast and clean while testing nothing is indistinguishable from good news. So the inputs are now asserted to exist before anything is timed -- if the path is wrong again the job says so instead of quietly passing. --- .github/workflows/orlyc-arm-repro.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/orlyc-arm-repro.yml b/.github/workflows/orlyc-arm-repro.yml index b000416d..18f7e6aa 100644 --- a/.github/workflows/orlyc-arm-repro.yml +++ b/.github/workflows/orlyc-arm-repro.yml @@ -80,7 +80,13 @@ jobs: set +e mkdir -p /tmp/pkgout && cd /tmp/pkgout ORLYC=$GITHUB_WORKSPACE/../out_orly/release/orly/orlyc - "$ORLYC" -o /tmp/pkgout "$GITHUB_WORKSPACE/orly/clients/mcp/smoke/sample.orly" \ + PKG=$GITHUB_WORKSPACE/clients/mcp/smoke/sample.orly + # Assert both inputs exist before timing anything. The first run of this + # job "passed" in 2 seconds because the package path had a doubled + # segment -- orlyc exited 1 on both arches and the repro proved nothing. + test -x "$ORLYC" || { echo "no orlyc at $ORLYC"; exit 1; } + test -f "$PKG" || { echo "no package at $PKG"; exit 1; } + "$ORLYC" -o /tmp/pkgout "$PKG" \ > /tmp/orlyc.out 2>&1 & PID=$! echo "orlyc pid=$PID" From fbecb66f1ba15fbcc8e66572694d60d8bca42738 Mon Sep 17 00:00:00 2001 From: Patrick O'Reilly Date: Tue, 15 Sep 2026 15:37:26 -0700 Subject: [PATCH 8/8] ci: capture full backtraces and a second snapshot for the orlyc hang (#554) The last run piped gdb through `tail -80`, which cut exactly the frame that matters -- the worker running the main job. Its stack is what distinguishes "RunPackageTests never returned" from "server->Shutdown() wedged", and those have different fixes. Now: full stacks to a file and uploaded as an artifact, the teardown-relevant frames grepped inline, and a SECOND snapshot 60s later. Identical stacks across the two means genuinely wedged; moving frames would mean livelock or very slow progress, which would be a different bug and worth knowing before assuming deadlock. --- .github/workflows/orlyc-arm-repro.yml | 29 ++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/.github/workflows/orlyc-arm-repro.yml b/.github/workflows/orlyc-arm-repro.yml index 18f7e6aa..51ad7d9c 100644 --- a/.github/workflows/orlyc-arm-repro.yml +++ b/.github/workflows/orlyc-arm-repro.yml @@ -106,9 +106,29 @@ jobs: "$(awk '/^State:/{print $2}' $t/status 2>/dev/null)" \ "$(cat $t/wchan 2>/dev/null)" done - echo "--- all-thread backtraces:" + # FULL backtraces to a file. The previous run piped this through + # `tail -80`, which cut exactly the frame that matters: the worker + # running the main job, whose stack says whether RunPackageTests + # never returned or server->Shutdown() wedged. + echo "--- all-thread backtraces -> /tmp/bt1.txt" sudo gdb -p "$PID" -batch \ - -ex "set pagination off" -ex "thread apply all bt" 2>&1 | tail -80 + -ex "set pagination off" -ex "thread apply all bt" > /tmp/bt1.txt 2>&1 + echo " captured $(grep -c '^Thread ' /tmp/bt1.txt) threads" + # The frames worth reading inline: anything in orlyc/server teardown. + grep -nE "RunPackageTests|TServer::Shutdown|RunTestsOnIndy|Fiber::TRunner|TScheduler::|::Wait|Join" /tmp/bt1.txt | head -25 + + # Second snapshot 60s later. If the two are identical the process is + # genuinely wedged; if frames move it is livelock or very slow + # progress, which is a different bug. + sleep 60 + sudo gdb -p "$PID" -batch \ + -ex "set pagination off" -ex "thread apply all bt" > /tmp/bt2.txt 2>&1 + if diff -q <(grep -oE '^#[0-9]+ +[A-Za-z_:~]+' /tmp/bt1.txt) \ + <(grep -oE '^#[0-9]+ +[A-Za-z_:~]+' /tmp/bt2.txt) >/dev/null; then + echo " VERDICT: identical stacks 60s apart -- wedged, not slow" + else + echo " VERDICT: stacks MOVED between snapshots -- livelock or slow progress" + fi kill -9 "$PID" 2>/dev/null else wait "$PID"; rc=$? @@ -132,5 +152,8 @@ jobs: uses: actions/upload-artifact@v4 with: name: orlyc-repro-${{ matrix.arch }} - path: /tmp/orlyc.out + path: | + /tmp/orlyc.out + /tmp/bt1.txt + /tmp/bt2.txt if-no-files-found: warn