From bd24f8a08375d2ffafbde7c37eba85a2d517122b Mon Sep 17 00:00:00 2001 From: nandithebull Date: Thu, 10 Sep 2026 22:24:06 -0700 Subject: [PATCH] feat: x86_64 builds via LAMBDA_ARCH, one-command jolt demo `jolt image` was pinned to linux/arm64, which fails with `exec format error` on an x86_64 Linux Docker host without qemu, and `jolt deploy` always created an arm64 function. - LAMBDA_ARCH=x86_64 (or amd64) builds under linux/amd64. arm64 stays the default. - deploy reads the architecture from dist/bootstrap's ELF header and passes --architectures on both create and update, so it always matches the last build and can switch an existing function. - `jolt demo` runs image, deploy and invoke in order, after checking tools, AWS credentials and region, Docker access, and that Docker can run containers for the target platform. Verified end to end on x86_64 Linux: built, deployed to provided.al2023 x86_64 in us-west-2, and invoked successfully. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CsPjhLv9dHzRwCbxB81Wco --- CHANGELOG.md | 2 + README.md | 12 +++--- bb.edn | 17 ++++++-- docs/guide/al2023-build.md | 26 ++++++++---- docs/guide/getting-started.md | 8 ++++ script/aws_lifecycle.clj | 29 ++++++++++--- script/demo.clj | 80 +++++++++++++++++++++++++++++++++++ 7 files changed, 153 insertions(+), 21 deletions(-) create mode 100644 script/demo.clj diff --git a/CHANGELOG.md b/CHANGELOG.md index 893f7f5..1757787 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ All notable changes to this project are documented here. ### Added - Initial public release preparation. +- x86_64 builds: `LAMBDA_ARCH=x86_64 jolt image` (default stays `arm64`); `jolt deploy` reads the architecture from `dist/bootstrap`'s ELF header and passes it on both create and update. +- `jolt demo`: `image` + `deploy` + `invoke` in one command, with every prerequisite (tools, AWS credentials and region, Docker access, emulation for `LAMBDA_ARCH`) checked before the build starts. ## [0.1.0] diff --git a/README.md b/README.md index adc851e..21d75d4 100644 --- a/README.md +++ b/README.md @@ -74,14 +74,18 @@ See `docs/guide/runtime-api-loop.md` for the loop's design notes and ```sh jolt probe # offline e2e: mock Runtime API + demo handler (no AWS, no Docker) jolt test # run script/bench.clj's unit tests (no AWS, no Docker) -jolt image # AL2023 Docker build -> dist/bootstrap + dist/lambda.zip (arm64) +jolt demo # image + deploy + invoke in one go, prerequisites checked first +jolt image # AL2023 Docker build -> dist/bootstrap + dist/lambda.zip (arm64 default) jolt deploy # idempotent: create/update the IAM role + Lambda function jolt invoke # single ad-hoc invoke, prints the response body + REPORT line jolt bench # cold/warm boot-time comparison across memory tiers jolt teardown # delete the function + role when you're done ``` -`jolt deploy`'s IAM role (`lambda-mvp-jlt-role`) and function +`jolt image` builds for arm64 unless `LAMBDA_ARCH=x86_64` (or `amd64`) is set. +Use that on an x86_64 Linux host without qemu, where an arm64 build fails with +`exec format error`. `jolt deploy` picks up the architecture from the built +binary. `jolt deploy`'s IAM role (`lambda-mvp-jlt-role`) and function (`lambda-mvp-jlt`) names are overridable via `LAMBDA_MVP_FUNCTION_NAME`. `jolt bench`'s memory tiers and warm-sample count are overridable via `BENCH_MEMORY_TIERS` (default `2048,3008`) and `BENCH_WARM_SAMPLES` @@ -103,7 +107,7 @@ Fetch the base image out-of-band over direct HTTPS and point the build at it: ```sh brew install crane -crane pull --platform linux/arm64 public.ecr.aws/amazonlinux/amazonlinux:2023 /tmp/al2023.tar +crane pull --platform linux/arm64 public.ecr.aws/amazonlinux/amazonlinux:2023 /tmp/al2023.tar # linux/amd64 with LAMBDA_ARCH=x86_64 docker load -i /tmp/al2023.tar docker tag public.ecr.aws/amazonlinux/amazonlinux:2023 my-local/amazonlinux:2023 BASE_IMAGE=my-local/amazonlinux:2023 jolt image @@ -129,8 +133,6 @@ Not built here, but straightforward follow-ups if you need them: - A function URL + bearer token, for an HTTP-reachable demo instead of `aws lambda invoke` only. -- x86_64 builds (same Dockerfile approach, different base image - architecture and jolt/Chez build targets). - A second, dependency-heavier handler, to isolate binary-size effects on cold init independent of the jolt-version comparison `jolt bench` already lets you reproduce. diff --git a/bb.edn b/bb.edn index 1d59430..41668b7 100644 --- a/bb.edn +++ b/bb.edn @@ -45,6 +45,7 @@ :task (do (println "lambda-mvp-jlt -- a shareable AWS Lambda custom runtime in jolt") (println "") (println " jolt probe offline e2e: mock Runtime API + jolt loop (no AWS, no Docker)") + (println " jolt demo image + deploy + invoke in one go, prerequisites checked first") (println " jolt image AL2023 Docker build -> dist/bootstrap + dist/lambda.zip") (println " jolt deploy idempotent: create/update the IAM role + Lambda function") (println " jolt invoke single ad-hoc invoke, prints response + REPORT line") @@ -78,16 +79,26 @@ "joltc run -m net.b12n.lambda-mvp.main") (System/exit (:exit (deref srv)))))))} - image {:doc "build bootstrap + lambda.zip in an amazonlinux:2023 container (linux/arm64)" + image {:doc "build bootstrap + lambda.zip in an amazonlinux:2023 container (LAMBDA_ARCH, default arm64)" :task (let [jolt-version (System/getenv "JOLT_VERSION") chez-version (System/getenv "CHEZ_VERSION") - base (System/getenv "BASE_IMAGE")] - (p/shell (str "docker build --target export --output dist --platform linux/arm64 " + base (System/getenv "BASE_IMAGE") + ;; Lambda spells it x86_64, Docker amd64 -- accept both. + arch (or (System/getenv "LAMBDA_ARCH") "arm64") + platform (case arch + "arm64" "linux/arm64" + ("x86_64" "amd64") "linux/amd64" + (do (println "image: LAMBDA_ARCH must be arm64 or x86_64, got" arch) + (System/exit 1)))] + (p/shell (str "docker build --target export --output dist --platform " platform " " (if jolt-version (str "--build-arg JOLT_VERSION=" jolt-version " ") "") (if chez-version (str "--build-arg CHEZ_VERSION=" chez-version " ") "") (if base (str "--build-arg BASE_IMAGE=" base " ") "") ".")))} + demo {:doc "image + deploy + invoke in one go, checking every prerequisite before starting" + :task (p/shell "bb" "script/demo.clj")} + clean {:doc "remove build artifacts" :task (p/shell "rm -rf dist bootstrap bootstrap.build")} diff --git a/docs/guide/al2023-build.md b/docs/guide/al2023-build.md index 71ea48c..46231d0 100644 --- a/docs/guide/al2023-build.md +++ b/docs/guide/al2023-build.md @@ -58,11 +58,21 @@ than a one-time historical claim. - These live in a **separate `RUN dnf install` layer placed after the Chez build** so fixing them doesn't invalidate the ~4-minute Chez layer. -## arm64 - -The build targets **linux/arm64** (native on Apple Silicon Docker, no qemu) and -deploys to Lambda `--architectures arm64` (Graviton, cheaper per GB-second). -Chez v10 supports aarch64le Linux; jolt's release binaries don't cover -aarch64 Linux, but the from-source build works. For x86_64, the identical -Dockerfile under `--platform linux/amd64` (qemu emulation on Apple Silicon: -slow but automatic) is the path, not built or tested here. +## arm64 and x86_64 + +By default the build targets **linux/arm64** (native on Apple Silicon Docker, +no qemu) and deploys to Lambda `--architectures arm64` (Graviton, cheaper per +GB-second). Chez v10 supports aarch64le Linux; jolt's release binaries don't +cover aarch64 Linux, but the from-source build works. + +`LAMBDA_ARCH=x86_64 jolt image` (`amd64` also accepted) builds the identical +Dockerfile under `--platform linux/amd64` instead. `jolt deploy` reads the +architecture from `dist/bootstrap`'s ELF header, so it always matches the last +build, including when it switches an existing function's architecture. + +Building for the architecture your machine isn't needs qemu. Docker Desktop +ships it; on a plain Linux Docker Engine, a missing emulator fails the first +`RUN` with `exec /bin/sh: exec format error`. Either build natively (on an +x86_64 Linux host, `LAMBDA_ARCH=x86_64`) or register the emulator, e.g. +`docker run --privileged --rm tonistiigi/binfmt --install arm64`. Expect an +emulated Chez build to be several times slower. diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 8bd16ed..b0415dc 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -50,6 +50,14 @@ jolt teardown `jolt deploy` is idempotent: it creates the IAM role and Lambda function the first time, and updates them on every later call. `jolt invoke` runs a single ad-hoc invocation and prints the response body plus the CloudWatch `REPORT` line, the same line [Cold vs. warm boot](cold-warm-boot.md) explains how to read. `jolt teardown` deletes both the function and the role, so nothing keeps running in your account. +### Or all at once + +```sh +jolt demo +``` + +`jolt demo` runs `image`, `deploy` and `invoke` in order. Before any of them it checks that `docker` and `aws` are on PATH, that the AWS CLI has credentials and a region (and prints the account it will deploy to), that your user can reach the Docker daemon, and that Docker can run containers for `LAMBDA_ARCH`. A missing piece stops it with the fix, before the build starts or anything changes in your account. Docker's layer cache makes repeat runs fast, so it's also the edit-and-redeploy loop for `handler.clj`. + ## Measure cold vs. warm boot time ```sh diff --git a/script/aws_lifecycle.clj b/script/aws_lifecycle.clj index 56a5853..59643b2 100644 --- a/script/aws_lifecycle.clj +++ b/script/aws_lifecycle.clj @@ -14,6 +14,7 @@ (def function-name (or (System/getenv "LAMBDA_MVP_FUNCTION_NAME") "lambda-mvp-jlt")) (def role-name (str function-name "-role")) (def zip-path "dist/lambda.zip") +(def bootstrap-path "dist/bootstrap") (def policy-arn "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole") (defn- sh [& args] @@ -25,6 +26,20 @@ (apply println "lambda-mvp-jlt:" msg)) (System/exit 1)) +(defn- bootstrap-arch + "Lambda --architectures value for the built binary, read from its ELF + e_machine field rather than an env var, so a deploy can never disagree + with whatever `jolt image` last built." + [] + (let [header (byte-array 20)] + (with-open [in (java.io.FileInputStream. bootstrap-path)] + (.read in header)) + (case (bit-or (bit-and (aget header 18) 0xff) + (bit-shift-left (bit-and (aget header 19) 0xff) 8)) + 62 "x86_64" + 183 "arm64" + (die! bootstrap-path "is not an x86_64 or arm64 ELF binary -- rerun `jolt image`.")))) + (defn- require-aws-identity! "Fail fast with a clear message if the aws CLI has no usable credentials/region, rather than letting a later call fail obscurely." @@ -76,13 +91,17 @@ (zero? (:exit (sh "aws" "lambda" "get-function" "--function-name" function-name)))) (defn- ensure-function! [] - (when-not (.exists (java.io.File. zip-path)) - (die! zip-path "not found -- run `jolt image` first.")) + (doseq [path [zip-path bootstrap-path]] + (when-not (.exists (java.io.File. path)) + (die! path "not found -- run `jolt image` first."))) (if (function-exists?) (do - (println "lambda-mvp-jlt: updating function code for" function-name) + (println "lambda-mvp-jlt: updating function code for" function-name (str "(" (bootstrap-arch) ")")) + ;; --architectures on update too: without it, a function created arm64 + ;; keeps arm64 and an x86_64 zip fails at init with Runtime.InvalidEntrypoint. (let [{:keys [exit err]} (sh "aws" "lambda" "update-function-code" "--function-name" function-name + "--architectures" (bootstrap-arch) "--zip-file" (str "fileb://" zip-path))] (when-not (zero? exit) (die! "update-function-code failed:" err))) (let [{:keys [exit err]} (sh "aws" "lambda" "wait" "function-updated" "--function-name" function-name)] @@ -92,11 +111,11 @@ "--timeout" "15" "--memory-size" "2048")] (when-not (zero? exit) (die! "update-function-configuration failed:" err)))) (do - (println "lambda-mvp-jlt: creating function" function-name) + (println "lambda-mvp-jlt: creating function" function-name (str "(" (bootstrap-arch) ")")) (let [{:keys [exit err]} (sh "aws" "lambda" "create-function" "--function-name" function-name "--runtime" "provided.al2023" - "--architectures" "arm64" + "--architectures" (bootstrap-arch) "--handler" "bootstrap" "--zip-file" (str "fileb://" zip-path) "--role" (role-arn) diff --git a/script/demo.clj b/script/demo.clj new file mode 100644 index 0000000..9a0caa7 --- /dev/null +++ b/script/demo.clj @@ -0,0 +1,80 @@ +(ns script.demo + "`jolt demo`: build, deploy and invoke the demo handler in one go. Every + check that can fail runs first (tools on PATH, AWS credentials and region, + Docker access, emulation for the target architecture), so a missing piece + stops the run before a multi-minute build or any change to the AWS account. + The steps themselves are the ordinary `image`, `deploy` and `invoke` tasks." + (:require [babashka.fs :as fs] + [babashka.process :as p] + [cheshire.core :as json] + [clojure.string :as str])) + +(def arch (or (System/getenv "LAMBDA_ARCH") "arm64")) +(def base-image (or (System/getenv "BASE_IMAGE") "public.ecr.aws/amazonlinux/amazonlinux:2023")) + +(defn- sh [& args] + (let [{:keys [exit out err]} (apply p/shell {:out :string :err :string :continue true} args)] + {:exit exit :out out :err err})) + +(defn- die! [& msg] + (binding [*out* *err*] (apply println "lambda-mvp-jlt:" msg)) + (System/exit 1)) + +(defn- require-tools! [] + (let [missing (remove fs/which ["docker" "aws"])] + (when (seq missing) + (die! "not on PATH:" (str/join ", " missing) + "-- see Prerequisites in docs/guide/getting-started.md.")))) + +(defn- require-aws! [] + (let [region (some not-empty [(System/getenv "AWS_REGION") + (System/getenv "AWS_DEFAULT_REGION") + (str/trim (:out (sh "aws" "configure" "get" "region")))])] + (when-not region + (die! "no AWS region set. Run `aws configure set region ` or set AWS_REGION, then retry.")) + (let [{:keys [exit out err]} (sh "aws" "sts" "get-caller-identity" "--output" "json")] + (when-not (zero? exit) + (die! "aws CLI has no usable credentials." + "Run `aws configure` (or `aws configure sso`), or set AWS_PROFILE, then retry.\n" + (str/trim (or err "")))) + (let [{:strs [Account Arn]} (json/parse-string out)] + (println "lambda-mvp-jlt: will deploy to account" Account "in" region "as" Arn))))) + +(defn- require-docker! [] + (let [{:keys [exit err]} (sh "docker" "info")] + (when-not (zero? exit) + (if (str/includes? err "permission denied") + (die! "this user can't reach the Docker daemon (permission denied)." + "Add yourself to the docker group once (`sudo usermod -aG docker $USER`, then log out and back in) and retry.") + (die! "Docker isn't reachable:\n" (str/trim err)))))) + +(defn- require-platform! + "Runs a no-op in the base image under the target platform. That pulls the + image (the build needs it anyway) and hits the same `exec format error` a + build without an emulator would, in seconds instead of mid-build." + [] + (let [platform (case arch + "arm64" "linux/arm64" + ("x86_64" "amd64") "linux/amd64" + (die! "LAMBDA_ARCH must be arm64 or x86_64, got" arch)) + native (case (System/getProperty "os.arch") "amd64" "x86_64" "aarch64" "arm64" nil)] + (println "lambda-mvp-jlt: checking Docker can run" platform "containers (pulls the base image the first time)") + (let [{:keys [exit err]} (sh "docker" "run" "--rm" "--platform" platform base-image "true")] + (when-not (zero? exit) + (if (str/includes? err "exec format error") + (die! "Docker can't run" platform "containers here: no emulator registered." + (if native (str "Build for this machine instead (LAMBDA_ARCH=" native " jolt demo)") "Build for this machine's architecture") + "or register one: docker run --privileged --rm tonistiigi/binfmt --install" (subs platform 6)) + (die! "could not run" base-image "for" (str platform ":\n") (str/trim err))))))) + +(defn- step! [task] + (println (str "\nlambda-mvp-jlt: == " task " ==")) + (when-not (zero? (:exit (p/shell {:continue true} "bb" task))) + (die! task "failed; stopping."))) + +(require-tools!) +(require-aws!) +(require-docker!) +(require-platform!) +(run! step! ["image" "deploy" "invoke"]) +(println "\nlambda-mvp-jlt: done. `jolt invoke` calls it again; `jolt teardown` deletes the function and role.")