diff --git a/CHANGELOG.md b/CHANGELOG.md index 893f7f5..1757787 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ All notable changes to this project are documented here. ### Added - Initial public release preparation. +- x86_64 builds: `LAMBDA_ARCH=x86_64 jolt image` (default stays `arm64`); `jolt deploy` reads the architecture from `dist/bootstrap`'s ELF header and passes it on both create and update. +- `jolt demo`: `image` + `deploy` + `invoke` in one command, with every prerequisite (tools, AWS credentials and region, Docker access, emulation for `LAMBDA_ARCH`) checked before the build starts. ## [0.1.0] diff --git a/README.md b/README.md index adc851e..21d75d4 100644 --- a/README.md +++ b/README.md @@ -74,14 +74,18 @@ See `docs/guide/runtime-api-loop.md` for the loop's design notes and ```sh jolt probe # offline e2e: mock Runtime API + demo handler (no AWS, no Docker) jolt test # run script/bench.clj's unit tests (no AWS, no Docker) -jolt image # AL2023 Docker build -> dist/bootstrap + dist/lambda.zip (arm64) +jolt demo # image + deploy + invoke in one go, prerequisites checked first +jolt image # AL2023 Docker build -> dist/bootstrap + dist/lambda.zip (arm64 default) jolt deploy # idempotent: create/update the IAM role + Lambda function jolt invoke # single ad-hoc invoke, prints the response body + REPORT line jolt bench # cold/warm boot-time comparison across memory tiers jolt teardown # delete the function + role when you're done ``` -`jolt deploy`'s IAM role (`lambda-mvp-jlt-role`) and function +`jolt image` builds for arm64 unless `LAMBDA_ARCH=x86_64` (or `amd64`) is set. +Use that on an x86_64 Linux host without qemu, where an arm64 build fails with +`exec format error`. `jolt deploy` picks up the architecture from the built +binary. `jolt deploy`'s IAM role (`lambda-mvp-jlt-role`) and function (`lambda-mvp-jlt`) names are overridable via `LAMBDA_MVP_FUNCTION_NAME`. `jolt bench`'s memory tiers and warm-sample count are overridable via `BENCH_MEMORY_TIERS` (default `2048,3008`) and `BENCH_WARM_SAMPLES` @@ -103,7 +107,7 @@ Fetch the base image out-of-band over direct HTTPS and point the build at it: ```sh brew install crane -crane pull --platform linux/arm64 public.ecr.aws/amazonlinux/amazonlinux:2023 /tmp/al2023.tar +crane pull --platform linux/arm64 public.ecr.aws/amazonlinux/amazonlinux:2023 /tmp/al2023.tar # linux/amd64 with LAMBDA_ARCH=x86_64 docker load -i /tmp/al2023.tar docker tag public.ecr.aws/amazonlinux/amazonlinux:2023 my-local/amazonlinux:2023 BASE_IMAGE=my-local/amazonlinux:2023 jolt image @@ -129,8 +133,6 @@ Not built here, but straightforward follow-ups if you need them: - A function URL + bearer token, for an HTTP-reachable demo instead of `aws lambda invoke` only. -- x86_64 builds (same Dockerfile approach, different base image - architecture and jolt/Chez build targets). - A second, dependency-heavier handler, to isolate binary-size effects on cold init independent of the jolt-version comparison `jolt bench` already lets you reproduce. diff --git a/bb.edn b/bb.edn index 1d59430..41668b7 100644 --- a/bb.edn +++ b/bb.edn @@ -45,6 +45,7 @@ :task (do (println "lambda-mvp-jlt -- a shareable AWS Lambda custom runtime in jolt") (println "") (println " jolt probe offline e2e: mock Runtime API + jolt loop (no AWS, no Docker)") + (println " jolt demo image + deploy + invoke in one go, prerequisites checked first") (println " jolt image AL2023 Docker build -> dist/bootstrap + dist/lambda.zip") (println " jolt deploy idempotent: create/update the IAM role + Lambda function") (println " jolt invoke single ad-hoc invoke, prints response + REPORT line") @@ -78,16 +79,26 @@ "joltc run -m net.b12n.lambda-mvp.main") (System/exit (:exit (deref srv)))))))} - image {:doc "build bootstrap + lambda.zip in an amazonlinux:2023 container (linux/arm64)" + image {:doc "build bootstrap + lambda.zip in an amazonlinux:2023 container (LAMBDA_ARCH, default arm64)" :task (let [jolt-version (System/getenv "JOLT_VERSION") chez-version (System/getenv "CHEZ_VERSION") - base (System/getenv "BASE_IMAGE")] - (p/shell (str "docker build --target export --output dist --platform linux/arm64 " + base (System/getenv "BASE_IMAGE") + ;; Lambda spells it x86_64, Docker amd64 -- accept both. + arch (or (System/getenv "LAMBDA_ARCH") "arm64") + platform (case arch + "arm64" "linux/arm64" + ("x86_64" "amd64") "linux/amd64" + (do (println "image: LAMBDA_ARCH must be arm64 or x86_64, got" arch) + (System/exit 1)))] + (p/shell (str "docker build --target export --output dist --platform " platform " " (if jolt-version (str "--build-arg JOLT_VERSION=" jolt-version " ") "") (if chez-version (str "--build-arg CHEZ_VERSION=" chez-version " ") "") (if base (str "--build-arg BASE_IMAGE=" base " ") "") ".")))} + demo {:doc "image + deploy + invoke in one go, checking every prerequisite before starting" + :task (p/shell "bb" "script/demo.clj")} + clean {:doc "remove build artifacts" :task (p/shell "rm -rf dist bootstrap bootstrap.build")} diff --git a/docs/guide/al2023-build.md b/docs/guide/al2023-build.md index 71ea48c..46231d0 100644 --- a/docs/guide/al2023-build.md +++ b/docs/guide/al2023-build.md @@ -58,11 +58,21 @@ than a one-time historical claim. - These live in a **separate `RUN dnf install` layer placed after the Chez build** so fixing them doesn't invalidate the ~4-minute Chez layer. -## arm64 - -The build targets **linux/arm64** (native on Apple Silicon Docker, no qemu) and -deploys to Lambda `--architectures arm64` (Graviton, cheaper per GB-second). -Chez v10 supports aarch64le Linux; jolt's release binaries don't cover -aarch64 Linux, but the from-source build works. For x86_64, the identical -Dockerfile under `--platform linux/amd64` (qemu emulation on Apple Silicon: -slow but automatic) is the path, not built or tested here. +## arm64 and x86_64 + +By default the build targets **linux/arm64** (native on Apple Silicon Docker, +no qemu) and deploys to Lambda `--architectures arm64` (Graviton, cheaper per +GB-second). Chez v10 supports aarch64le Linux; jolt's release binaries don't +cover aarch64 Linux, but the from-source build works. + +`LAMBDA_ARCH=x86_64 jolt image` (`amd64` also accepted) builds the identical +Dockerfile under `--platform linux/amd64` instead. `jolt deploy` reads the +architecture from `dist/bootstrap`'s ELF header, so it always matches the last +build, including when it switches an existing function's architecture. + +Building for the architecture your machine isn't needs qemu. Docker Desktop +ships it; on a plain Linux Docker Engine, a missing emulator fails the first +`RUN` with `exec /bin/sh: exec format error`. Either build natively (on an +x86_64 Linux host, `LAMBDA_ARCH=x86_64`) or register the emulator, e.g. +`docker run --privileged --rm tonistiigi/binfmt --install arm64`. Expect an +emulated Chez build to be several times slower. diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 8bd16ed..b0415dc 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -50,6 +50,14 @@ jolt teardown `jolt deploy` is idempotent: it creates the IAM role and Lambda function the first time, and updates them on every later call. `jolt invoke` runs a single ad-hoc invocation and prints the response body plus the CloudWatch `REPORT` line, the same line [Cold vs. warm boot](cold-warm-boot.md) explains how to read. `jolt teardown` deletes both the function and the role, so nothing keeps running in your account. +### Or all at once + +```sh +jolt demo +``` + +`jolt demo` runs `image`, `deploy` and `invoke` in order. Before any of them it checks that `docker` and `aws` are on PATH, that the AWS CLI has credentials and a region (and prints the account it will deploy to), that your user can reach the Docker daemon, and that Docker can run containers for `LAMBDA_ARCH`. A missing piece stops it with the fix, before the build starts or anything changes in your account. Docker's layer cache makes repeat runs fast, so it's also the edit-and-redeploy loop for `handler.clj`. + ## Measure cold vs. warm boot time ```sh diff --git a/script/aws_lifecycle.clj b/script/aws_lifecycle.clj index 56a5853..59643b2 100644 --- a/script/aws_lifecycle.clj +++ b/script/aws_lifecycle.clj @@ -14,6 +14,7 @@ (def function-name (or (System/getenv "LAMBDA_MVP_FUNCTION_NAME") "lambda-mvp-jlt")) (def role-name (str function-name "-role")) (def zip-path "dist/lambda.zip") +(def bootstrap-path "dist/bootstrap") (def policy-arn "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole") (defn- sh [& args] @@ -25,6 +26,20 @@ (apply println "lambda-mvp-jlt:" msg)) (System/exit 1)) +(defn- bootstrap-arch + "Lambda --architectures value for the built binary, read from its ELF + e_machine field rather than an env var, so a deploy can never disagree + with whatever `jolt image` last built." + [] + (let [header (byte-array 20)] + (with-open [in (java.io.FileInputStream. bootstrap-path)] + (.read in header)) + (case (bit-or (bit-and (aget header 18) 0xff) + (bit-shift-left (bit-and (aget header 19) 0xff) 8)) + 62 "x86_64" + 183 "arm64" + (die! bootstrap-path "is not an x86_64 or arm64 ELF binary -- rerun `jolt image`.")))) + (defn- require-aws-identity! "Fail fast with a clear message if the aws CLI has no usable credentials/region, rather than letting a later call fail obscurely." @@ -76,13 +91,17 @@ (zero? (:exit (sh "aws" "lambda" "get-function" "--function-name" function-name)))) (defn- ensure-function! [] - (when-not (.exists (java.io.File. zip-path)) - (die! zip-path "not found -- run `jolt image` first.")) + (doseq [path [zip-path bootstrap-path]] + (when-not (.exists (java.io.File. path)) + (die! path "not found -- run `jolt image` first."))) (if (function-exists?) (do - (println "lambda-mvp-jlt: updating function code for" function-name) + (println "lambda-mvp-jlt: updating function code for" function-name (str "(" (bootstrap-arch) ")")) + ;; --architectures on update too: without it, a function created arm64 + ;; keeps arm64 and an x86_64 zip fails at init with Runtime.InvalidEntrypoint. (let [{:keys [exit err]} (sh "aws" "lambda" "update-function-code" "--function-name" function-name + "--architectures" (bootstrap-arch) "--zip-file" (str "fileb://" zip-path))] (when-not (zero? exit) (die! "update-function-code failed:" err))) (let [{:keys [exit err]} (sh "aws" "lambda" "wait" "function-updated" "--function-name" function-name)] @@ -92,11 +111,11 @@ "--timeout" "15" "--memory-size" "2048")] (when-not (zero? exit) (die! "update-function-configuration failed:" err)))) (do - (println "lambda-mvp-jlt: creating function" function-name) + (println "lambda-mvp-jlt: creating function" function-name (str "(" (bootstrap-arch) ")")) (let [{:keys [exit err]} (sh "aws" "lambda" "create-function" "--function-name" function-name "--runtime" "provided.al2023" - "--architectures" "arm64" + "--architectures" (bootstrap-arch) "--handler" "bootstrap" "--zip-file" (str "fileb://" zip-path) "--role" (role-arn) diff --git a/script/demo.clj b/script/demo.clj new file mode 100644 index 0000000..9a0caa7 --- /dev/null +++ b/script/demo.clj @@ -0,0 +1,80 @@ +(ns script.demo + "`jolt demo`: build, deploy and invoke the demo handler in one go. Every + check that can fail runs first (tools on PATH, AWS credentials and region, + Docker access, emulation for the target architecture), so a missing piece + stops the run before a multi-minute build or any change to the AWS account. + The steps themselves are the ordinary `image`, `deploy` and `invoke` tasks." + (:require [babashka.fs :as fs] + [babashka.process :as p] + [cheshire.core :as json] + [clojure.string :as str])) + +(def arch (or (System/getenv "LAMBDA_ARCH") "arm64")) +(def base-image (or (System/getenv "BASE_IMAGE") "public.ecr.aws/amazonlinux/amazonlinux:2023")) + +(defn- sh [& args] + (let [{:keys [exit out err]} (apply p/shell {:out :string :err :string :continue true} args)] + {:exit exit :out out :err err})) + +(defn- die! [& msg] + (binding [*out* *err*] (apply println "lambda-mvp-jlt:" msg)) + (System/exit 1)) + +(defn- require-tools! [] + (let [missing (remove fs/which ["docker" "aws"])] + (when (seq missing) + (die! "not on PATH:" (str/join ", " missing) + "-- see Prerequisites in docs/guide/getting-started.md.")))) + +(defn- require-aws! [] + (let [region (some not-empty [(System/getenv "AWS_REGION") + (System/getenv "AWS_DEFAULT_REGION") + (str/trim (:out (sh "aws" "configure" "get" "region")))])] + (when-not region + (die! "no AWS region set. Run `aws configure set region ` or set AWS_REGION, then retry.")) + (let [{:keys [exit out err]} (sh "aws" "sts" "get-caller-identity" "--output" "json")] + (when-not (zero? exit) + (die! "aws CLI has no usable credentials." + "Run `aws configure` (or `aws configure sso`), or set AWS_PROFILE, then retry.\n" + (str/trim (or err "")))) + (let [{:strs [Account Arn]} (json/parse-string out)] + (println "lambda-mvp-jlt: will deploy to account" Account "in" region "as" Arn))))) + +(defn- require-docker! [] + (let [{:keys [exit err]} (sh "docker" "info")] + (when-not (zero? exit) + (if (str/includes? err "permission denied") + (die! "this user can't reach the Docker daemon (permission denied)." + "Add yourself to the docker group once (`sudo usermod -aG docker $USER`, then log out and back in) and retry.") + (die! "Docker isn't reachable:\n" (str/trim err)))))) + +(defn- require-platform! + "Runs a no-op in the base image under the target platform. That pulls the + image (the build needs it anyway) and hits the same `exec format error` a + build without an emulator would, in seconds instead of mid-build." + [] + (let [platform (case arch + "arm64" "linux/arm64" + ("x86_64" "amd64") "linux/amd64" + (die! "LAMBDA_ARCH must be arm64 or x86_64, got" arch)) + native (case (System/getProperty "os.arch") "amd64" "x86_64" "aarch64" "arm64" nil)] + (println "lambda-mvp-jlt: checking Docker can run" platform "containers (pulls the base image the first time)") + (let [{:keys [exit err]} (sh "docker" "run" "--rm" "--platform" platform base-image "true")] + (when-not (zero? exit) + (if (str/includes? err "exec format error") + (die! "Docker can't run" platform "containers here: no emulator registered." + (if native (str "Build for this machine instead (LAMBDA_ARCH=" native " jolt demo)") "Build for this machine's architecture") + "or register one: docker run --privileged --rm tonistiigi/binfmt --install" (subs platform 6)) + (die! "could not run" base-image "for" (str platform ":\n") (str/trim err))))))) + +(defn- step! [task] + (println (str "\nlambda-mvp-jlt: == " task " ==")) + (when-not (zero? (:exit (p/shell {:continue true} "bb" task))) + (die! task "failed; stopping."))) + +(require-tools!) +(require-aws!) +(require-docker!) +(require-platform!) +(run! step! ["image" "deploy" "invoke"]) +(println "\nlambda-mvp-jlt: done. `jolt invoke` calls it again; `jolt teardown` deletes the function and role.")