From 494ce5f78b46f1aa1aab6d2ab9b0a8e2414f4334 Mon Sep 17 00:00:00 2001 From: David Maseda Neira Date: Fri, 31 Jul 2026 19:55:55 +0200 Subject: [PATCH] tests(e2e): Add an e2e test for lambdas --- .github/workflows/ci.yml | 34 ++++++++++++ test/e2e/transform.sh | 114 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100755 test/e2e/transform.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee0c50e..0ed0548 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -182,6 +182,40 @@ jobs: - name: Run conditional-requests e2e run: bash test/e2e/conditional.sh + e2e-transform: + name: E2E (object transforms) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Elixir/OTP + uses: erlef/setup-beam@v1 + with: + otp-version: "29" + elixir-version: "1.20" + + - name: Cache deps and build + uses: actions/cache@v4 + with: + path: | + deps + _build + key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }} + restore-keys: | + ${{ runner.os }}-mix- + + - name: Install dependencies + run: mix deps.get + + - name: Verify S3 client (aws-cli) + run: aws --version + + # Streams a single-blob and a multipart (manifest) object through the built-in + # gzip transform over a real socket, asserting the wire headers and a gunzip + # round-trip — the live path Plug.Test can't cover — plus 400 on an unknown name. + - name: Run object-transform e2e + run: bash test/e2e/transform.sh + e2e-docker: name: E2E (docker cluster) runs-on: ubuntu-latest diff --git a/test/e2e/transform.sh b/test/e2e/transform.sh new file mode 100755 index 0000000..1161ba5 --- /dev/null +++ b/test/e2e/transform.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# End-to-end test: object-read transforms ("Lambda", ?transform=) over the wire. +# One AetherS3 node with auth OFF. Drives the built-in gzip transform with curl so +# the assertions are the real streamed bytes and wire headers — content-encoding, +# x-amz-transform, and a gunzip round-trip — which Plug.Test's in-memory conn and +# the router's inline-blob test transform can't exercise. +# +# Covers: a small (single-blob) object gzipped and gunzipped back to the original; +# a multipart object (a manifest, streamed part-by-part through the same transform) +# round-tripping too; the transform-marker + content-encoding headers; and an +# unregistered transform name answering 400. +# +# Requires: elixir/mix, aws-cli, curl, gzip on PATH. +set -euo pipefail + +cd "$(dirname "$0")/../.." + +PORT=9000 +ADMIN_PORT=9001 +WORKDIR="$(mktemp -d)" +PID="" +BUCKET="xform" + +export AWS_ACCESS_KEY_ID=x AWS_SECRET_ACCESS_KEY=x +export AWS_DEFAULT_REGION=us-east-1 AWS_EC2_METADATA_DISABLED=true + +log() { echo "[xform] $*"; } +fail() { echo "[xform] FAIL: $*" >&2; exit 1; } + +cleanup() { + local code=$? + [ "$code" -ne 0 ] && tail -30 "$WORKDIR/node.log" 2>/dev/null >&2 + [ -n "$PID" ] && kill "$PID" 2>/dev/null || true + wait 2>/dev/null || true + rm -rf "$WORKDIR" +} +trap cleanup EXIT + +aws_() { aws --endpoint-url "http://127.0.0.1:$PORT" "$@"; } +url() { echo "http://127.0.0.1:$PORT/$BUCKET/$1"; } + +expect() { # expect + [ "$2" = "$3" ] || fail "$1: expected $3, got $2" +} + +# GET ?transform=gzip, capturing headers to $2 and the (gzip) body to $3. +get_gzip() { curl -s -D "$2" -o "$3" "$(url "$1")?transform=gzip"; } + +# A response header's value (unquoted, lowercased name), read from a saved header dump. +hdr() { tr -d '\r' <"$1" | awk -v k="$(echo "$2" | tr 'A-Z' 'a-z'):" 'tolower($1) == k { print $2 }'; } + +log "compiling..." +mix deps.get >/dev/null +mix compile >/dev/null + +log "starting a single node (auth off, RF=1)..." +AETHER_PORT=$PORT AETHER_ADMIN_PORT=$ADMIN_PORT \ + AETHER_DATA_DIR="$WORKDIR/data" AETHER_REQUIRE_AUTH=false AETHER_REPLICATION_FACTOR=1 \ + elixir -S mix run --no-halt >"$WORKDIR/node.log" 2>&1 & +PID=$! + +log "waiting for readiness..." +for _ in $(seq 1 60); do + curl -fsS "http://127.0.0.1:$ADMIN_PORT/ready/cp" >/dev/null 2>&1 && break + sleep 1 +done +curl -fsS "http://127.0.0.1:$ADMIN_PORT/ready/cp" >/dev/null 2>&1 || fail "node not ready" + +log "creating bucket..." +for _ in $(seq 1 30); do aws_ s3 mb "s3://$BUCKET" >/dev/null 2>&1 && break; sleep 1; done + +# --- small (single-blob) object ---------------------------------------------- + +# Compressible text so we can also assert the transform actually shrank it. +# (awk, not `yes | head`, which would SIGPIPE under `set -o pipefail`.) +awk 'BEGIN { for (i = 0; i < 2000; i++) print "the quick brown fox jumps over the lazy dog" }' \ + >"$WORKDIR/small.txt" +aws_ s3 cp "$WORKDIR/small.txt" "s3://$BUCKET/small.txt" >/dev/null || fail "small PUT failed" + +log "GET ?transform=gzip on a single-blob object..." +get_gzip small.txt "$WORKDIR/h.small" "$WORKDIR/small.gz" +expect "content-encoding" "$(hdr "$WORKDIR/h.small" content-encoding)" "gzip" +expect "x-amz-transform" "$(hdr "$WORKDIR/h.small" x-amz-transform)" "gzip" + +gunzip -c "$WORKDIR/small.gz" >"$WORKDIR/small.out" || fail "gunzip failed (not a valid gzip stream)" +cmp -s "$WORKDIR/small.txt" "$WORKDIR/small.out" || fail "gunzip'd bytes differ from the original" + +orig=$(wc -c <"$WORKDIR/small.txt"); comp=$(wc -c <"$WORKDIR/small.gz") +[ "$comp" -lt "$orig" ] || fail "transform did not compress ($comp >= $orig)" +log "single-blob gzip round-trips ($orig -> $comp bytes) ✓" + +# --- unknown transform -> 400 ------------------------------------------------ + +log "an unregistered transform name -> 400..." +code=$(curl -s -o /dev/null -w '%{http_code}' "$(url small.txt)?transform=nope") +expect "unknown transform" "$code" 400 + +# --- multipart object (manifest streamed part-by-part) ----------------------- + +# Force aws-cli to split the upload into parts (min part size is 5 MiB), so the +# stored object is a manifest — the transform must stream its parts in order. +aws configure set default.s3.multipart_threshold 5MB +aws configure set default.s3.multipart_chunksize 5MB +head -c 12582912 /dev/urandom >"$WORKDIR/big.bin" # 12 MiB -> 3 parts +aws_ s3 cp "$WORKDIR/big.bin" "s3://$BUCKET/big.bin" >/dev/null || fail "multipart PUT failed" + +log "GET ?transform=gzip on a multipart (manifest) object..." +get_gzip big.bin "$WORKDIR/h.big" "$WORKDIR/big.gz" +expect "mp content-encoding" "$(hdr "$WORKDIR/h.big" content-encoding)" "gzip" +gunzip -c "$WORKDIR/big.gz" >"$WORKDIR/big.out" || fail "multipart gunzip failed" +cmp -s "$WORKDIR/big.bin" "$WORKDIR/big.out" || fail "multipart gunzip'd bytes differ from the original" +log "multipart manifest streamed through gzip round-trips ✓" + +log "PASS: object transforms — single-blob + multipart gzip round-trip and headers over the wire, unknown name -> 400"