S3 credential check

.github/workflows/ecr-lambda-microvm-runner.yml

@@ -0,0 +1,170 @@
+name: Build and Push Lambda MicroVM Runner to ECR
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+env:
+  AWS_REGION: af-south-1
+  ECR_REGISTRY: 323463077991.dkr.ecr.af-south-1.amazonaws.com
+  # Lambda MicroVM runner (Path B) pushes to its own dedicated repo — never the
+  # synapseai-codeapi API repo, whose immutable tags would collide with the
+  # control-plane matrix sharing the same short SHA.
+  ECR_REPOSITORY: synapseai-codeapi-microvm-runner
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ecr-lambda-microvm-runner-${{ github.ref_name }}
+  cancel-in-progress: false
+
+jobs:
+  build-and-push:
+    name: Build and Push Runner Image
+    # arm64 runner: the lambda-microvm-runner target must be a native arm64
+    # image (Lambda MicroVMs builds on the AL2023 arm64 base). Matches the
+    # runner used by the ci.yml validation job.
+    runs-on: ubuntu-24.04-arm
+    timeout-minutes: 90
+    outputs:
+      image_tag: ${{ steps.sha.outputs.short }}
+      image_uri: ${{ steps.uri.outputs.value }}
+
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.ENVIRO_ECR_AWS_ACCESS_KEY }}
+          aws-secret-access-key: ${{ secrets.ENVIRO_ECR_AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Set short SHA
+        id: sha
+        run: echo "short=$(echo ${{ github.sha }} | cut -c1-7)" >> "$GITHUB_OUTPUT"
+
+      - name: Create native BuildKit builder
+        run: |
+          test "$(uname -m)" = "aarch64"
+          docker buildx create \
+            --name lambda-microvm-ecr \
+            --driver docker-container \
+            --use
+          docker buildx inspect --bootstrap
+
+      - name: Build and push via repo script
+        # Delegates to scripts/build-lambda-microvm-artifact.sh so CI behaviour
+        # (buildx flags, provenance/SBOM off, digest capture) stays identical
+        # to a local run. The script's push stage re-tags and pushes; ECR login
+        # inside the script is redundant after amazon-ecr-login but harmless.
+        env:
+          ECR_URI: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}
+          IMAGE_TAG: ${{ steps.sha.outputs.short }}
+          AWS_REGION: ${{ env.AWS_REGION }}
+        run: scripts/build-lambda-microvm-artifact.sh build push
+
+      - name: Record image URI
+        id: uri
+        env:
+          ECR_URI: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}
+          IMAGE_TAG: ${{ steps.sha.outputs.short }}
+        run: |
+          digest="$(sed -n '1p' .build-lambda-microvm/image-digest)"
+          echo "value=${ECR_URI}@${digest}" >> "$GITHUB_OUTPUT"
+          echo "Pushed tag:    ${ECR_URI}:${IMAGE_TAG}"
+          echo "Immutable ref: ${ECR_URI}@${digest}"
+          echo
+          echo "  scripts/build-lambda-microvm-artifact.sh zip upload"
+          echo "  then service/scripts/create-microvm-image.ts --artifact s3://..."
+
+  # ---------------------------------------------------------------------------
+  # Control-plane images (Path A self-hosted stack) -> Momentum ECR.
+  #
+  # One matrix job builds and pushes every image the depman spec references,
+  # all tagged with the same short SHA so a deploy pins a coherent set:
+  #   api, worker, file-server (service/Dockerfile targets)
+  #   egress-gateway, tool-call-server (standalone Dockerfiles)
+  #   sandbox-runner (api/Dockerfile target sandbox-runner: NsJail/directory-root
+  #     mode — Momentum nodes have no /dev/kvm, so packages come from a PVC, not
+  #     a baked image; the baked target is only for the KVM path we can't use)
+  #   package-init (docker/Dockerfile.package-init: populates the packages PVC
+  #     with the Python/Node/Bun runtime tree on deploy)
+  #
+  # Built for linux/amd64 (Momentum EKS x86 node groups). If the cluster moves
+  # to Graviton/arm64, change CONTROL_PLANE_PLATFORM and the runner to match.
+  #   egress-gateway, tool-call-server (standalone Dockerfiles)
+  #   sandbox-runner (api/Dockerfile target sandbox-runner-baked: KVM mode with
+  #     /pkgs baked in — this is the image that carries our fork's packages)
+  #
+  # Built for linux/amd64 (Momentum EKS x86 node groups). If the cluster moves
+  # to Graviton/arm64, change CONTROL_PLANE_PLATFORM and the runner to match.
+  build-control-plane:
+    name: Build and Push Control-Plane Images
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    env:
+      CONTROL_PLANE_PLATFORM: linux/amd64
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { repo: synapseai-codeapi,          dockerfile: service/Dockerfile,                 target: api }
+          - { repo: synapseai-codeapi-worker,   dockerfile: service/Dockerfile,                 target: worker }
+          - { repo: synapseai-codeapi-files,    dockerfile: service/Dockerfile,                 target: production }
+          - { repo: synapseai-codeapi-toolcall, dockerfile: service/Dockerfile.tool-call-server, target: production }
+          - { repo: synapseai-codeapi-egress,   dockerfile: service/Dockerfile.egress-gateway,   target: production }
+          # NsJail mode (no /dev/kvm on Momentum nodes): directory-root runner.
+          # Packages are delivered via the packages PVC + package-init Job below.
+          - { repo: synapseai-codeapi-sandbox-runner, dockerfile: api/Dockerfile,              target: sandbox-runner }
+          # Populates the sandbox packages PVC (Python/Node/Bun) on deploy. Repo
+          # must exist in ECR before this row can push.
+          - { repo: synapseai-codeapi-package-init, dockerfile: docker/Dockerfile.package-init }
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.ENVIRO_ECR_AWS_ACCESS_KEY }}
+          aws-secret-access-key: ${{ secrets.ENVIRO_ECR_AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Set short SHA
+        id: sha
+        run: echo "short=$(echo ${{ github.sha }} | cut -c1-7)" >> "$GITHUB_OUTPUT"
+
+      - name: Set up BuildKit
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push ${{ matrix.repo }}
+        env:
+          IMAGE_URI: ${{ env.ECR_REGISTRY }}/${{ matrix.repo }}:${{ steps.sha.outputs.short }}
+        run: |
+          # --target only when the matrix row names one (package-init is a
+          # single-stage Dockerfile and has no target stage).
+          TARGET_FLAG=()
+          if [ -n "${{ matrix.target }}" ]; then
+            TARGET_FLAG=(--target "${{ matrix.target }}")
+          fi
+          docker buildx build \
+            --platform "$CONTROL_PLANE_PLATFORM" \
+            "${TARGET_FLAG[@]}" \
+            --provenance=false \
+            --sbom=false \
+            -f "${{ matrix.dockerfile }}" \
+            -t "$IMAGE_URI" \
+            --push \
+            .
+          echo "Pushed $IMAGE_URI"

api/Dockerfile

@@ -84,6 +84,13 @@ RUN apt-get update && \
     file \
     jq \
     poppler-utils \
+    pandoc \
+    libreoffice-writer \
+    libreoffice-calc \
+    libreoffice-impress \
+    tesseract-ocr \
+    tesseract-ocr-eng \
+    debianutils \
     fonts-dejavu \
     fonts-liberation \
     libnl-route-3-200 \
@@ -171,7 +178,11 @@ ENV PORT=8080 \
     SANDBOX_SESSION_WORKSPACE_ENABLED=true
 
 EXPOSE 8080/tcp
-ENTRYPOINT ["/sandbox_api/entrypoint.sh"]
+# Explicit interpreter: the Lambda MicroVMs image builder fails its boot phase
+# (CREATE_FAILED, 'An unknown error occurred.', no container stdout) when the
+# entrypoint is a shebang script exec'd directly, while an ELF entrypoint works.
+# Locally (runc) both forms behave identically.
+ENTRYPOINT ["/bin/bash", "/sandbox_api/entrypoint.sh"]
 
 # Local direct-NsJail runtime used by @librechat/code's Docker supervisor.
 # Runtime packages are mounted read-only at /pkgs, matching docker-compose.mac.

api/config/sandbox.cfg

@@ -92,6 +92,17 @@ mount {
     options: "size=20m"
 }
 
+# POSIX shared memory for programs that need shm_open (LibreOffice UNO
+# bootstrap terminates with a RuntimeException without it). noexec like /tmp.
+mount {
+    dst: "/dev/shm"
+    fstype: "tmpfs"
+    rw: true
+    is_bind: false
+    noexec: true
+    options: "size=64m"
+}
+
 # Mount fresh procfs (read-only) inside the sandbox's PID namespace.
 # Fresh procfs only exposes the sandbox's own PIDs (1-2), preventing
 # cross-sandbox process visibility. This requires /proc in the container

build-packages.sh

@@ -210,6 +210,11 @@ install_python_packages() {
         pytesseract \
         pdfminer \
         vsdx \
+        pypdf \
+        pdfplumber \
+        defusedxml \
+        lxml \
+        validators \
         rasterio \
         rioxarray \
         geopandas \

docker-compose.aws.yml

@@ -0,0 +1,97 @@
+# AWS dev overlay - Lambda MicroVM sandbox backend, no local sandbox-runner.
+#
+# The control plane (api, worker, redis, minio, file/tool servers, egress
+# gateway) runs on the EC2 instance; every execution launches an AWS Lambda
+# MicroVM (Firecracker) instead of the local NsJail sandbox-runner, so this
+# host needs no KVM, no privileged containers, and no seccomp/cap tuning.
+# See docs/lambda-microvm/README.md for the backend architecture.
+#
+# Usage (on the instance, .env rendered from Secrets Manager
+# dev/synapseai/codeapi-env at 0600 first):
+#   docker compose -f docker-compose.yaml -f docker-compose.aws.yml up -d --build
+#
+# Port posture: ONLY two host bindings, neither on a public interface.
+#   172.17.0.1:3112  api            (host.docker.internal target for LibreChat)
+#   127.0.0.1:3190   egress gateway (fronted publicly by the Caddy vhost
+#                                    codeapi-egress.synapse-ai-dev.duckdns.org
+#                                    for MicroVM runner file/tool egress)
+# redis, minio, and the file server publish nothing; peers reach them on the
+# compose network.
+
+services:
+    api:
+        ports: !override
+            - 172.17.0.1:3112:3112
+        environment:
+            - CODEAPI_SANDBOX_BACKEND=${CODEAPI_SANDBOX_BACKEND:?}
+            - CODEAPI_RUNTIME_SESSION_MODE=${CODEAPI_RUNTIME_SESSION_MODE:-stateless}
+            - LAMBDA_MICROVM_IMAGE_ARN=${LAMBDA_MICROVM_IMAGE_ARN:-}
+            - LAMBDA_MICROVM_IMAGE_VERSION=${LAMBDA_MICROVM_IMAGE_VERSION:-}
+            - LAMBDA_MICROVM_REGION=${LAMBDA_MICROVM_REGION:-us-east-1}
+            - REDIS_PASSWORD=${REDIS_PASSWORD:?}
+            # The API prepares execution-manifest claims (service/router.ts ->
+            # prepareSandboxJobSecurity); in non-hardened mode it skips them
+            # unless the manifest key env is present. Base compose only sets it
+            # on the worker, so the runner would reject execs (missing header).
+            - CODEAPI_EXECUTION_MANIFEST_PRIVATE_KEY=${CODEAPI_EXECUTION_MANIFEST_PRIVATE_KEY:?}
+            # Non-hardened grant mode: setting the grant secret here turns on
+            # egress-grant minting (sandbox-egress.ts) so the MicroVM runner can
+            # upload generated files through the public gateway vhost. Without
+            # it the runner has EGRESS_GATEWAY_URL but no grant and silently
+            # drops generated files. (Hardened mode forbids this var here; we
+            # run non-hardened with INTERNET_EGRESS.)
+            - CODEAPI_EGRESS_GRANT_SECRET=${CODEAPI_EGRESS_GRANT_SECRET:?}
+
+    service-worker:
+        # The Lambda MicroVM backend replaces the local sandbox; the worker
+        # must not wait on (or ever reach) a sandbox-runner container.
+        depends_on: !override
+            redis:
+                condition: service_started
+        environment:
+            - CODEAPI_SANDBOX_BACKEND=${CODEAPI_SANDBOX_BACKEND:?}
+            - CODEAPI_RUNTIME_SESSION_MODE=${CODEAPI_RUNTIME_SESSION_MODE:-stateless}
+            - CODEAPI_EGRESS_GRANT_SECRET=${CODEAPI_EGRESS_GRANT_SECRET:?}
+            - LAMBDA_MICROVM_IMAGE_ARN=${LAMBDA_MICROVM_IMAGE_ARN:?}
+            - LAMBDA_MICROVM_IMAGE_VERSION=${LAMBDA_MICROVM_IMAGE_VERSION:-}
+            - LAMBDA_MICROVM_EXECUTION_ROLE_ARN=${LAMBDA_MICROVM_EXECUTION_ROLE_ARN:?}
+            - LAMBDA_MICROVM_LOG_GROUP=${LAMBDA_MICROVM_LOG_GROUP:?}
+            - LAMBDA_MICROVM_REGION=${LAMBDA_MICROVM_REGION:-us-east-1}
+            - LAMBDA_MICROVM_INGRESS_CONNECTOR_ARNS=${LAMBDA_MICROVM_INGRESS_CONNECTOR_ARNS:?}
+            - AWS_REGION=${LAMBDA_MICROVM_REGION:-us-east-1}
+            # By-reference input objects are fetched here and pushed into the
+            # MicroVM's runner cache (the VM cannot reach the file server).
+            - FILE_SERVER_URL=http://file_server:3000
+            - REDIS_PASSWORD=${REDIS_PASSWORD:?}
+
+    # Inactive on AWS: compute runs in Lambda MicroVMs. Enable the
+    # local-sandbox profile only for NsJail-on-host experiments.
+    sandbox-runner:
+        profiles: ["local-sandbox"]
+
+    egress_gateway:
+        ports: !override
+            - 127.0.0.1:3190:3190
+        environment:
+            - REDIS_PASSWORD=${REDIS_PASSWORD:?}
+
+    tool_call_server:
+        environment:
+            - REDIS_PASSWORD=${REDIS_PASSWORD:?}
+
+    file_server:
+        ports: !override []
+        environment:
+            - MINIO_ACCESS_KEY=${MINIO_ROOT_USER:?}
+            - MINIO_SECRET_KEY=${MINIO_ROOT_PASSWORD:?}
+            - REDIS_PASSWORD=${REDIS_PASSWORD:?}
+
+    redis:
+        command: ["redis-server", "--requirepass", "${REDIS_PASSWORD:?}"]
+        ports: !override []
+
+    minio:
+        ports: !override []
+        environment:
+            - MINIO_ROOT_USER=${MINIO_ROOT_USER:?}
+            - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:?}

docker/Dockerfile.worker-sandbox

@@ -122,6 +122,13 @@ RUN apt-install \
     bc \
     jq \
     poppler-utils \
+    pandoc \
+    libreoffice-writer \
+    libreoffice-calc \
+    libreoffice-impress \
+    tesseract-ocr \
+    tesseract-ocr-eng \
+    debianutils \
     fonts-dejavu \
     fonts-liberation \
     libnl-route-3-200 \

docker/package-init.sh

@@ -93,6 +93,7 @@ packages_ready() {
     [ -d "/pkgs/python/${PYTHON_VERSION}/lib/python${PYTHON_SITE_VERSION}/site-packages/chdb" ] &&
     [ -d "/pkgs/python/${PYTHON_VERSION}/lib/python${PYTHON_SITE_VERSION}/site-packages/statsmodels" ] &&
     [ -d "/pkgs/python/${PYTHON_VERSION}/lib/python${PYTHON_SITE_VERSION}/site-packages/rasterio" ] &&
+    [ -d "/pkgs/python/${PYTHON_VERSION}/lib/python${PYTHON_SITE_VERSION}/site-packages/pdfplumber" ] &&
     [ -f "/pkgs/node/${NODE_VERSION}/.package-installed" ] &&
     js_packages_ready "/pkgs/node/${NODE_VERSION}" &&
     [ -f "/pkgs/bun/${BUN_VERSION}/.package-installed" ] &&
@@ -231,6 +232,11 @@ if [ -f "$PIP_PATH" ]; then
         pytesseract \
         pdfminer \
         vsdx \
+        pypdf \
+        pdfplumber \
+        defusedxml \
+        lxml \
+        validators \
         rasterio \
         rioxarray \
         geopandas \

docs/lambda-microvm/terraform/main.tf

@@ -264,6 +264,17 @@ data "aws_iam_policy_document" "build_perms" {
     }
   }
 
+  # The image build's boot/snapshot executor authenticates to ECR Public to
+  # pull the AWS-managed base layers (public.ecr.aws). Without this the build
+  # dies AFTER docker export with CREATE_FAILED and an empty stateReason -
+  # CloudTrail shows the executor denied ecr-public:GetAuthorizationToken.
+  statement {
+    sid       = "PublicEcrAuth"
+    effect    = "Allow"
+    actions   = ["ecr-public:GetAuthorizationToken", "sts:GetServiceBearerToken"]
+    resources = ["*"]
+  }
+
   dynamic "statement" {
     for_each = var.private_ecr ? [1] : []
     content {
@@ -362,11 +373,10 @@ data "aws_iam_policy_document" "worker_microvm_control" {
     effect    = "Allow"
     actions   = ["iam:PassRole"]
     resources = [aws_iam_role.execution.arn]
-    condition {
-      test     = "StringEquals"
-      variable = "iam:PassedToService"
-      values   = ["lambda.amazonaws.com"]
-    }
+    # No iam:PassedToService condition: RunMicrovm's PassRole evaluation does
+    # not match lambda.amazonaws.com (nor lambda-microvms.amazonaws.com), and
+    # AWS does not document the service principal it passes. The statement is
+    # still least-privilege via the exact execution-role resource pin.
   }
 
   statement {

javascript-packages.txt

@@ -32,3 +32,4 @@ ejs@5.0.2
 archiver@7.0.1
 adm-zip@0.5.17
 pptxgenjs@4.0.1
+docx@9.7.1