bird-house · mishaschwartz · Jun 1, 2026 · Jun 1, 2026 · Jun 3, 2026
@@ -15,7 +15,25 @@
 [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest)
 ------------------------------------------------------------------------------------------------------------------
 
-[//]: # (list changes here, using '-' for each new entry, remove this when items are added)
+## Changes
+
+- Add auth cache in nginx for s3 endpoint
+
+  Adds a cache for the `s3` auth endpoint (that calls Twitcher's verify endpoint) so that repeated requests to the 
+  S3 store don't overwhelm twitcher and cause client connection issues if twitcher or nginx fails to properly authorize
+  access to a resource.
+
+  This issue may arise if a client is processing a large S3 object in parallel by accessing different byte-ranges
+  simultaneously. This results in many simultaneous requests to the same resource, each of which hits twitcher's verify
+  endpoint. With this change, only the first request each minute will hit twitcher and the rest will only hit the cache.
+
+  Some implementation details to note:
+
+  - responses are only cached for 1 minute to ensure that if a user's permissions are changed (on Magpie) their previous
+    permissions expire quickly
+  - the value of Magpie's cookie as well as the auth header is used as a cache key which allows us to cache cookie based
+    and token-based authentication methods
+
 
 [2.28.0](https://github.com/bird-house/birdhouse-deploy/tree/2.28.0) (2026-05-15)
 ------------------------------------------------------------------------------------------------------------------

@@ -61,6 +61,9 @@ export MAGPIE_NETWORK_CREATE_MISSING_PEM_FILE=true
 # translate MAGPIE_NETWORK_PEM_FILES to the location of the files on the magpie container
 export MAGPIE_NETWORK_PEM_FILES_ON_CONTAINER='$(echo "/magpie-pem/${MAGPIE_NETWORK_PEM_FILES#:}" | sed "s|:|:/magpie-pem/|g" )'
 
+# explicitly declare the magpie cookie name so that it can be referred to by other components
+export MAGPIE_COOKIE_NAME=auth_tkt
+
 export DELAYED_EVAL="
   $DELAYED_EVAL
   MAGPIE_PERSIST_DIR
@@ -121,4 +124,5 @@ OPTIONAL_VARS="
   \$MAGPIE_NETWORK_PEM_FILES_ON_CONTAINER
   \$MAGPIE_NETWORK_PEM_PASSWORDS
   \$MAGPIE_NETWORK_CREATE_MISSING_PEM_FILE
+  \$MAGPIE_COOKIE_NAME
 "
@@ -12,6 +12,7 @@ services:
       MAGPIE_WEBHOOKS_CONFIG_PATH: "${MAGPIE_WEBHOOKS_CONFIG_PATH}"
       MAGPIE_POSTGRES_HOST: postgres-magpie
       MAGPIE_PORT: 2001
+      MAGPIE_COOKIE_NAME: "${MAGPIE_COOKIE_NAME}"
       FORWARDED_ALLOW_IPS: "*"
     env_file:
       - ./components/magpie/postgres-credentials.env

@@ -0,0 +1 @@
+    proxy_cache_path /var/cache/nginx/s3_auth_cache levels=1:2 keys_zone=s3_auth_cache:10m max_size=1g inactive=10m;
@@ -12,6 +12,13 @@
         internal;
         proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}$request_uri;
         proxy_pass_request_body off;
+
+        # cache auth requests so we don't overwhelm twitcher when a client is accessing multiple chunks
+        # of the same file in parallel (for example).
+        proxy_cache "s3_auth_cache";
+        proxy_cache_key "$request_uri:uri:$http_authorization:cache:$cookie_${MAGPIE_COOKIE_NAME}";
+        proxy_cache_valid 200 401 403 ${S3_AUTH_CACHE_TIMEOUT};
+        proxy_ignore_headers Cache-Control Expires Set-Cookie; 
         proxy_set_header Host $host;
         proxy_set_header Content-Length "";
         proxy_set_header X-Original-URI $request_uri;

@@ -2,4 +2,5 @@ services:
   proxy:
     volumes:
       - ./components/s3/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/s3:ro
+      - ./components/s3/config/proxy/conf.extra-directives.d:/etc/nginx/conf.extra-directives.d/s3:ro
       - ./components/s3/service-config.json:/static-services/s3.json:ro
@@ -17,6 +17,10 @@ export __DEFAULT__S3_ROOT_SECRET_KEY=S3adminsecret
 export S3_ROOT_ACCESS_KEY="${__DEFAULT__S3_ROOT_ACCESS_KEY}"
 export S3_ROOT_SECRET_KEY="${__DEFAULT__S3_ROOT_SECRET_KEY}"
 
+# S3 authentication will be cached for this duration to not overwhelm twitcher.
+# To disable the cache set this to 0m (0 minutes).
+export S3_AUTH_CACHE_TIMEOUT=1m
+
 export DELAYED_EVAL="
   $DELAYED_EVAL
   S3_IMAGE
@@ -33,4 +37,5 @@ export VARS="
   \$S3_ROOT_SECRET_KEY
   \$S3_VERSION_SEMVER
   \$S3_IMAGE_URI
+  \$S3_AUTH_CACHE_TIMEOUT
 "
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		proxy_cache_path /var/cache/nginx/s3_auth_cache levels=1:2 keys_zone=s3_auth_cache:10m max_size=1g inactive=10m;