From f4f6cef00ee284a12ed236b1bcf5678db1272cc2 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Mon, 1 Jun 2026 14:14:32 -0400 Subject: [PATCH 1/3] s3 auth cache --- CHANGES.md | 20 ++++++++++++++++++- birdhouse/components/magpie/default.env | 4 ++++ .../proxy/conf.extra-directives.d/stac.conf | 1 + .../conf.extra-service.d/s3.conf.template | 7 +++++++ .../s3/config/proxy/docker-compose-extra.yml | 1 + 5 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 birdhouse/components/s3/config/proxy/conf.extra-directives.d/stac.conf diff --git a/CHANGES.md b/CHANGES.md index 203803484..64bc45542 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,25 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ -[//]: # (list changes here, using '-' for each new entry, remove this when items are added) +## Changes + +- Add auth cache in nginx for s3 endpoint + + Adds a cache for the `s3` auth endpoint (that calls Twitcher's verify endpoint) so that repeated requests to the + S3 store don't overwhelm twitcher and cause client connection issues if twitcher or nginx fails to properly authorize + access to a resource. + + This issue may arise if a client is processing a large S3 object in parallel by accessing different byte-ranges + simultaneously. This results in many simultaneous requests to the same resource, each of which hits twitcher's verify + endpoint. With this change, only the first request each minute will hit twitcher and the rest will only hit the cache. + + Some implementation details to note: + + - responses are only cached for 1 minute to ensure that if a user's permissions are changed (on Magpie) their previous + permissions expire quickly + - the value of Magpie's cookie as well as the auth header is used as a cache key which allows us to cache cookie based + and token-based authentication methods + [2.28.0](https://github.com/bird-house/birdhouse-deploy/tree/2.28.0) (2026-05-15) ------------------------------------------------------------------------------------------------------------------ diff --git a/birdhouse/components/magpie/default.env b/birdhouse/components/magpie/default.env index e89ae7bae..a462521fa 100644 --- a/birdhouse/components/magpie/default.env +++ b/birdhouse/components/magpie/default.env @@ -61,6 +61,9 @@ export MAGPIE_NETWORK_CREATE_MISSING_PEM_FILE=true # translate MAGPIE_NETWORK_PEM_FILES to the location of the files on the magpie container export MAGPIE_NETWORK_PEM_FILES_ON_CONTAINER='$(echo "/magpie-pem/${MAGPIE_NETWORK_PEM_FILES#:}" | sed "s|:|:/magpie-pem/|g" )' +# explicitly declare the magpie cookie name so that it can be referred to by other components +export MAGPIE_COOKIE_NAME=auth_tkt + export DELAYED_EVAL=" $DELAYED_EVAL MAGPIE_PERSIST_DIR @@ -121,4 +124,5 @@ OPTIONAL_VARS=" \$MAGPIE_NETWORK_PEM_FILES_ON_CONTAINER \$MAGPIE_NETWORK_PEM_PASSWORDS \$MAGPIE_NETWORK_CREATE_MISSING_PEM_FILE + \$MAGPIE_COOKIE_NAME " diff --git a/birdhouse/components/s3/config/proxy/conf.extra-directives.d/stac.conf b/birdhouse/components/s3/config/proxy/conf.extra-directives.d/stac.conf new file mode 100644 index 000000000..dd0b9a376 --- /dev/null +++ b/birdhouse/components/s3/config/proxy/conf.extra-directives.d/stac.conf @@ -0,0 +1 @@ + proxy_cache_path /var/cache/nginx/s3_auth_cache levels=1:2 keys_zone=s3_auth_cache:10m max_size=1g inactive=10m; diff --git a/birdhouse/components/s3/config/proxy/conf.extra-service.d/s3.conf.template b/birdhouse/components/s3/config/proxy/conf.extra-service.d/s3.conf.template index f1b119a83..df2ad1c1c 100644 --- a/birdhouse/components/s3/config/proxy/conf.extra-service.d/s3.conf.template +++ b/birdhouse/components/s3/config/proxy/conf.extra-service.d/s3.conf.template @@ -12,6 +12,13 @@ internal; proxy_pass ${BIRDHOUSE_PROXY_SCHEME}://${BIRDHOUSE_FQDN_PUBLIC}${TWITCHER_VERIFY_PATH}$request_uri; proxy_pass_request_body off; + + # cache auth requests so we don't overwhelm twitcher when a client is accessing multiple chunks + # of the same file in parallel (for example). + proxy_cache "s3_auth_cache"; + proxy_cache_key "$http_authorization:cache:$cookie_${MAGPIE_COOKIE_NAME}"; + proxy_cache_valid 200 401 403 1m; + proxy_ignore_headers Cache-Control Expires Set-Cookie; proxy_set_header Host $host; proxy_set_header Content-Length ""; proxy_set_header X-Original-URI $request_uri; diff --git a/birdhouse/components/s3/config/proxy/docker-compose-extra.yml b/birdhouse/components/s3/config/proxy/docker-compose-extra.yml index 490ec70a9..09d2ba744 100644 --- a/birdhouse/components/s3/config/proxy/docker-compose-extra.yml +++ b/birdhouse/components/s3/config/proxy/docker-compose-extra.yml @@ -2,4 +2,5 @@ services: proxy: volumes: - ./components/s3/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/s3:ro + - ./components/s3/config/proxy/conf.extra-directives.d:/etc/nginx/conf.extra-directives.d/s3:ro - ./components/s3/service-config.json:/static-services/s3.json:ro From ee1bb9fa4c7ecb591e2c22f908dc0227ba605ae7 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Mon, 1 Jun 2026 14:20:20 -0400 Subject: [PATCH 2/3] include uri in cache key [skip jenkins] --- .../s3/config/proxy/conf.extra-service.d/s3.conf.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/birdhouse/components/s3/config/proxy/conf.extra-service.d/s3.conf.template b/birdhouse/components/s3/config/proxy/conf.extra-service.d/s3.conf.template index df2ad1c1c..71ed76703 100644 --- a/birdhouse/components/s3/config/proxy/conf.extra-service.d/s3.conf.template +++ b/birdhouse/components/s3/config/proxy/conf.extra-service.d/s3.conf.template @@ -16,7 +16,7 @@ # cache auth requests so we don't overwhelm twitcher when a client is accessing multiple chunks # of the same file in parallel (for example). proxy_cache "s3_auth_cache"; - proxy_cache_key "$http_authorization:cache:$cookie_${MAGPIE_COOKIE_NAME}"; + proxy_cache_key "$request_uri:uri:$http_authorization:cache:$cookie_${MAGPIE_COOKIE_NAME}"; proxy_cache_valid 200 401 403 1m; proxy_ignore_headers Cache-Control Expires Set-Cookie; proxy_set_header Host $host; From 23d30bbd7ccb50ecf41bc0f2c0af4e4c9711648e Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 3 Jun 2026 09:05:58 -0400 Subject: [PATCH 3/3] review comments --- birdhouse/components/magpie/docker-compose-extra.yml | 1 + .../s3/config/proxy/conf.extra-service.d/s3.conf.template | 2 +- birdhouse/components/s3/default.env | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/birdhouse/components/magpie/docker-compose-extra.yml b/birdhouse/components/magpie/docker-compose-extra.yml index 6935e42ac..979206e85 100644 --- a/birdhouse/components/magpie/docker-compose-extra.yml +++ b/birdhouse/components/magpie/docker-compose-extra.yml @@ -12,6 +12,7 @@ services: MAGPIE_WEBHOOKS_CONFIG_PATH: "${MAGPIE_WEBHOOKS_CONFIG_PATH}" MAGPIE_POSTGRES_HOST: postgres-magpie MAGPIE_PORT: 2001 + MAGPIE_COOKIE_NAME: "${MAGPIE_COOKIE_NAME}" FORWARDED_ALLOW_IPS: "*" env_file: - ./components/magpie/postgres-credentials.env diff --git a/birdhouse/components/s3/config/proxy/conf.extra-service.d/s3.conf.template b/birdhouse/components/s3/config/proxy/conf.extra-service.d/s3.conf.template index 71ed76703..acde2894d 100644 --- a/birdhouse/components/s3/config/proxy/conf.extra-service.d/s3.conf.template +++ b/birdhouse/components/s3/config/proxy/conf.extra-service.d/s3.conf.template @@ -17,7 +17,7 @@ # of the same file in parallel (for example). proxy_cache "s3_auth_cache"; proxy_cache_key "$request_uri:uri:$http_authorization:cache:$cookie_${MAGPIE_COOKIE_NAME}"; - proxy_cache_valid 200 401 403 1m; + proxy_cache_valid 200 401 403 ${S3_AUTH_CACHE_TIMEOUT}; proxy_ignore_headers Cache-Control Expires Set-Cookie; proxy_set_header Host $host; proxy_set_header Content-Length ""; diff --git a/birdhouse/components/s3/default.env b/birdhouse/components/s3/default.env index ed8144c74..a61e76672 100644 --- a/birdhouse/components/s3/default.env +++ b/birdhouse/components/s3/default.env @@ -17,6 +17,10 @@ export __DEFAULT__S3_ROOT_SECRET_KEY=S3adminsecret export S3_ROOT_ACCESS_KEY="${__DEFAULT__S3_ROOT_ACCESS_KEY}" export S3_ROOT_SECRET_KEY="${__DEFAULT__S3_ROOT_SECRET_KEY}" +# S3 authentication will be cached for this duration to not overwhelm twitcher. +# To disable the cache set this to 0m (0 minutes). +export S3_AUTH_CACHE_TIMEOUT=1m + export DELAYED_EVAL=" $DELAYED_EVAL S3_IMAGE @@ -33,4 +37,5 @@ export VARS=" \$S3_ROOT_SECRET_KEY \$S3_VERSION_SEMVER \$S3_IMAGE_URI + \$S3_AUTH_CACHE_TIMEOUT "