SASL E2E lane, rolling-restart coverage, collection pin, and credential-handling fixes

.buildkite/pipeline.yml

@@ -1,6 +1,7 @@
 agents:
   queue: "k8s-m6ixlarge"
 
+
 steps:
   - label: aws ubuntu
     key: aws-up-ubuntu
@@ -20,8 +21,30 @@ steps:
             - AWS_SECRET_ACCESS_KEY
             - AWS_DEFAULT_REGION
             - BASELINE_COLLECTION_REF
-            - CANDIDATE_COLLECTION_REF
+            # TEMP (DO NOT MERGE): the aws-sm secret exports git+...,main into the
+            # job env AFTER yaml env applies; only an in-container pin wins
+            - CANDIDATE_COLLECTION_REF=git+https://github.com/redpanda-data/redpanda-ansible-collection.git,collection-hardening
             - REDPANDA_VERSION
+  - label: aws ubuntu sasl
+    key: aws-up-ubuntu-sasl
+    concurrency_group: aws-ub
+    concurrency: 1
+    timeout_in_minutes: 120
+    command: DEPLOYMENT_ID=ci-sa-ub-`tr -dc a-z0-9 </dev/urandom | head -c 4` DISTRO=ubuntu-focal task ci:aws:rp:sasl
+    plugins:
+      - seek-oss/aws-sm#v2.3.2:
+          json-to-env:
+            - json-key: .
+              secret-id: sdlc/prod/buildkite/deployment_automation
+      - docker#v5.8.0:
+          image: glrp/atgt:latest
+          environment:
+            - AWS_ACCESS_KEY_ID
+            - AWS_SECRET_ACCESS_KEY
+            - AWS_DEFAULT_REGION
+            # TEMP (DO NOT MERGE): the aws-sm secret exports git+...,main into the
+            # job env AFTER yaml env applies; only an in-container pin wins
+            - CANDIDATE_COLLECTION_REF=git+https://github.com/redpanda-data/redpanda-ansible-collection.git,collection-hardening
   - label: aws ubuntu tiered
     key: aws-up-ubuntu-tiered
     concurrency_group: aws-ub
@@ -41,7 +64,9 @@ steps:
             - REDPANDA_LICENSE
             - AWS_DEFAULT_REGION
             - BASELINE_COLLECTION_REF
-            - CANDIDATE_COLLECTION_REF
+            # TEMP (DO NOT MERGE): the aws-sm secret exports git+...,main into the
+            # job env AFTER yaml env applies; only an in-container pin wins
+            - CANDIDATE_COLLECTION_REF=git+https://github.com/redpanda-data/redpanda-ansible-collection.git,collection-hardening
             - REDPANDA_VERSION
   - label: aws ubuntu tiered large
     key: aws-up-ubuntu-ts-large
@@ -62,7 +87,9 @@ steps:
             - REDPANDA_LICENSE
             - AWS_DEFAULT_REGION
             - BASELINE_COLLECTION_REF
-            - CANDIDATE_COLLECTION_REF
+            # TEMP (DO NOT MERGE): the aws-sm secret exports git+...,main into the
+            # job env AFTER yaml env applies; only an in-container pin wins
+            - CANDIDATE_COLLECTION_REF=git+https://github.com/redpanda-data/redpanda-ansible-collection.git,collection-hardening
             - REDPANDA_VERSION
   - label: aws fedora
     key: aws-up-fedora
@@ -82,7 +109,9 @@ steps:
             - AWS_SECRET_ACCESS_KEY
             - AWS_DEFAULT_REGION
             - BASELINE_COLLECTION_REF
-            - CANDIDATE_COLLECTION_REF
+            # TEMP (DO NOT MERGE): the aws-sm secret exports git+...,main into the
+            # job env AFTER yaml env applies; only an in-container pin wins
+            - CANDIDATE_COLLECTION_REF=git+https://github.com/redpanda-data/redpanda-ansible-collection.git,collection-hardening
             - REDPANDA_VERSION
   - label: aws fedora connect
     key: aws-up-fed-con
@@ -103,7 +132,9 @@ steps:
             - AWS_DEFAULT_REGION
             - CONNECT_RPM_TOKEN
             - BASELINE_COLLECTION_REF
-            - CANDIDATE_COLLECTION_REF
+            # TEMP (DO NOT MERGE): the aws-sm secret exports git+...,main into the
+            # job env AFTER yaml env applies; only an in-container pin wins
+            - CANDIDATE_COLLECTION_REF=git+https://github.com/redpanda-data/redpanda-ansible-collection.git,collection-hardening
             - REDPANDA_VERSION
   - label: aws fedora tiered
     key: aws-up-fedora-tiered
@@ -124,7 +155,9 @@ steps:
             - REDPANDA_LICENSE
             - AWS_DEFAULT_REGION
             - BASELINE_COLLECTION_REF
-            - CANDIDATE_COLLECTION_REF
+            # TEMP (DO NOT MERGE): the aws-sm secret exports git+...,main into the
+            # job env AFTER yaml env applies; only an in-container pin wins
+            - CANDIDATE_COLLECTION_REF=git+https://github.com/redpanda-data/redpanda-ansible-collection.git,collection-hardening
             - REDPANDA_VERSION
   - label: aws fedora tiered connect
     key: aws-up-fed-cts
@@ -146,7 +179,9 @@ steps:
             - REDPANDA_LICENSE
             - CONNECT_RPM_TOKEN
             - BASELINE_COLLECTION_REF
-            - CANDIDATE_COLLECTION_REF
+            # TEMP (DO NOT MERGE): the aws-sm secret exports git+...,main into the
+            # job env AFTER yaml env applies; only an in-container pin wins
+            - CANDIDATE_COLLECTION_REF=git+https://github.com/redpanda-data/redpanda-ansible-collection.git,collection-hardening
             - REDPANDA_VERSION
   - label: aws fedora tiered large
     key: aws-up-fedora-ts-large
@@ -167,7 +202,9 @@ steps:
             - REDPANDA_LICENSE
             - AWS_DEFAULT_REGION
             - BASELINE_COLLECTION_REF
-            - CANDIDATE_COLLECTION_REF
+            # TEMP (DO NOT MERGE): the aws-sm secret exports git+...,main into the
+            # job env AFTER yaml env applies; only an in-container pin wins
+            - CANDIDATE_COLLECTION_REF=git+https://github.com/redpanda-data/redpanda-ansible-collection.git,collection-hardening
             - REDPANDA_VERSION
   - label: gcp ubuntu basic
     key: gcp-up-ubuntu
@@ -185,7 +222,9 @@ steps:
           environment:
             - GCP_CREDS
             - BASELINE_COLLECTION_REF
-            - CANDIDATE_COLLECTION_REF
+            # TEMP (DO NOT MERGE): the aws-sm secret exports git+...,main into the
+            # job env AFTER yaml env applies; only an in-container pin wins
+            - CANDIDATE_COLLECTION_REF=git+https://github.com/redpanda-data/redpanda-ansible-collection.git,collection-hardening
             - REDPANDA_VERSION
   - label: gcp ubuntu tiered
     key: gcp-up-ubuntu-tiered
@@ -205,7 +244,9 @@ steps:
             - GCP_CREDS
             - REDPANDA_LICENSE
             - BASELINE_COLLECTION_REF
-            - CANDIDATE_COLLECTION_REF
+            # TEMP (DO NOT MERGE): the aws-sm secret exports git+...,main into the
+            # job env AFTER yaml env applies; only an in-container pin wins
+            - CANDIDATE_COLLECTION_REF=git+https://github.com/redpanda-data/redpanda-ansible-collection.git,collection-hardening
             - REDPANDA_VERSION
   - label: gcp fedora basic
     key: gcp-up-fedora
@@ -223,7 +264,9 @@ steps:
           environment:
             - GCP_CREDS
             - BASELINE_COLLECTION_REF
-            - CANDIDATE_COLLECTION_REF
+            # TEMP (DO NOT MERGE): the aws-sm secret exports git+...,main into the
+            # job env AFTER yaml env applies; only an in-container pin wins
+            - CANDIDATE_COLLECTION_REF=git+https://github.com/redpanda-data/redpanda-ansible-collection.git,collection-hardening
             - REDPANDA_VERSION
   - label: gcp fedora tiered
     key: gcp-up-fedora-tiered
@@ -242,7 +285,9 @@ steps:
             - GCP_CREDS
             - REDPANDA_LICENSE
             - BASELINE_COLLECTION_REF
-            - CANDIDATE_COLLECTION_REF
+            # TEMP (DO NOT MERGE): the aws-sm secret exports git+...,main into the
+            # job env AFTER yaml env applies; only an in-container pin wins
+            - CANDIDATE_COLLECTION_REF=git+https://github.com/redpanda-data/redpanda-ansible-collection.git,collection-hardening
             - REDPANDA_VERSION
   - label: unstable aws fedora tiered
     key: aws-us-fedora-tiered
@@ -263,7 +308,9 @@ steps:
             - REDPANDA_LICENSE
             - AWS_DEFAULT_REGION
             - BASELINE_COLLECTION_REF
-            - CANDIDATE_COLLECTION_REF
+            # TEMP (DO NOT MERGE): the aws-sm secret exports git+...,main into the
+            # job env AFTER yaml env applies; only an in-container pin wins
+            - CANDIDATE_COLLECTION_REF=git+https://github.com/redpanda-data/redpanda-ansible-collection.git,collection-hardening
             - REDPANDA_VERSION
   - label: unstable aws fedora tiered large
     key: aws-us-fedora-ts-large
@@ -284,7 +331,9 @@ steps:
             - REDPANDA_LICENSE
             - AWS_DEFAULT_REGION
             - BASELINE_COLLECTION_REF
-            - CANDIDATE_COLLECTION_REF
+            # TEMP (DO NOT MERGE): the aws-sm secret exports git+...,main into the
+            # job env AFTER yaml env applies; only an in-container pin wins
+            - CANDIDATE_COLLECTION_REF=git+https://github.com/redpanda-data/redpanda-ansible-collection.git,collection-hardening
             - REDPANDA_VERSION
   - label: unstable aws ubuntu tiered
     key: aws-us-ubuntu-tiered
@@ -305,7 +354,9 @@ steps:
             - REDPANDA_LICENSE
             - AWS_DEFAULT_REGION
             - BASELINE_COLLECTION_REF
-            - CANDIDATE_COLLECTION_REF
+            # TEMP (DO NOT MERGE): the aws-sm secret exports git+...,main into the
+            # job env AFTER yaml env applies; only an in-container pin wins
+            - CANDIDATE_COLLECTION_REF=git+https://github.com/redpanda-data/redpanda-ansible-collection.git,collection-hardening
             - REDPANDA_VERSION
   - label: unstable aws ubuntu tiered large
     key: aws-us-ubuntu-ts-large
@@ -326,7 +377,9 @@ steps:
             - REDPANDA_LICENSE
             - AWS_DEFAULT_REGION
             - BASELINE_COLLECTION_REF
-            - CANDIDATE_COLLECTION_REF
+            # TEMP (DO NOT MERGE): the aws-sm secret exports git+...,main into the
+            # job env AFTER yaml env applies; only an in-container pin wins
+            - CANDIDATE_COLLECTION_REF=git+https://github.com/redpanda-data/redpanda-ansible-collection.git,collection-hardening
             - REDPANDA_VERSION
 
   - label: cleanup aws resources

.gitignore

@@ -22,3 +22,4 @@ ansible/tls/clients/client.crt
 /aws-extra/
 ansible/proxy/tls/**
 .ansible
+.env

.tasks/ci.yml

@@ -149,6 +149,62 @@ tasks:
         vars: {UPGRADE_SENTINEL_TOPIC: "{{.UPGRADE_SENTINEL_TOPIC}}", SEED_COUNT: "{{.SEED_COUNT}}", RPK_EXTRA: "{{.RPK_EXTRA}}"}
       - task: :infra:aws:destroy
 
+  aws:rp:sasl:
+    desc: >-
+      CI workflow - AWS Redpanda with SASL authorization. Single-phase: fresh-installs the
+      CANDIDATE collection with kafka_enable_authorization, reconciles a test user + topic ACL
+      via user_management, verifies enforcement (superuser ok, anonymous denied, ACL-scoped
+      user constrained to its topic), performs a SASL-authenticated rolling restart via
+      operation-rolling-restart.yml, and re-verifies enforcement and data access afterwards.
+    vars:
+      DISTRO: '{{.DISTRO | default "ubuntu-focal"}}'
+      SASL_USER: admin
+      SASL_PASSWORD: '{{.DEPLOYMENT_ID}}-sasl-pw'
+      SASL_TEST_USER: apptest
+      SASL_TEST_PASSWORD: '{{.DEPLOYMENT_ID}}-app-pw'
+      SASL_TEST_TOPIC: apptopic
+    cmds:
+      - defer:
+          task: :infra:aws:destroy
+      - task: :tools:keygen
+      - task: :tools:rpk:install
+      - task: :infra:aws:build
+        vars: {DISTRO: "{{.DISTRO}}"}
+      - task: :ansible:prereqs
+      - task: :ansible:collection:candidate
+        vars: {CANDIDATE_COLLECTION_REF: "{{.CANDIDATE_COLLECTION_REF}}"}
+      - task: :cluster:sasl
+        vars:
+          SASL_USER: "{{.SASL_USER}}"
+          SASL_PASSWORD: "{{.SASL_PASSWORD}}"
+      - task: :ops:users:apply
+        vars:
+          SASL_USER: "{{.SASL_USER}}"
+          SASL_PASSWORD: "{{.SASL_PASSWORD}}"
+          SASL_USERS_JSON: '{"sasl_users":[{"username":"{{.SASL_TEST_USER}}","password":"{{.SASL_TEST_PASSWORD}}","state":"present"}]}'
+          SASL_ACLS_JSON: '{"sasl_acls":[{"principal":"{{.SASL_TEST_USER}}","operation":["read","write","describe"],"resource_type":"topic","resource_name":"{{.SASL_TEST_TOPIC}}","permission":"allow"}]}'
+      - task: :test:cluster:sasl
+        vars:
+          SASL_USER: "{{.SASL_USER}}"
+          SASL_PASSWORD: "{{.SASL_PASSWORD}}"
+          SASL_TEST_USER: "{{.SASL_TEST_USER}}"
+          SASL_TEST_PASSWORD: "{{.SASL_TEST_PASSWORD}}"
+          SASL_TEST_TOPIC: "{{.SASL_TEST_TOPIC}}"
+      # rolling restart with SASL credentials; enforcement and the app user's
+      # data must survive it
+      - task: :ops:broker:restart
+        vars:
+          SASL_USER: "{{.SASL_USER}}"
+          SASL_PASSWORD: "{{.SASL_PASSWORD}}"
+      - task: :test:cluster:sasl
+        vars:
+          SASL_USER: "{{.SASL_USER}}"
+          SASL_PASSWORD: "{{.SASL_PASSWORD}}"
+          SASL_TEST_USER: "{{.SASL_TEST_USER}}"
+          SASL_TEST_PASSWORD: "{{.SASL_TEST_PASSWORD}}"
+          SASL_TEST_TOPIC: "{{.SASL_TEST_TOPIC}}"
+      - task: :infra:aws:destroy
+
   aws:rp:tiered:unstable:
     desc: >-
       CI workflow - AWS tiered storage (TLS) against the UNSTABLE Redpanda repo. Single-phase,

.tasks/cleanup.yml

@@ -7,15 +7,12 @@ vars:
   GCP_PROJECT_ID: '{{.GCP_PROJECT_ID | default "hallowed-ray-376320"}}'
   MIN_AGE: '{{.MIN_AGE | default ""}}'
 
-env:
-  # AWS Credentials (inherited from parent or environment)
-  AWS_ACCESS_KEY_ID: '{{.AWS_ACCESS_KEY_ID}}'
-  AWS_SECRET_ACCESS_KEY: '{{.AWS_SECRET_ACCESS_KEY}}'
-  AWS_DEFAULT_REGION: '{{.AWS_CLEANUP_REGION}}'
-
-  # GCP Credentials (supports both variable names)
-  GCP_CREDS: '{{.GCP_CREDS}}'
-  GOOGLE_CREDENTIALS_BASE64: '{{.GOOGLE_CREDENTIALS_BASE64}}'
+# No env block here: included-taskfile env merges into the WHOLE run, and
+# these entries clobbered dotenv-supplied credentials globally (the cred
+# templates resolve empty before dotenv merges) while exporting the
+# comma-joined reaper region list as everyone's AWS_DEFAULT_REGION. The
+# reaper gets its regions via --region; credentials flow from process env
+# or .env untouched.
 
 tasks:
   aws:

.tasks/cluster.yml

@@ -21,6 +21,24 @@ tasks:
         {{if .RP_VERSION}}--extra-vars redpanda_version={{.RP_VERSION}}{{end}}
         {{if .RP_INSTALL_STATUS}}--extra-vars redpanda_install_status={{.RP_INSTALL_STATUS}}{{end}}
 
+  sasl:
+    desc: >-
+      Provision/converge Redpanda with SASL authorization enabled. SASL_USER/SASL_PASSWORD
+      set the bootstrap superuser; RP_VERSION/RP_INSTALL_STATUS as in :cluster:provision.
+    deps:
+      - :ansible:prereqs
+      - :ensure-logs-dir
+    cmds:
+      - >-
+        ansible-playbook ansible/provision-cluster-sasl.yml
+        --private-key {{.PRIVATE_KEY}}
+        --inventory {{.ANSIBLE_INVENTORY}}
+        --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}}
+        --extra-vars sasl_user={{.SASL_USER}}
+        --extra-vars sasl_password={{.SASL_PASSWORD}}
+        {{if .RP_VERSION}}--extra-vars redpanda_version={{.RP_VERSION}}{{end}}
+        {{if .RP_INSTALL_STATUS}}--extra-vars redpanda_install_status={{.RP_INSTALL_STATUS}}{{end}}
+
   tiered:
     desc: >-
       Provision/converge Redpanda with tiered storage. RP_VERSION/RP_INSTALL_STATUS as in

.tasks/monitoring.yml

@@ -8,15 +8,29 @@ tasks:
       - :ensure-logs-dir
     cmds:
       - export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
-      - ansible-playbook ansible/deploy-monitor.yml --private-key {{.PRIVATE_KEY}} --inventory {{.ANSIBLE_INVENTORY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}}
+      - |
+        # node_exporter downloads from GitHub flake under concurrent-lane rate
+        # limiting (builds 445/446); the play is idempotent, so retry once
+        for attempt in 1 2; do
+          ansible-playbook ansible/deploy-monitor.yml --private-key {{.PRIVATE_KEY}} --inventory {{.ANSIBLE_INVENTORY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} && break
+          [ "$attempt" = 2 ] && exit 1
+          echo "monitor deploy failed (attempt $attempt); retrying in 30s"; sleep 30
+        done
 
   deploy:tls:
     desc: Deploy monitoring stack with TLS
     deps:
       - :ansible:prereqs
       - :ensure-logs-dir
     cmds:
-      - ansible-playbook ansible/deploy-monitor-tls.yml --private-key {{.PRIVATE_KEY}} --inventory {{.ANSIBLE_INVENTORY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}}
+      - |
+        # node_exporter downloads from GitHub flake under concurrent-lane rate
+        # limiting (builds 445/446); the play is idempotent, so retry once
+        for attempt in 1 2; do
+          ansible-playbook ansible/deploy-monitor-tls.yml --private-key {{.PRIVATE_KEY}} --inventory {{.ANSIBLE_INVENTORY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} && break
+          [ "$attempt" = 2 ] && exit 1
+          echo "monitor deploy failed (attempt $attempt); retrying in 30s"; sleep 30
+        done
 
   extra:deploy:
     desc: Deploy monitoring on secondary cluster