diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 64722fe9..874c5467 100755 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,6 +1,7 @@ agents: queue: "k8s-m6ixlarge" + steps: - label: aws ubuntu key: aws-up-ubuntu @@ -20,8 +21,30 @@ steps: - AWS_SECRET_ACCESS_KEY - AWS_DEFAULT_REGION - BASELINE_COLLECTION_REF - - CANDIDATE_COLLECTION_REF + # TEMP (DO NOT MERGE): the aws-sm secret exports git+...,main into the + # job env AFTER yaml env applies; only an in-container pin wins + - CANDIDATE_COLLECTION_REF=git+https://github.com/redpanda-data/redpanda-ansible-collection.git,collection-hardening - REDPANDA_VERSION + - label: aws ubuntu sasl + key: aws-up-ubuntu-sasl + concurrency_group: aws-ub + concurrency: 1 + timeout_in_minutes: 120 + command: DEPLOYMENT_ID=ci-sa-ub-`tr -dc a-z0-9 - + CI workflow - AWS Redpanda with SASL authorization. Single-phase: fresh-installs the + CANDIDATE collection with kafka_enable_authorization, reconciles a test user + topic ACL + via user_management, verifies enforcement (superuser ok, anonymous denied, ACL-scoped + user constrained to its topic), performs a SASL-authenticated rolling restart via + operation-rolling-restart.yml, and re-verifies enforcement and data access afterwards. + vars: + DISTRO: '{{.DISTRO | default "ubuntu-focal"}}' + SASL_USER: admin + SASL_PASSWORD: '{{.DEPLOYMENT_ID}}-sasl-pw' + SASL_TEST_USER: apptest + SASL_TEST_PASSWORD: '{{.DEPLOYMENT_ID}}-app-pw' + SASL_TEST_TOPIC: apptopic + cmds: + - defer: + task: :infra:aws:destroy + - task: :tools:keygen + - task: :tools:rpk:install + - task: :infra:aws:build + vars: {DISTRO: "{{.DISTRO}}"} + - task: :ansible:prereqs + - task: :ansible:collection:candidate + vars: {CANDIDATE_COLLECTION_REF: "{{.CANDIDATE_COLLECTION_REF}}"} + - task: :cluster:sasl + vars: + SASL_USER: "{{.SASL_USER}}" + SASL_PASSWORD: "{{.SASL_PASSWORD}}" + - task: :ops:users:apply + vars: + SASL_USER: "{{.SASL_USER}}" + SASL_PASSWORD: "{{.SASL_PASSWORD}}" + SASL_USERS_JSON: '{"sasl_users":[{"username":"{{.SASL_TEST_USER}}","password":"{{.SASL_TEST_PASSWORD}}","state":"present"}]}' + SASL_ACLS_JSON: '{"sasl_acls":[{"principal":"{{.SASL_TEST_USER}}","operation":["read","write","describe"],"resource_type":"topic","resource_name":"{{.SASL_TEST_TOPIC}}","permission":"allow"}]}' + - task: :test:cluster:sasl + vars: + SASL_USER: "{{.SASL_USER}}" + SASL_PASSWORD: "{{.SASL_PASSWORD}}" + SASL_TEST_USER: "{{.SASL_TEST_USER}}" + SASL_TEST_PASSWORD: "{{.SASL_TEST_PASSWORD}}" + SASL_TEST_TOPIC: "{{.SASL_TEST_TOPIC}}" + # rolling restart with SASL credentials; enforcement and the app user's + # data must survive it + - task: :ops:broker:restart + vars: + SASL_USER: "{{.SASL_USER}}" + SASL_PASSWORD: "{{.SASL_PASSWORD}}" + - task: :test:cluster:sasl + vars: + SASL_USER: "{{.SASL_USER}}" + SASL_PASSWORD: "{{.SASL_PASSWORD}}" + SASL_TEST_USER: "{{.SASL_TEST_USER}}" + SASL_TEST_PASSWORD: "{{.SASL_TEST_PASSWORD}}" + SASL_TEST_TOPIC: "{{.SASL_TEST_TOPIC}}" + - task: :infra:aws:destroy + aws:rp:tiered:unstable: desc: >- CI workflow - AWS tiered storage (TLS) against the UNSTABLE Redpanda repo. Single-phase, diff --git a/.tasks/cleanup.yml b/.tasks/cleanup.yml index 968307d8..f137daae 100644 --- a/.tasks/cleanup.yml +++ b/.tasks/cleanup.yml @@ -7,15 +7,12 @@ vars: GCP_PROJECT_ID: '{{.GCP_PROJECT_ID | default "hallowed-ray-376320"}}' MIN_AGE: '{{.MIN_AGE | default ""}}' -env: - # AWS Credentials (inherited from parent or environment) - AWS_ACCESS_KEY_ID: '{{.AWS_ACCESS_KEY_ID}}' - AWS_SECRET_ACCESS_KEY: '{{.AWS_SECRET_ACCESS_KEY}}' - AWS_DEFAULT_REGION: '{{.AWS_CLEANUP_REGION}}' - - # GCP Credentials (supports both variable names) - GCP_CREDS: '{{.GCP_CREDS}}' - GOOGLE_CREDENTIALS_BASE64: '{{.GOOGLE_CREDENTIALS_BASE64}}' +# No env block here: included-taskfile env merges into the WHOLE run, and +# these entries clobbered dotenv-supplied credentials globally (the cred +# templates resolve empty before dotenv merges) while exporting the +# comma-joined reaper region list as everyone's AWS_DEFAULT_REGION. The +# reaper gets its regions via --region; credentials flow from process env +# or .env untouched. tasks: aws: diff --git a/.tasks/cluster.yml b/.tasks/cluster.yml index d998309d..d1fdf968 100644 --- a/.tasks/cluster.yml +++ b/.tasks/cluster.yml @@ -21,6 +21,24 @@ tasks: {{if .RP_VERSION}}--extra-vars redpanda_version={{.RP_VERSION}}{{end}} {{if .RP_INSTALL_STATUS}}--extra-vars redpanda_install_status={{.RP_INSTALL_STATUS}}{{end}} + sasl: + desc: >- + Provision/converge Redpanda with SASL authorization enabled. SASL_USER/SASL_PASSWORD + set the bootstrap superuser; RP_VERSION/RP_INSTALL_STATUS as in :cluster:provision. + deps: + - :ansible:prereqs + - :ensure-logs-dir + cmds: + - >- + ansible-playbook ansible/provision-cluster-sasl.yml + --private-key {{.PRIVATE_KEY}} + --inventory {{.ANSIBLE_INVENTORY}} + --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} + --extra-vars sasl_user={{.SASL_USER}} + --extra-vars sasl_password={{.SASL_PASSWORD}} + {{if .RP_VERSION}}--extra-vars redpanda_version={{.RP_VERSION}}{{end}} + {{if .RP_INSTALL_STATUS}}--extra-vars redpanda_install_status={{.RP_INSTALL_STATUS}}{{end}} + tiered: desc: >- Provision/converge Redpanda with tiered storage. RP_VERSION/RP_INSTALL_STATUS as in diff --git a/.tasks/monitoring.yml b/.tasks/monitoring.yml index cd2e2dbe..6fa0eb39 100644 --- a/.tasks/monitoring.yml +++ b/.tasks/monitoring.yml @@ -8,7 +8,14 @@ tasks: - :ensure-logs-dir cmds: - export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES - - ansible-playbook ansible/deploy-monitor.yml --private-key {{.PRIVATE_KEY}} --inventory {{.ANSIBLE_INVENTORY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} + - | + # node_exporter downloads from GitHub flake under concurrent-lane rate + # limiting (builds 445/446); the play is idempotent, so retry once + for attempt in 1 2; do + ansible-playbook ansible/deploy-monitor.yml --private-key {{.PRIVATE_KEY}} --inventory {{.ANSIBLE_INVENTORY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} && break + [ "$attempt" = 2 ] && exit 1 + echo "monitor deploy failed (attempt $attempt); retrying in 30s"; sleep 30 + done deploy:tls: desc: Deploy monitoring stack with TLS @@ -16,7 +23,14 @@ tasks: - :ansible:prereqs - :ensure-logs-dir cmds: - - ansible-playbook ansible/deploy-monitor-tls.yml --private-key {{.PRIVATE_KEY}} --inventory {{.ANSIBLE_INVENTORY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} + - | + # node_exporter downloads from GitHub flake under concurrent-lane rate + # limiting (builds 445/446); the play is idempotent, so retry once + for attempt in 1 2; do + ansible-playbook ansible/deploy-monitor-tls.yml --private-key {{.PRIVATE_KEY}} --inventory {{.ANSIBLE_INVENTORY}} --extra-vars is_using_unstable={{.IS_USING_UNSTABLE}} && break + [ "$attempt" = 2 ] && exit 1 + echo "monitor deploy failed (attempt $attempt); retrying in 30s"; sleep 30 + done extra:deploy: desc: Deploy monitoring on secondary cluster diff --git a/.tasks/operations.yml b/.tasks/operations.yml index c63aec43..d6af2d0e 100644 --- a/.tasks/operations.yml +++ b/.tasks/operations.yml @@ -18,12 +18,35 @@ tasks: - ansible-playbook ansible/operation-rolling-restart-connect.yml --private-key {{.PRIVATE_KEY}} --inventory {{.ANSIBLE_INVENTORY}} broker:restart: - desc: Perform rolling restart of Redpanda broker cluster + desc: >- + Perform rolling restart of Redpanda broker cluster. On SASL clusters pass + SASL_USER/SASL_PASSWORD so the playbook's rpk calls authenticate. deps: - :ansible:prereqs - :ensure-logs-dir cmds: - - ansible-playbook ansible/operation-rolling-restart.yml --private-key {{.PRIVATE_KEY}} --inventory {{.ANSIBLE_INVENTORY}} + - >- + ansible-playbook ansible/operation-rolling-restart.yml + --private-key {{.PRIVATE_KEY}} + --inventory {{.ANSIBLE_INVENTORY}} + {{if .SASL_PASSWORD}}--extra-vars kafka_enable_authorization=true --extra-vars sasl_user={{.SASL_USER}} --extra-vars sasl_password={{.SASL_PASSWORD}}{{end}} + + users:apply: + desc: >- + Reconcile SASL users and ACLs via the user_management role. Desired state is passed + as JSON in SASL_USERS_JSON / SASL_ACLS_JSON; admin creds via SASL_USER/SASL_PASSWORD. + deps: + - :ansible:prereqs + - :ensure-logs-dir + cmds: + - >- + ansible-playbook ansible/operation-user-management.yml + --private-key {{.PRIVATE_KEY}} + --inventory {{.ANSIBLE_INVENTORY}} + --extra-vars sasl_user={{.SASL_USER}} + --extra-vars sasl_password={{.SASL_PASSWORD}} + --extra-vars '{{.SASL_USERS_JSON}}' + --extra-vars '{{.SASL_ACLS_JSON}}' license:apply: desc: Apply Redpanda license to the cluster diff --git a/.tasks/testing.yml b/.tasks/testing.yml index 45bebc4a..86eb91ef 100644 --- a/.tasks/testing.yml +++ b/.tasks/testing.yml @@ -92,6 +92,43 @@ tasks: echo "Consuming from topic" {{.RPK_PATH}} topic consume {{.TEST_TOPIC_NAME}} -X brokers=$REDPANDA_BROKERS -v -o :end | grep squirrel || exit 1 + cluster:sasl: + desc: >- + Verify SASL enforcement: superuser rpk works, anonymous rpk is denied, and a + user_management-created user can use exactly the topic its ACL allows. + cmds: + - chmod 775 {{.RPK_PATH}} + - | + REDPANDA_BROKERS=$(sed -n '/^\[redpanda\]/,/^\[/p' "{{.HOSTS_FILE}}" | grep '^[0-9]' | cut -d' ' -f1 | sed 's/$/:9092/' | paste -sd ',' -) + echo "Brokers: $REDPANDA_BROKERS" + SUPER="-X user={{.SASL_USER}} -X pass={{.SASL_PASSWORD}} -X sasl.mechanism=SCRAM-SHA-256" + APP="-X user={{.SASL_TEST_USER}} -X pass={{.SASL_TEST_PASSWORD}} -X sasl.mechanism=SCRAM-SHA-256" + + echo "Superuser can read cluster status" + {{.RPK_PATH}} cluster status -X brokers=$REDPANDA_BROKERS $SUPER -v || exit 1 + + echo "Anonymous topic creation must be denied" + if {{.RPK_PATH}} topic create anon-denied -X brokers=$REDPANDA_BROKERS 2>/dev/null; then + echo "ERROR: anonymous client was allowed to create a topic on a SASL cluster" + exit 1 + fi + + echo "Superuser creates the app topic (or confirms it exists on re-verification)" + {{.RPK_PATH}} topic create {{.SASL_TEST_TOPIC}} -p 3 -X brokers=$REDPANDA_BROKERS $SUPER \ + || {{.RPK_PATH}} topic describe {{.SASL_TEST_TOPIC}} -X brokers=$REDPANDA_BROKERS $SUPER >/dev/null \ + || exit 1 + + echo "ACL-scoped user produces and consumes its topic" + echo sasl-squirrel | {{.RPK_PATH}} topic produce {{.SASL_TEST_TOPIC}} -X brokers=$REDPANDA_BROKERS $APP || exit 1 + {{.RPK_PATH}} topic consume {{.SASL_TEST_TOPIC}} -X brokers=$REDPANDA_BROKERS $APP -o :end | grep sasl-squirrel || exit 1 + + echo "ACL-scoped user must not create unrelated topics" + if {{.RPK_PATH}} topic create off-limits -X brokers=$REDPANDA_BROKERS $APP 2>/dev/null; then + echo "ERROR: ACL-scoped user was allowed to create a topic outside its ACL" + exit 1 + fi + echo "SASL enforcement verified" + cluster:tls: desc: Test Redpanda cluster with TLS cmds: diff --git a/Taskfile.yml b/Taskfile.yml index e7a48347..1e471460 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -158,9 +158,12 @@ env: # Terraform TF_IN_AUTOMATION: "{{.CI}}" - # AWS Credentials - AWS_ACCESS_KEY_ID: '{{.AWS_ACCESS_KEY_ID | default .DA_AWS_ACCESS_KEY_ID}}' - AWS_SECRET_ACCESS_KEY: '{{.AWS_SECRET_ACCESS_KEY | default .DA_AWS_SECRET_ACCESS_KEY}}' + # AWS credentials are deliberately NOT re-exported here: global env + # templates evaluate before dotenv merges, so aliasing lines like + # '{{.AWS_ACCESS_KEY_ID}}' resolve empty for .env users and clobber their + # credentials (leaving terraform to fall back to stale ~/.aws keys). + # Process env (CI) and dotenv (.env) both flow through untouched without + # them. The legacy DA_AWS_* fallback aliases are dropped with this. AWS_DEFAULT_REGION: '{{.AWS_DEFAULT_REGION}}' # Ansible Environment diff --git a/ansible/deploy-monitor-tls.yml b/ansible/deploy-monitor-tls.yml index c99a21af..ee6f8dc0 100644 --- a/ansible/deploy-monitor-tls.yml +++ b/ansible/deploy-monitor-tls.yml @@ -7,11 +7,28 @@ - name: Set node_exporter_arch based on host architecture ansible.builtin.set_fact: rp_arch: "{{ 'amd64' if ansible_architecture == 'x86_64' else 'arm64' }}" + # Prefetched with retries because GitHub resets connections under + # concurrent-lane rate limiting (five separate CI lane failures), and the + # role's own download/latest-lookup hit GitHub per host with no retry. + # Pinning the version also skips the role's 'latest' GitHub API lookup. + - name: Prefetch node_exporter tarball with retries + ansible.builtin.get_url: + url: "https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_pinned_version }}/node_exporter-{{ node_exporter_pinned_version }}.linux-{{ rp_arch }}.tar.gz" + dest: "/tmp/node_exporter-{{ node_exporter_pinned_version }}.linux-{{ rp_arch }}.tar.gz" + mode: "0644" + register: node_exporter_prefetch + until: node_exporter_prefetch is succeeded + retries: 10 + delay: 15 + vars: + node_exporter_pinned_version: "1.12.1" - name: Include geerlingguy nodeexporter ansible.builtin.include_role: name: geerlingguy.node_exporter vars: node_exporter_arch: "{{ rp_arch }}" + node_exporter_version: "1.12.1" + node_exporter_download_url: "/tmp/node_exporter-1.12.1.linux-{{ rp_arch }}.tar.gz" - name: Install certs on monitor hosts: monitor diff --git a/ansible/deploy-monitor.yml b/ansible/deploy-monitor.yml index 52eb635b..be139ed3 100644 --- a/ansible/deploy-monitor.yml +++ b/ansible/deploy-monitor.yml @@ -5,11 +5,28 @@ - name: Set node_exporter_arch based on host architecture ansible.builtin.set_fact: rp_arch: "{{ 'amd64' if ansible_architecture == 'x86_64' else 'arm64' }}" + # Prefetched with retries because GitHub resets connections under + # concurrent-lane rate limiting (five separate CI lane failures), and the + # role's own download/latest-lookup hit GitHub per host with no retry. + # Pinning the version also skips the role's 'latest' GitHub API lookup. + - name: Prefetch node_exporter tarball with retries + ansible.builtin.get_url: + url: "https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_pinned_version }}/node_exporter-{{ node_exporter_pinned_version }}.linux-{{ rp_arch }}.tar.gz" + dest: "/tmp/node_exporter-{{ node_exporter_pinned_version }}.linux-{{ rp_arch }}.tar.gz" + mode: "0644" + register: node_exporter_prefetch + until: node_exporter_prefetch is succeeded + retries: 10 + delay: 15 + vars: + node_exporter_pinned_version: "1.12.1" - name: Include geerlingguy nodeexporter ansible.builtin.include_role: name: geerlingguy.node_exporter vars: node_exporter_arch: "{{ rp_arch }}" + node_exporter_version: "1.12.1" + node_exporter_download_url: "/tmp/node_exporter-1.12.1.linux-{{ rp_arch }}.tar.gz" - name: Install prometheus hosts: monitor diff --git a/ansible/operation-rolling-restart.yml b/ansible/operation-rolling-restart.yml index 63224d69..c8bdd902 100644 --- a/ansible/operation-rolling-restart.yml +++ b/ansible/operation-rolling-restart.yml @@ -5,13 +5,23 @@ serial: 1 vars: rpk_bin: rpk + # On SASL-enabled clusters every rpk invocation below needs credentials; + # rpk rejects empty RPK_USER/RPK_PASS ("Malformed Authorization header"), + # so the environment is only populated when authorization is on. + environment: "{{ {'RPK_USER': sasl_user | default('admin'), 'RPK_PASS': sasl_password | default('')} if (kafka_enable_authorization | default(false) | bool) else {} }}" tasks: + # With serial: 1 this gate runs right after the previous node rejoined; + # the cluster is often still healing, so wait for health like the + # post-restart checks do instead of failing on a one-shot probe. - name: Check cluster health ansible.builtin.shell: | - {{ rpk_bin }} cluster health | grep -i 'healthy:' | tr -d '[:space:]' | awk -F ':' '{print tolower($2)}' + {{ rpk_bin }} cluster health --watch --exit-when-healthy | grep -i 'healthy:' | tr -d '[:space:]' | awk -F ':' '{print tolower($2)}' register: health_check failed_when: "health_check.stdout != 'true'" + retries: 10 + delay: 30 + until: "health_check.stdout == 'true'" changed_when: false - name: Get node ID diff --git a/ansible/operation-user-management.yml b/ansible/operation-user-management.yml new file mode 100644 index 00000000..4fde0f86 --- /dev/null +++ b/ansible/operation-user-management.yml @@ -0,0 +1,19 @@ +# Reconciles SASL users and ACLs via the user_management role. +# The desired users/ACLs are passed by the caller via --extra-vars, e.g.: +# -e sasl_password=... \ +# -e '{"sasl_users":[{"username":"app","password":"s3cret","state":"present"}]}' \ +# -e '{"sasl_acls":[{"principal":"app","operation":["read","write","describe","create"],"resource_type":"topic","resource_name":"apptopic","permission":"allow"}]}' +# Admin credentials flow from sasl_superuser_* (the role follows the broker's +# variables), so the same extra-vars used for provisioning work here. +--- +- name: Manage SASL users and ACLs + hosts: redpanda + run_once: true + vars: + kafka_enable_authorization: true + sasl_superuser_username: "{{ sasl_user | default('admin') }}" + sasl_superuser_password: "{{ sasl_password | default('ci-only-password') }}" + tasks: + - name: Reconcile users, roles and ACLs + ansible.builtin.include_role: + name: redpanda.cluster.user_management diff --git a/ansible/provision-cluster-sasl.yml b/ansible/provision-cluster-sasl.yml new file mode 100644 index 00000000..f06b72fa --- /dev/null +++ b/ansible/provision-cluster-sasl.yml @@ -0,0 +1,25 @@ +# Provisions a Redpanda cluster with SASL authorization enabled. +# variables +# advertise_public_ips : advertise public ips so the CI runner can reach the cluster +# sasl_superuser_username / sasl_superuser_password : bootstrap superuser; supply +# real values via --extra-vars (the defaults here are CI-only placeholders) +--- +- name: Provision SASL-enabled cluster + hosts: redpanda + vars: + advertise_public_ips: true + redpanda_version: latest + kafka_enable_authorization: true + sasl_superuser_username: "{{ sasl_user | default('admin') }}" + sasl_superuser_password: "{{ sasl_password | default('ci-only-password') }}" + tasks: + - name: Install system prereqs + ansible.builtin.include_role: + name: redpanda.cluster.system_setup + - name: Handle sysctl changes + ansible.builtin.include_role: + name: redpanda.cluster.sysctl_setup + - name: Install and start redpanda + ansible.builtin.include_role: + name: redpanda.cluster.redpanda_broker + when: not skip_node | default(false) | bool diff --git a/requirements.yml b/requirements.yml index ed394291..80b681f8 100644 --- a/requirements.yml +++ b/requirements.yml @@ -1,7 +1,11 @@ collections: - name: community.general - - name: redpanda.cluster - type: galaxy + # TEMP (DO NOT MERGE): points at the collection-hardening branch so CI can + # validate this harness against the 0.13.0 candidate. Revert to the pinned + # Galaxy release before merging. + - name: https://github.com/redpanda-data/redpanda-ansible-collection.git + type: git + version: collection-hardening - name: ansible.posix - name: grafana.grafana version: 5.6.0