From 5f4298f54b12e84c1940c98740b6a0aad5b28635 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 23 Jun 2025 18:32:35 -0500 Subject: [PATCH 01/97] Update original images with new container (#1052) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .original-images.json | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.original-images.json b/.original-images.json index 876901b0e..51a567792 100644 --- a/.original-images.json +++ b/.original-images.json @@ -1,4 +1,5 @@ [ + "cr.fluentbit.io/fluent/fluent-bit", "docker.io/docker:17.07.0", "docker.io/kolla/centos-source-openvswitch-vswitchd:master", "docker.io/kolla/ubuntu-source-nova-compute-ironic:master", @@ -11,13 +12,15 @@ "docker.io/openstackhelm/designate:2024.1-ubuntu_jammy", "docker.io/openstackhelm/glance:2024.1-ubuntu_jammy", "docker.io/openstackhelm/horizon:2023.1-ubuntu_jammy", + "docker.io/openstackhelm/ironic:2024.1-ubuntu_jammy", "docker.io/openstackhelm/magnum:2024.1-ubuntu_jammy", - "docker.io/openstackhelm/masakari:2024.1-ubuntu_jammy", "docker.io/openstackhelm/masakari-monitors:2024.1-ubuntu_jammy", + "docker.io/openstackhelm/masakari:2024.1-ubuntu_jammy", "docker.io/openstackhelm/neutron:2024.1-ubuntu_jammy", "docker.io/openstackhelm/osh-selenium:latest-ubuntu_jammy", "docker.io/openstackhelm/ospurge:latest", "docker.io/openstackhelm/placement:2024.1-ubuntu_jammy", + "docker.io/pbandark/barbican-exporter", "docker.io/rabbitmq:3.13-management", "docker.io/wrouesnel/postgres_exporter:v0.4.6", "docker.io/xrally/xrally-openstack:2.0.0", @@ -30,15 +33,13 @@ "ghcr.io/rackerlabs/genestack/neutron-oslodb:2024.1-ubuntu_jammy-1738626982", "ghcr.io/rackerlabs/genestack/neutron-oslodb:2024.1-ubuntu_jammy-1739651767", "ghcr.io/rackerlabs/genestack/neutron-oslodb:2024.1-ubuntu_jammy-1742943886", + "ghcr.io/rackerlabs/genestack/neutron-oslodb:2024.1-ubuntu_jammy-1750715539", "ghcr.io/rackerlabs/genestack/nova-efi:2024.1-ubuntu_jammy-1737928811", "ghcr.io/rackerlabs/genestack/octavia-ovn:2024.1-ubuntu_jammy-1737651745", - "ghcr.io/rackerlabs/keystone-rxt:2024.1-ubuntu_jammy-1747958291", "ghcr.io/rackerlabs/keystone-rxt/shibd:1747958286", + "ghcr.io/rackerlabs/keystone-rxt:2024.1-ubuntu_jammy-1747958291", "ghcr.io/rackerlabs/skyline-rxt:master-ubuntu_jammy-1748595671", - "docker.io/openstackhelm/ironic:2024.1-ubuntu_jammy", "ghcr.io/vexxhost/netoffload:v1.0.1", "quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_jammy", - "quay.io/airshipit/porthole-postgresql-utility:latest-ubuntu_bionic", - "cr.fluentbit.io/fluent/fluent-bit", - "docker.io/pbandark/barbican-exporter" + "quay.io/airshipit/porthole-postgresql-utility:latest-ubuntu_bionic" ] From 4a331c619db6e5c1ea7a1c108c6062e9e1728979 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 23 Jun 2025 19:08:24 -0500 Subject: [PATCH 02/97] Update original images with new container (#1053) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .original-images.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.original-images.json b/.original-images.json index 51a567792..0b48966d8 100644 --- a/.original-images.json +++ b/.original-images.json @@ -34,6 +34,7 @@ "ghcr.io/rackerlabs/genestack/neutron-oslodb:2024.1-ubuntu_jammy-1739651767", "ghcr.io/rackerlabs/genestack/neutron-oslodb:2024.1-ubuntu_jammy-1742943886", "ghcr.io/rackerlabs/genestack/neutron-oslodb:2024.1-ubuntu_jammy-1750715539", + "ghcr.io/rackerlabs/genestack/neutron-oslodb:2024.1-ubuntu_jammy-1750723622", "ghcr.io/rackerlabs/genestack/nova-efi:2024.1-ubuntu_jammy-1737928811", "ghcr.io/rackerlabs/genestack/octavia-ovn:2024.1-ubuntu_jammy-1737651745", "ghcr.io/rackerlabs/keystone-rxt/shibd:1747958286", From fc27bed2a9d45291bc0e45bb7edebc61d424b730 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Wed, 25 Jun 2025 13:52:45 -0500 Subject: [PATCH 03/97] chore: fix heat liveness probe (#1056) Signed-off-by: Kevin Carter --- base-kustomize/heat/base/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base-kustomize/heat/base/kustomization.yaml b/base-kustomize/heat/base/kustomization.yaml index 8da8bc0bf..c686f86e4 100644 --- a/base-kustomize/heat/base/kustomization.yaml +++ b/base-kustomize/heat/base/kustomization.yaml @@ -37,7 +37,7 @@ patches: failureThreshold: 3 httpGet: path: / - port: 8004 + port: 8000 scheme: HTTP initialDelaySeconds: 10 periodSeconds: 60 From 2623a681419b59b0baf9873e509808ed02292671 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 25 Jun 2025 21:11:07 -0500 Subject: [PATCH 04/97] Update original images with new container (#1055) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .original-images.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.original-images.json b/.original-images.json index 0b48966d8..1eeec5b35 100644 --- a/.original-images.json +++ b/.original-images.json @@ -35,6 +35,7 @@ "ghcr.io/rackerlabs/genestack/neutron-oslodb:2024.1-ubuntu_jammy-1742943886", "ghcr.io/rackerlabs/genestack/neutron-oslodb:2024.1-ubuntu_jammy-1750715539", "ghcr.io/rackerlabs/genestack/neutron-oslodb:2024.1-ubuntu_jammy-1750723622", + "ghcr.io/rackerlabs/genestack/neutron-oslodb:2024.1-ubuntu_jammy-1750797661", "ghcr.io/rackerlabs/genestack/nova-efi:2024.1-ubuntu_jammy-1737928811", "ghcr.io/rackerlabs/genestack/octavia-ovn:2024.1-ubuntu_jammy-1737651745", "ghcr.io/rackerlabs/keystone-rxt/shibd:1747958286", From 91a51a3cf1bb4dc34a34246db50b305d0765564a Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Thu, 26 Jun 2025 08:13:09 -0500 Subject: [PATCH 05/97] chore: update nova-efi to 2025.1 (#1057) Signed-off-by: Kevin Carter --- .github/workflows/release-nova-oslodb.yaml | 119 --------------------- .github/workflows/release-nova-uefi.yml | 6 +- Containerfiles/Nova-oslo_db-Containerfile | 12 --- Containerfiles/NovaEFI-Containerfile | 4 +- 4 files changed, 4 insertions(+), 137 deletions(-) delete mode 100644 .github/workflows/release-nova-oslodb.yaml delete mode 100644 Containerfiles/Nova-oslo_db-Containerfile diff --git a/.github/workflows/release-nova-oslodb.yaml b/.github/workflows/release-nova-oslodb.yaml deleted file mode 100644 index d8c54a984..000000000 --- a/.github/workflows/release-nova-oslodb.yaml +++ /dev/null @@ -1,119 +0,0 @@ -# -name: Create and publish a Nova oslodb patched image - -# Configures this workflow to run every time a change is pushed to the branch called `release`. -on: - workflow_dispatch: - inputs: - imageTag: - description: 'Set tag for the image' - required: true - default: 'master-ubuntu_jammy' - type: choice - options: - - master-ubuntu_jammy - - 2023.1-ubuntu_jammy - - 2023.2-ubuntu_jammy - - 2024.1-ubuntu_jammy - pluginTag: - description: 'Set release used for the build environment' - required: true - default: 'master' - type: choice - options: - - "master" - - "2023.1" - - "2023.2" - - "2024.1" - -# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - build-and-push-image: - outputs: - MY_DATE: ${{ steps.mydate.outputs.MY_DATE }} - MY_CONTAINER: ${{ steps.mycontainer.outputs.MY_CONTAINER }} - runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. - # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. - # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - - name: Dynamically set MY_DATE environment variable - run: echo "MY_DATE=$(date +%s)" >> $GITHUB_ENV - - name: Build and push Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: Containerfiles/Nova-oslo_db-Containerfile - push: true - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/nova-oslodb:${{ github.event.inputs.imageTag }} - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/nova-oslodb:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - VERSION=${{ github.event.inputs.imageTag }} - PLUGIN_VERSION=${{ github.event.inputs.pluginTag }} - - name: Dynamically set MY_CONTAINER output option - id: mycontainer - run: echo "MY_CONTAINER=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/nova-oslodb:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }}" >> $GITHUB_OUTPUT - - name: Dynamically set MY_DATE output option - id: mydate - run: echo "MY_DATE=${{ env.MY_DATE }}" >> $GITHUB_OUTPUT - - - change-original-images: - runs-on: ubuntu-latest - needs: [build-and-push-image] - permissions: - contents: write - pull-requests: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Dynamically update the original images file - run: jq '. + ["${{ needs.build-and-push-image.outputs.MY_CONTAINER }}"] | sort' .original-images.json | tee .original-images.json.new - - name: Rewrite original images file - run: mv .original-images.json.new .original-images.json - - name: Create Pull Request - id: cpr - uses: peter-evans/create-pull-request@v7 - with: - commit-message: Update original images with new container - committer: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> - author: ${{ github.actor }} <${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com> - signoff: false - branch: ${{ needs.build-and-push-image.outputs.MY_DATE }} - sign-commits: true - delete-branch: true - title: 'chore: Update original images' - body: | - Update container image - - Updated original image file with container ${{needs.build-and-push-image.outputs.MY_CONTAINER}} - change request Auto-generated - labels: | - container images - automated pr - draft: false diff --git a/.github/workflows/release-nova-uefi.yml b/.github/workflows/release-nova-uefi.yml index 8a6b88397..388f5e1ac 100644 --- a/.github/workflows/release-nova-uefi.yml +++ b/.github/workflows/release-nova-uefi.yml @@ -12,8 +12,7 @@ on: type: choice options: - master-ubuntu_jammy - - 2023.1-ubuntu_jammy - - 2023.2-ubuntu_jammy + - 2025.1-ubuntu_jammy - 2024.1-ubuntu_jammy pluginTag: description: 'Set release used for the build environment' @@ -22,8 +21,7 @@ on: type: choice options: - "master" - - "2023.1" - - "2023.2" + - "2025.1" - "2024.1" # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. diff --git a/Containerfiles/Nova-oslo_db-Containerfile b/Containerfiles/Nova-oslo_db-Containerfile deleted file mode 100644 index a335e8fa7..000000000 --- a/Containerfiles/Nova-oslo_db-Containerfile +++ /dev/null @@ -1,12 +0,0 @@ -# Patch oslo_db to help with deadlocks -ARG VERSION=master-ubuntu_jammy -FROM openstackhelm/nova:$VERSION as build -ARG PLUGIN_VERSION=master -RUN apt update && apt install -y git -RUN export ORIG_PLUGIN_VERSION="${PLUGIN_VERSION}"; \ -if [ "${PLUGIN_VERSION}" != 'master' ]; then export PLUGIN_VERSION=stable/${PLUGIN_VERSION}; fi; \ -. /var/lib/openstack/bin/activate; \ -/var/lib/openstack/bin/pip install git+https://github.com/openstack/oslo.db@${PLUGIN_VERSION}#egg=oslo_db - -FROM openstackhelm/nova:${VERSION} -COPY --from=build /var/lib/openstack/. /var/lib/openstack/ diff --git a/Containerfiles/NovaEFI-Containerfile b/Containerfiles/NovaEFI-Containerfile index 0c7ab5e2d..5a82d2803 100644 --- a/Containerfiles/NovaEFI-Containerfile +++ b/Containerfiles/NovaEFI-Containerfile @@ -1,5 +1,5 @@ ARG VERSION=master-ubuntu_jammy -FROM openstackhelm/nova:$VERSION as build +FROM quay.io/airshipit/nova:$VERSION as build ARG PLUGIN_VERSION=master RUN apt update && apt install -y git RUN export ORIG_PLUGIN_VERSION="${PLUGIN_VERSION}"; \ @@ -8,7 +8,7 @@ if [ "${PLUGIN_VERSION}" != 'master' ]; then export PLUGIN_VERSION=stable/${PLUG /var/lib/openstack/bin/pip install git+https://github.com/openstack/oslo.db@${PLUGIN_VERSION}#egg=oslo_db && \ /var/lib/openstack/bin/pip install python-barbicanclient -FROM openstackhelm/nova:${VERSION} +FROM quay.io/airshipit/nova:${VERSION} COPY --from=build /var/lib/openstack/. /var/lib/openstack/ # Packages for the following features: # - Nova: EFI From 7a1a3b94c06d69bc805cf1c224daad4bc015aaa0 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Thu, 26 Jun 2025 10:54:37 -0500 Subject: [PATCH 06/97] chore: tune kube-ovn (#1059) Updating our defaults to use the new tuned options we're running in production. Signed-off-by: Kevin Carter Co-authored-by: Jake Briggs --- .../kube-ovn/kube-ovn-helm-overrides.yaml | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/base-helm-configs/kube-ovn/kube-ovn-helm-overrides.yaml b/base-helm-configs/kube-ovn/kube-ovn-helm-overrides.yaml index 961c044ff..6406adb72 100644 --- a/base-helm-configs/kube-ovn/kube-ovn-helm-overrides.yaml +++ b/base-helm-configs/kube-ovn/kube-ovn-helm-overrides.yaml @@ -45,23 +45,23 @@ networking: DEFAULT_SUBNET: "ovn-default" DEFAULT_VPC: "ovn-cluster" NODE_SUBNET: "join" #mesh network - ENABLE_ECMP: false + ENABLE_ECMP: true ENABLE_METRICS: true # comma-separated string of nodelocal DNS ip addresses NODE_LOCAL_DNS_IP: "" - PROBE_INTERVAL: 180000 - OVN_NORTHD_PROBE_INTERVAL: 5000 - OVN_LEADER_PROBE_INTERVAL: 5 - OVN_REMOTE_PROBE_INTERVAL: 10000 + PROBE_INTERVAL: 60000 + OVN_NORTHD_PROBE_INTERVAL: 15000 + OVN_LEADER_PROBE_INTERVAL: 15 + OVN_REMOTE_PROBE_INTERVAL: 30000 OVN_REMOTE_OPENFLOW_INTERVAL: 180 - OVN_NORTHD_N_THREADS: 1 - ENABLE_COMPACT: false + OVN_NORTHD_N_THREADS: 4 # Number of threads for ovn-northd, default is 4 production environments could set it to a higher value. + ENABLE_COMPACT: true func: ENABLE_LB: true ENABLE_NP: true ENABLE_EXTERNAL_VPC: true - HW_OFFLOAD: false + HW_OFFLOAD: false # Enable hardware offload, if supported by the underlying network hardware. ENABLE_LB_SVC: false ENABLE_KEEP_VM_IP: true LS_DNAT_MOD_DL_DST: true @@ -77,8 +77,8 @@ func: ENABLE_OVN_IPSEC: false ENABLE_ANP: false SET_VXLAN_TX_OFF: false - OVSDB_CON_TIMEOUT: 3 - OVSDB_INACTIVITY_TIMEOUT: 10 + OVSDB_CON_TIMEOUT: 5 + OVSDB_INACTIVITY_TIMEOUT: 30 ENABLE_LIVE_MIGRATION_OPTIMIZE: true ipv4: @@ -91,8 +91,8 @@ ipv4: performance: GC_INTERVAL: 0 - INSPECT_INTERVAL: 20 - OVS_VSCTL_CONCURRENCY: 100 + INSPECT_INTERVAL: 300 + OVS_VSCTL_CONCURRENCY: 150 debug: ENABLE_MIRROR: false From bce6163aaf7abfa5ad9a063a24d180e1c53e3837 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 26 Jun 2025 10:54:52 -0500 Subject: [PATCH 07/97] Update original images with new container (#1058) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .original-images.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.original-images.json b/.original-images.json index 1eeec5b35..ae10dc7e7 100644 --- a/.original-images.json +++ b/.original-images.json @@ -37,6 +37,7 @@ "ghcr.io/rackerlabs/genestack/neutron-oslodb:2024.1-ubuntu_jammy-1750723622", "ghcr.io/rackerlabs/genestack/neutron-oslodb:2024.1-ubuntu_jammy-1750797661", "ghcr.io/rackerlabs/genestack/nova-efi:2024.1-ubuntu_jammy-1737928811", + "ghcr.io/rackerlabs/genestack/nova-efi:2025.1-ubuntu_jammy-1750943616", "ghcr.io/rackerlabs/genestack/octavia-ovn:2024.1-ubuntu_jammy-1737651745", "ghcr.io/rackerlabs/keystone-rxt/shibd:1747958286", "ghcr.io/rackerlabs/keystone-rxt:2024.1-ubuntu_jammy-1747958291", From 990ab034244f1dab46a20b5d9fa68c2a57477b4e Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Sat, 28 Jun 2025 21:56:36 -0500 Subject: [PATCH 08/97] fix: disable ceph by default (#1060) Fixes broken builds from master where ceph was attempting to be included by default. Signed-off-by: Kevin Carter --- base-helm-configs/nova/nova-helm-overrides.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/base-helm-configs/nova/nova-helm-overrides.yaml b/base-helm-configs/nova/nova-helm-overrides.yaml index eb1ef1fbe..94c29136d 100644 --- a/base-helm-configs/nova/nova-helm-overrides.yaml +++ b/base-helm-configs/nova/nova-helm-overrides.yaml @@ -88,6 +88,8 @@ conf: handlers: - stdout level: INFO + ceph: + enabled: false nova: DEFAULT: block_device_allocate_retries: 180 From 2e05f3895bcc637fedc476a55b840cf324051954 Mon Sep 17 00:00:00 2001 From: Bjoern Teipel Date: Tue, 1 Jul 2025 10:59:38 -0500 Subject: [PATCH 09/97] Update metallb documentation (#1022) - Fix typos - Add instructions how to re-IP the pools --- docs/infrastructure-envoy-gateway-api.md | 2 +- docs/infrastructure-metallb.md | 28 ++++++++++++++++++- .../metallb/metallb-openstack-service-lb.yml | 2 ++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/docs/infrastructure-envoy-gateway-api.md b/docs/infrastructure-envoy-gateway-api.md index 415ff5b5f..60617dbcb 100644 --- a/docs/infrastructure-envoy-gateway-api.md +++ b/docs/infrastructure-envoy-gateway-api.md @@ -60,5 +60,5 @@ kubectl -n envoy-gateway get gateways.gateway.networking.k8s.io flex-gateway If you encounter any issues, check the logs of the `envoy-gateway` deployment. ``` shell -kubectl logs -n envoy-gateway-system deployment/envoy-gateway +kubectl logs -n envoygateway-system deployment/envoy-gateway ``` diff --git a/docs/infrastructure-metallb.md b/docs/infrastructure-metallb.md index 9d0131af2..707b5e82b 100644 --- a/docs/infrastructure-metallb.md +++ b/docs/infrastructure-metallb.md @@ -4,6 +4,11 @@ The MetalLb loadbalancer can be setup by editing the following file `metallb-ope your "external" VIP(s) to the loadbalancer so that they can be used within services. These IP addresses are unique and will need to be customized to meet the needs of your environment. +!!! tip + + When L2Advertisement is used, you should use a CIDR that is not overlapping with any local interface CIDR. + This also enables later migration to BGP advertisement. + ## Create the MetalLB namespace ``` shell @@ -37,8 +42,29 @@ Verify the deployment of MetalLB by checking the pods in the `metallb-system` na kubectl --namespace metallb-system get deployment.apps/metallb-controller ``` -Once MetalLB is operatoinal, apply the metallb service manifest. +Once MetalLB is operatianal, apply the metallb service manifest. ``` shell kubectl apply -f /etc/genestack/manifests/metallb/metallb-openstack-service-lb.yml ``` + +## Re-IP the advertisement pools +In situations where the advertisement pools must be changed, the following disruptive procedure can be used: + +Update existing metallb configuration: + +```shell +kubectl -n metallb-system delete IPAddressPool/primary +kubectl -n metallb-system delete IPAddressPool/gateway-api-external +kubectl apply -f /etc/genestack/manifests/metallb/metallb-openstack-service-lb.yml +``` +``` + +Restart the metallb controller: + +```shell +kubectl rollout restart deployment metallb-controller -n metallb-system +``` + +Once the metallb controller restarts it'll begin to reip the external service IP associations which typically +requires DNS entry updates. This change including the DNS refresh (TTL) time will be disruptive. diff --git a/manifests/metallb/metallb-openstack-service-lb.yml b/manifests/metallb/metallb-openstack-service-lb.yml index 733e5d20c..d31e010a8 100644 --- a/manifests/metallb/metallb-openstack-service-lb.yml +++ b/manifests/metallb/metallb-openstack-service-lb.yml @@ -8,6 +8,7 @@ spec: addresses: - 10.74.8.99/32 # This is assumed to be the public LB vip address autoAssign: false + avoidBuggyIPs: true --- apiVersion: metallb.io/v1beta1 kind: L2Advertisement @@ -32,6 +33,7 @@ spec: addresses: - 10.234.0.0/24 autoAssign: false + avoidBuggyIPs: true --- apiVersion: metallb.io/v1beta1 kind: L2Advertisement From 55dedc9b4217967e1efc2ec3182e67758cc056d4 Mon Sep 17 00:00:00 2001 From: Bjoern Teipel Date: Tue, 1 Jul 2025 11:03:23 -0500 Subject: [PATCH 10/97] Update for secret creation and custom region (#1023) --- bin/create-secrets.sh | 2 +- docs/infrastructure-namespace.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/create-secrets.sh b/bin/create-secrets.sh index 01d2fbc30..dae05fb2b 100755 --- a/bin/create-secrets.sh +++ b/bin/create-secrets.sh @@ -2,7 +2,7 @@ # shellcheck disable=SC2086 usage() { - echo "Usage: $0 [--region default: RegionOne]" + echo "Usage: $0 [--region " exit 1 } diff --git a/docs/infrastructure-namespace.md b/docs/infrastructure-namespace.md index aa3046cbd..31831c66b 100644 --- a/docs/infrastructure-namespace.md +++ b/docs/infrastructure-namespace.md @@ -11,9 +11,9 @@ Then you can create all needed secrets by running the create-secrets.sh command !!! tip "Optional --region param" Note that the `create-secrets.sh` script by default creates a secret - with a default region of RegionOne. This can be overridden with the + with a default region of *RegionOne*. This can be overridden with the `--region` parameter to specify your custom region name in Keystone. - > Usage: ./create-secrets.sh [--region default: RegionOne] + > Usage: ./create-secrets.sh [--region ] ``` shell /opt/genestack/bin/create-secrets.sh From f3d6405aa2efc871215044f5093a2d3efe3eeff1 Mon Sep 17 00:00:00 2001 From: Pratik Bandarkar Date: Wed, 2 Jul 2025 13:06:59 +0100 Subject: [PATCH 11/97] feat(monitoring): add script to import grafana dashboards --- docs/import-grafana-dashboard.md | 38 +++++++++ docs/monitoring-info.md | 2 +- scripts/import-grafana-dashboard.py | 124 ++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 docs/import-grafana-dashboard.md create mode 100644 scripts/import-grafana-dashboard.py diff --git a/docs/import-grafana-dashboard.md b/docs/import-grafana-dashboard.md new file mode 100644 index 000000000..6c294c346 --- /dev/null +++ b/docs/import-grafana-dashboard.md @@ -0,0 +1,38 @@ +# Grafana Dashboard Import Script + +This script helps you **import Grafana dashboards** from a local directory that contains JSON files. Each file must contain a valid Grafana dashboard definition. + + +## Prerequisites +- A running [monitoring stack](https://github.com/rackerlabs/genestack/blob/main/docs/monitoring-info.md) +- Dashboards exported as valid [JSON files](https://github.com/rackerlabs/genestack/tree/main/etc/grafana-dashboards) + +## Environment Variables +Set the following environment variables before running the script: + +| Variable | Required | Description | Default | +|-------------------|----------|-------------------------------------------------------|---------------------------------| +| `GRAFANA_PASSWORD`| True | Grafana admin password | None. | +| `GRAFANA_USERNAME`| False | Grafana admin username | `admin` | +| `GRAFANA_URL` | False | URL of your Grafana instance | `http://grafana.grafana.svc.cluster.local:80` | + + +## Usage +```bash +# python import_dashboard.py -h +usage: import_dashboard.py [-h] -d DIR [-ds DATASOURCE] + +Import Grafana dashboards from a local directory. + +options: + -h, --help show this help message and exit + -d DIR, --dir DIR Path to directory containing dashboard JSON files + -ds DATASOURCE, --datasource DATASOURCE + Name of the Prometheus datasource. Default: "Prometheus" + +export GRAFANA_USERNAME=admin +export GRAFANA_URL=https://grafana.sjc3.rackspacecloud.com +export GRAFANA_PASSWORD=your_admin_password + +python import_dashboards.py --dir /opt/genestack/etc/grafana-dashboards/ --datasource Prometheus +``` diff --git a/docs/monitoring-info.md b/docs/monitoring-info.md index c2b440e96..bee1a6035 100644 --- a/docs/monitoring-info.md +++ b/docs/monitoring-info.md @@ -177,7 +177,7 @@ You can manually add additional datasources by following the [add datasource](ht More information about the primary datasources can be found in the [Prometheus datasource](https://grafana.com/docs/grafana/latest/datasources/prometheus/) and [Loki datasource](https://grafana.com/docs/grafana/latest/datasources/loki/) documentation. As things stand now, the Grafana deployment does not deploy dashboards as part of the default deployment instructions. However, there are dashboards available found in the [etc directory](https://github.com/rackerlabs/genestack/tree/main/etc/grafana-dashboards) of the Genestack repo that can be installed manually by importing them into Grafana. -View the [importing dashboards](https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/import-dashboards/) documentation for more information. +View the [importing dashboards](https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/import-dashboards/) documentation for more information. You can also use [import-grafana-dashboard.py](https://github.com/rackerlabs/genestack/tree/main/scripts/import-grafana-dashboard.py) script for the same. The dashboards available cover just about every exporter/metric noted here and then some. Some of the dashboards may not be complete or may not provide the desired view. Please feel free to adjust them as needed and submit a PR to [Genestack repo](https://github.com/rackerlabs/genestack) if they may help others! ## Next Steps diff --git a/scripts/import-grafana-dashboard.py b/scripts/import-grafana-dashboard.py new file mode 100644 index 000000000..24f550876 --- /dev/null +++ b/scripts/import-grafana-dashboard.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +import requests +from requests.auth import HTTPBasicAuth +import json +import glob +import os +import sys +import argparse + + +def import_dashboards( + grafana_url, grafana_user, grafana_password, dashboard_dir, prometheus_datasource +): + headers = {"Content-Type": "application/json", "Accept": "application/json"} + + if not os.path.isdir(dashboard_dir): + print(f"Error: '{dashboard_dir}' is not a valid directory.") + sys.exit(1) + + os.chdir(dashboard_dir) + + # Create folders first + folder_cache = {} + + for file in glob.glob("*.json"): + with open(file, "r") as f: + dashboard_json = json.load(f) + folder_title = dashboard_json.get("folderTitle", "General") + + if folder_title != "General" and folder_title not in folder_cache: + folder_response = requests.post( + f"{grafana_url}/api/folders", + headers=headers, + json={"title": folder_title}, + auth=HTTPBasicAuth(grafana_user, grafana_password), + ) + if folder_response.ok: + folder_cache[folder_title] = folder_response.json()["id"] + else: + print( + f"Failed to create folder '{folder_title}': {folder_response.status_code} {folder_response.text}" + ) + continue + + # Import dashboards + for file in glob.glob("*.json"): + with open(file, "r") as f: + dashboard_json = json.load(f) + dashboard_json.pop("id", None) + folder_title = dashboard_json.get("folderTitle", "General") + import_json = { + "dashboard": dashboard_json, + "overwrite": True, + "folderId": folder_cache.get(folder_title, 0), + "inputs": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "pluginId": "prometheus", + "value": prometheus_datasource, + } + ], + } + response = requests.post( + f"{grafana_url}/api/dashboards/import", + headers=headers, + json=import_json, + auth=HTTPBasicAuth(grafana_user, grafana_password), + ) + if response.ok: + print(f"Imported {file}: {response.status_code}") + else: + print( + f"Failed to import {file}: {response.status_code} - {response.text}" + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Import Grafana dashboards from a local directory." + ) + parser.add_argument( + "-d", + "--dir", + required=True, + help="Path to directory containing dashboard JSON files", + ) + parser.add_argument( + "-ds", + "--datasource", + required=False, + help='Name of the Prometheus datasource. Default: "Prometheus"', + default="Prometheus", + ) + args = parser.parse_args() + required_vars = ["GRAFANA_PASSWORD"] + optional_vars = { + "GRAFANA_USERNAME": "admin", + "GRAFANA_URL": "http://grafana.grafana.svc.cluster.local:80", + } + + missing = [var for var in required_vars if var not in os.environ] + if missing: + print(f"Error: Missing required environment variable(s): {', '.join(missing)}") + sys.exit(1) + + for var, default in optional_vars.items(): + if var not in os.environ: + print( + f"Info: Environment variable '{var}' not set. Using default: '{default}'" + ) + os.environ[var] = default + + grafana_username = os.environ.get("GRAFANA_USERNAME") + grafana_password = os.environ.get("GRAFANA_PASSWORD") + grafana_url = os.environ.get("GRAFANA_URL") + + import_dashboards( + grafana_url, grafana_username, grafana_password, args.dir, args.datasource + ) + + +if __name__ == "__main__": + main() From 1cdc0fecf05d05bc5a8eaf4d43e64d37b15dca5f Mon Sep 17 00:00:00 2001 From: Daniel With Date: Mon, 7 Jul 2025 11:38:05 -0500 Subject: [PATCH 12/97] Fix: md_info_detail.sh Removes trailing whitespace in state and Status fields. Previously, alerts may have been triggered or not have been triggered because the pattern in the Prometheus rule might have taken the whitespace into account. This removes that possibility until this bash script can be re-written in Python instrumentation for Prometheus. --- ansible/playbooks/extra/custom_exporters/md_info_detail.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/extra/custom_exporters/md_info_detail.sh b/ansible/playbooks/extra/custom_exporters/md_info_detail.sh index 5c21b4c5a..c85eb0461 100644 --- a/ansible/playbooks/extra/custom_exporters/md_info_detail.sh +++ b/ansible/playbooks/extra/custom_exporters/md_info_detail.sh @@ -79,7 +79,7 @@ for MD_DEVICE in /dev/md*; do if echo "$line" | grep -E -qv "^/|Array Size|Used Dev Size|Events|Update Time|Check Status|Rebuild Status" ; then echo -n ", " MDADM_DETAIL_KEY=$(echo "$line" | cut -d ":" -f 1 | tr -cd '[a-zA-Z0-9]._-') - MDADM_DETAIL_VALUE=$(echo "$line" | cut -d ":" -f 2- | sed 's:^ ::') + MDADM_DETAIL_VALUE=$(echo "$line" | cut -d ":" -f 2- | sed 's:^ ::' | sed 's: $::') echo -n "${MDADM_DETAIL_KEY}=\"${MDADM_DETAIL_VALUE}\"" fi fi From cffc4b7396cca92d36bc3521afbd5a583377463d Mon Sep 17 00:00:00 2001 From: phillip-toohill Date: Thu, 10 Jul 2025 12:00:25 -0500 Subject: [PATCH 13/97] feature: Grafana alert for catching Neutron IPAM Errors while attempting to create new VM *This is an alert when Neutron logs an error while attempting to assign an already assigned IP address to a VM. Log entries on the nova log side look like below: ``` 2025-03-20 03:19:13.009 9 ERROR oslo_db.api pymysql.err.IntegrityError: (1062, "Duplicate entry '-uuid-...' for key 'PRIMARY'") ``` Authored by: anande, Fixed and added into main by: the2hill --- .../grafana/grafana-helm-overrides.yaml | 83 +++++++++++++++++-- 1 file changed, 75 insertions(+), 8 deletions(-) diff --git a/base-helm-configs/grafana/grafana-helm-overrides.yaml b/base-helm-configs/grafana/grafana-helm-overrides.yaml index df672ec91..6662ba0fa 100644 --- a/base-helm-configs/grafana/grafana-helm-overrides.yaml +++ b/base-helm-configs/grafana/grafana-helm-overrides.yaml @@ -1,16 +1,26 @@ --- + +# Set the custom_host variable to the desired hostname for Grafana +# This is used to set the domain and root_url in the grafana.ini file +# and the value of the custom_host variable must be a valid domain. custom_host: grafana.example.com + persistence: type: pvc enabled: true + storageClassName: general accessModes: - ReadWriteMany + nodeSelector: openstack-control-plane: enabled + ingress: enabled: false + image: - tag: 10.3.3 + tag: "10.3.3" + grafana.ini: paths: data: /var/lib/grafana/ @@ -24,14 +34,15 @@ grafana.ini: grafana_net: url: https://grafana.net server: - domain: "{{ .Values.custom_host }}" - root_url: https://{{ .Values.custom_host }} + domain: "{{ .Values.custom_host }}" # Ref: custom_host variable above + root_url: "https://{{ .Values.custom_host }}" # Ref: custom_host variable above database: type: mysql host: mariadb-cluster.grafana.svc:3306 user: $__file{/etc/secrets/grafana-db/username} password: $__file{/etc/secrets/grafana-db/password} name: grafana + datasources: datasources.yaml: apiversion: 1 @@ -46,6 +57,7 @@ datasources: access: proxy url: http://loki-gateway.{{ $.Release.Namespace }}.svc.cluster.local:80 editable: false + alerting: rules.yaml: groups: @@ -106,13 +118,66 @@ alerting: notifications: - uid: prom-alertmanager-notification annotations: - description: Checks app=ovs (ovs-ovn) pod logs for lines with string + description: >- + Checks app=ovs (ovs-ovn) pod logs for lines with string 'binding|INFO|cr-lrp' - summary: This alerts on rapid port claims for cr-lrp ports on OVN gateway nodes, - which overloads the OVN south database and interferes with the - function of the affected ports. + summary: >- + This alerts on rapid port claims for cr-lrp ports on OVN + gateway nodes, which overloads the OVN south database and + interferes with the function of the affected ports. labels: {} isPaused: false + # Generated UUID using 'uuidgen' + - uid: c14dd8fd-54ec-4e15-9813-e02cc3269899 + title: Neutron IPAM Duplicate Entry Error + condition: B + data: + - refId: A + queryType: instant + relativeTimeRange: + from: 60 + to: 0 + # Using same loki datasource as rule#ba943125-33ca-4e4e-85f8-13359a8e4d65 + datasourceUid: P8E80F9AEF21F6940 + model: + expr: rate({app="fluentbit"} |= `Duplicate entry|ERROR` [1m]) + queryType: instant + refId: A + - refId: B + relativeTimeRange: + # Past 60 seconds (can be adjusted further) + from: 60 + # 0 denotes till current time + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 1 + - 0 + type: gt + operator: + type: and + reducer: + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: A + refId: B + type: threshold + noDataState: OK + execErrState: Error + for: 0s + notifications: + - uid: prom-alertmanager-notification + annotations: + summary: > + Checks for log lines containing 'Duplicate entry|ERROR' in nova logs. + isPaused: false contactpoints.yaml: secret: apiVersion: 1 @@ -134,11 +199,13 @@ alerting: group_wait: 1s group_interval: 1s repeat_interval: 1s + plugins: - camptocamp-prometheus-alertmanager-datasource + extraSecretMounts: - name: grafana-db-secret-mount secretName: grafana-db - defaultMode: 440 + defaultMode: 0440 mountPath: /etc/secrets/grafana-db readOnly: true From d8eb22c7d4439a31d0a84406efe520289a933941 Mon Sep 17 00:00:00 2001 From: Jitendra Date: Thu, 3 Jul 2025 22:39:39 +0530 Subject: [PATCH 14/97] Docs: Added document for mariadb backup restore with swift tempauth --- docs/maridb-backuprestore-from-tempauth.md | 185 +++++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 docs/maridb-backuprestore-from-tempauth.md diff --git a/docs/maridb-backuprestore-from-tempauth.md b/docs/maridb-backuprestore-from-tempauth.md new file mode 100644 index 000000000..6d53c21ce --- /dev/null +++ b/docs/maridb-backuprestore-from-tempauth.md @@ -0,0 +1,185 @@ +# MariaDB Restore Procedures with Swift Tempauth + +This document provides procedures to restore MariaDB backups stored in Rackspace's Swift object storage with tempauth for the production environments: DFW, SJC, and IAD as part of Jira:OSPC-1141. It details two methods: using theKubernetes Restore CRD with the MariaDB Operator and a manual restore using AWS S3 commands. These procedures ensure recovery from backups in the mariadb-backups container, based on the reference document present. + +## 1. Prerequisites + ### Software: + - Kubernetes CLI (`kubectl`) installed and configured with access to the respective production cluster (DFW, SJC, IAD). + - AWS CLI installed on the overseer node for each production environment (`pip install awscli awscli-plugin-endpoint`). + ### Credentials: + - Kubernetes secret (e.g., `dfw-credentials`, `sjc-credentials`, `iad-credentials`) with `access-key-id` and `secret-access-key` keys, generated via `openstack ec2 credentials create`. + - AWS CLI profiles (e.g., `dfw_admin`, `sjc_admin`, `iad_admin`) configured on the respective overseers. + ### Environment: + - Access to the Kubernetes cluster and overseer node for each production region. + - Network access to the region-specific Swift endpoint. + - MariaDB Operator deployed in each cluster with a `mariadb` resource. + +## 2. Backup/Restore Flow +```mermaid +graph TD + + subgraph Locations + I[DFW] + J[SJC] + K[IAD] + end + + A["Kubernetes Cluster
DFW, SJC, IAD"] --> B[MariaDB Instances] + B -->|Backup Data| C[MariaDB Operator] + C -->|Create Backup| D[Backup CRD] + D -->|Store Backup| E["Swift Object Storage
mariadb-backups"] + E -->|Retrieve Backup| F[Restore CRD] + F -->|Restore Data| C + C -->|Restore to MariaDB| B + E -->|Download Backup| G[Overseer Nodes] + G -->|Execute Restore| H[AWS CLI] + H -->|Restore to MariaDB| B + + I --> A + J --> A + K --> A +``` + +## 3. Restore Using Kubernetes `Restore` CRD +### CRD(Custom Resource Definition) +- The Restore CRD is a Custom Resource Definintion, a kubernetes feature that extend the API to define custom resources for managing restore operations. For detailed information on CRD, refers to the [Kubernetes Documentation on Custom Resources.](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). + +This method automates the restore process using the MariaDB Operator, applicable to all production regions. + + ### 3.1 Backup and Restore of Specific Databases + Backup Context: Backups are created with the Backup resource, which by default includes all logical databases. To back up specific databases, the databases field can be used (e.g., db1, db2, db3), influencing the content available for restoration. For detailed backup creation, refer to the backup documentation or administrator. + + Restore Configuration: By default, all databases in the backup are restored. To restore a single database, specify the database field in the Restore resource: + + ``` yaml + apiVersion: k8s.mariadb.com/v1alpha1 + kind: Restore + metadata: + name: restore + spec: + mariaDbRef: + name: mariadb + backupRef: + name: backup + database: db1 + ``` + ### 3.2 Procedure + #### 1. Configure the Restore CRD: + Create a file named `restore.yaml` with the following content, adjusting the region-specific details: + + ``` yaml + apiVersion: k8s.mariadb.com/v1alpha1 + kind: Restore + metadata: + name: maria-restore + namespace: # Replace with the actual namespace (e.g., default or mariadb) + spec: + mariaDbRef: + name: mariadb # Must match the existing MariaDB resource name + s3: + bucket: mariadb-backups + prefix: cron + endpoint: # See table below + accessKeyIdSecretKeyRef: + name: # e.g., dfw-credentials + key: access-key-id + secretAccessKeySecretKeyRef: + name: # e.g., dfw-credentials + key: secret-access-key + database: # e.g., nova + ``` + + Replace and with the appropriate values for each environment. + Use the following region-specific endpoints: + + | Region | Environment | Endpoint | Profile | Credential Secret | + |--------|-------------|-----------------------------------------------|-----------|-------------------| + | DFW | DFW | https://swift.api.dfw3.rackspacecloud.com | dfw_admin | dfw-credentials | + | SJC | SJC | https://swift.api.sjc3.rackspacecloud.com | sjc_admin | sjc-credentials | + | IAD | IAD | https://swift.api.iad3.rackspacecloud.com | iad_admin | iad-credentials | + + #### 2. Apply the CRD: + Execute the deployement: + ```shell + kubectl apply -f restore.yaml + ``` + #### 3. Monitor the Restore: + Check the status: + ```shell + kubectl describe restore maria-restore -n . + ``` + Here the resource type : restore + resource name : maria-restore + + Monitor logs: + ```shell + kubectl logs -f -n . + ``` + Identify the pod with kubectl get pods + Wait for the status to change to Succeeded. + #### 4. Verify Restore: + Access the mariadb Pod: + ```shell + kubectl exec -it -n -- mysql -u root -p. + ``` + Run a query: SELECT COUNT(*) FROM ; (e.g., nova.instances) to confirm data. + + **Notes:** + Ensure the region-specific credentials secret exists: kubectl get secret -n -o yaml. + + **Note:** This procedure some reference present in + +## 4. Manual Restore Using AWS S3 Commands + This method retrieves the backup from the overseer and restores it manually, applicable to all production regions as a fallback. + + ### Steps: + #### 1. Access the Region-Specific Overseer: + Log in to the overseer node (e.g., ssh user@dfw-prod-overseer-ip or any available method which is allowed to login into DFW Prod in a secured manner). + #### 2.Verify AWS CLI Configuration: + Ensure the region-specific profile is set up (e.g., dfw_admin for DFW): + ```yaml + [profile dfw_admin] + region = dfw + s3 = + endpoint_url = https://swift.api.dfw.rackspacecloud.com + signature_version = s3v4 + ``` + ```yaml + [dfw_admin] + aws_access_key_id = YOUR_ACCESS_KEY + aws_secret_access_key = YOUR_SECRET_KEY + ``` + Adjust for SJC (sjc_admin), IAD (iad_admin) with their endpoints (see table above). + Test with below command to list backups: + ```shell + aws --profile _admin s3 ls s3://mariadb-backups/ . + ``` + #### 3. Retrieve the Backup: + List available backups: aws --profile _admin s3 ls s3://mariadb-backups/cron/. + Download a specific backup: + As a example given here: + ```shell + aws --profile dfw_admin s3 cp s3://mariadb-backups/cron/backup.2025-02-04T19:05:57Z.gzip.sql /tmp/backup.2025-02-04T19:05:57Z.gzip.sql + ``` + Note: Need to replace the particular file to download as per the requirement. + #### 4. Restore the Backup: + Access a test MariaDB instance (e.g., via kubectl exec or a local DB): mysql -u user -p < backup.2025-02-04T19:05:57Z.gzip.sql. + #### 5. Single Database Restore: + If the backup contains multiple databases, extract the desired database (e.g., nova) using a tool like sed or mysql filters, then restore: mysql -u user -p nova < nova_backup.sql. + #### 6. Verify: + Check the return code: echo $? (0 indicates success). + Query the database: mysql -u user -p -e "SELECT COUNT(*) FROM ;". + + **Notes:** + Ensure the overseer has network access to the region-specific Swift endpoint. + + **Before applying/executing into Production, First need to apply these commands on DEV or staging environment.** + +## 5. References + ### https://github.com/mariadb-operator/mariadb-operator/blob/main/docs/BACKUP.md + ### https://docs.rackspacecloud.com/storage-object-store-s3-cli/ + ### https://mariadb.com/docs/server/server-usage/backup-and-restore/backup-and-restore-overview + +## 6. Escalation: + If validation fails, coordinate with Admin Team or Data Base team to resolve network or any related configuration issues.\ + **Note: This step might be extended further** From 77e149a16bf93e49556a175279183438a55585f1 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Fri, 11 Jul 2025 10:27:45 -0500 Subject: [PATCH 15/97] chore: update the first round of images to use ENT images by default (#1027) This change will now pull images from genestack-images by default, which is now maintaining enterprise ready images. Signed-off-by: Kevin Carter --- .github/workflows/enterprise-patching.yaml | 129 ------------------ .github/workflows/release-glance.yml | 118 ---------------- .github/workflows/release-heat-rxt.yml | 115 ---------------- .github/workflows/release-neutron-oslodb.yaml | 123 ----------------- .github/workflows/release-nova-uefi.yml | 116 ---------------- .github/workflows/release-octavia-ovn.yml | 124 ----------------- .github/workflows/smoke-glance.yml | 42 ------ .github/workflows/smoke-heat-rxt.yml | 41 ------ .github/workflows/smoke-nova-uefi.yml | 42 ------ .github/workflows/smoke-octavia-ovn.yml | 42 ------ .../workflows/testing-deploy-openstack.yaml | 46 ------- Containerfiles/Glance-Containerfile | 15 -- Containerfiles/HeatRXT-Containerfile | 13 -- Containerfiles/Neutron-oslo_db-Containerfile | 15 -- Containerfiles/NovaEFI-Containerfile | 22 --- Containerfiles/OctaviaOVN-Containerfile | 18 --- .../barbican/barbican-helm-overrides.yaml | 20 +-- .../ceilometer/ceilometer-helm-overrides.yaml | 6 +- .../cinder/cinder-helm-overrides.yaml | 14 +- .../designate/designate-helm-overrides.yaml | 14 +- .../glance/glance-helm-overrides.yaml | 20 +-- .../gnocchi/gnocchi-helm-overrides.yaml | 8 +- .../heat/heat-helm-overrides.yaml | 28 ++-- .../horizon/horizon-helm-overrides.yaml | 6 +- .../ironic/ironic-helm-overrides.yaml | 20 +-- .../keystone/keystone-helm-overrides.yaml | 26 ++-- .../libvirt/libvirt-helm-overrides.yaml | 4 +- .../magnum/magnum-helm-overrides.yaml | 14 +- .../masakari/masakari-helm-overrides.yaml | 12 +- .../neutron/neutron-helm-overrides.yaml | 48 +++---- .../nova/nova-helm-overrides.yaml | 40 +++--- .../octavia/octavia-helm-overrides.yaml | 26 ++-- .../placement/placement-helm-overrides.yaml | 16 +-- .../cinder/netapp/kustomization.yaml | 8 +- .../octavia/base/kustomization.yaml | 8 +- base-kustomize/ovn/base/ovn-setup.yaml | 4 +- .../skyline/base/deployment-apiserver.yaml | 4 +- .../utils/utils-openstack-client-admin.yaml | 2 +- 38 files changed, 174 insertions(+), 1195 deletions(-) delete mode 100644 .github/workflows/enterprise-patching.yaml delete mode 100644 .github/workflows/release-glance.yml delete mode 100644 .github/workflows/release-heat-rxt.yml delete mode 100644 .github/workflows/release-neutron-oslodb.yaml delete mode 100644 .github/workflows/release-nova-uefi.yml delete mode 100644 .github/workflows/release-octavia-ovn.yml delete mode 100644 .github/workflows/smoke-glance.yml delete mode 100644 .github/workflows/smoke-heat-rxt.yml delete mode 100644 .github/workflows/smoke-nova-uefi.yml delete mode 100644 .github/workflows/smoke-octavia-ovn.yml delete mode 100644 .github/workflows/testing-deploy-openstack.yaml delete mode 100644 Containerfiles/Glance-Containerfile delete mode 100644 Containerfiles/HeatRXT-Containerfile delete mode 100644 Containerfiles/Neutron-oslo_db-Containerfile delete mode 100644 Containerfiles/NovaEFI-Containerfile delete mode 100644 Containerfiles/OctaviaOVN-Containerfile diff --git a/.github/workflows/enterprise-patching.yaml b/.github/workflows/enterprise-patching.yaml deleted file mode 100644 index ff5ad6ed3..000000000 --- a/.github/workflows/enterprise-patching.yaml +++ /dev/null @@ -1,129 +0,0 @@ -name: Patch and Retag Images - -on: - workflow_dispatch: - workflow_run: - workflows: ["Migrate Images to QUAY"] - types: - - completed - branches: - - main - -jobs: - generate-matrix: - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} - outputs: - images: ${{ steps.generate-matrix.outputs.images }} - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Generate Matrix - id: generate-matrix - run: | - images=$(jq -r '.[]' .original-images.json | jq -R -s -c 'split("\n") | map(select(length > 0))') - echo "images=$images" >> $GITHUB_OUTPUT - - patch-and-retag: - needs: generate-matrix - runs-on: ubuntu-latest - strategy: - matrix: - image: ${{ fromJson(needs.generate-matrix.outputs.images) }} - fail-fast: false - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - driver: docker-container - driver-opts: | - image=moby/buildkit:master - network=host - - - name: Install Copacetic - run: | - wget https://github.com/project-copacetic/copacetic/releases/download/v0.9.0/copa_0.9.0_linux_amd64.tar.gz - tar -xzf copa_0.9.0_linux_amd64.tar.gz - chmod +x copa - sudo mv copa /usr/local/bin/ - - - name: Install Trivy - run: | - TRIVY_VERSION="0.55.0" - wget https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz - tar -xzf trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz - chmod +x trivy - sudo mv trivy /usr/local/bin/ - - - name: Login to Quay.io - uses: docker/login-action@v3 - with: - registry: quay.io - username: ${{ secrets.QUAY_USER }} - password: ${{ secrets.QUAY_TOKEN }} - - - name: Process Image - run: | - sudo apt-get update && sudo apt-get install -y jq python3-pip - image="${{ matrix.image }}" - echo "Processing $image" - base_name=$(echo "$image" | awk -F'/' '{print $NF}' | cut -d':' -f1) - tag=$(echo "$image" | awk -F':' '{print $NF}') - new_image="quay.io/rackspace/rackerlabs-${base_name}:${tag}" - patched_tag="${tag}-enterprise" - patched_image="quay.io/rackspace/rackerlabs-${base_name}:${patched_tag}" - - # Pull the image - docker pull "$new_image" || { echo "Failed to pull $new_image"; exit 1; } - - # Scan all vulnerabilities (OS and language-specific) - trivy image -f json -o "report-${base_name}-${tag}.json" "$new_image" || { echo "Failed to scan $new_image"; exit 1; } - - # Scan OS vulnerabilities with fixes for Copacetic - trivy image --vuln-type os --ignore-unfixed -f json -o "os-report-${base_name}-${tag}.json" "$new_image" || { echo "Failed to scan OS vulnerabilities for $new_image"; exit 1; } - - # Attempt to patch OS vulnerabilities; set intermediate image - if copa patch -i "$new_image" -r "os-report-${base_name}-${tag}.json" -t "$patched_tag"; then - echo "Patched OS vulnerabilities in $new_image" - intermediate_image="$patched_image" - else - echo "No OS vulnerabilities patched for $new_image" - intermediate_image="$new_image" - fi - - # Filter cve/requirements.txt to only update installed packages - docker run --rm -v "$(pwd):/output" "$intermediate_image" sh -c "/var/lib/openstack/bin/pip3 list --format=json > /output/installed.json 2>/dev/null || echo '[]' > /output/installed.json" - python3 cve/filter.py - - if [ -s "filtered-requirements.txt" ]; then - echo "Applying Python package updates from cve/requirements.txt" - echo "FROM $intermediate_image" > Dockerfile.temp - echo "COPY filtered-requirements.txt /tmp/filtered-requirements.txt" >> Dockerfile.temp - echo "RUN /var/lib/openstack/bin/pip3 install -r /tmp/filtered-requirements.txt" >> Dockerfile.temp - docker build -f Dockerfile.temp -t "$patched_image" . || { echo "Failed to build $patched_image with Python patches"; exit 1; } - intermediate_image="$patched_image" - else - echo "No Python packages updated from cve/requirements.txt" - fi - - # Flatten the image - echo "Flattening $patched_image" - container_id=$(docker create "$intermediate_image") - docker export "$container_id" > "flattened-${base_name}-${patched_tag}.tar" - docker import "flattened-${base_name}-${patched_tag}.tar" "$patched_image" - docker rm "$container_id" - rm "flattened-${base_name}-${patched_tag}.tar" - - # Push the flattened image - docker push "$patched_image" || { echo "Failed to push $patched_image"; exit 1; } - echo "Pushed $patched_image" - - # Clean up - rm -f "report-${base_name}-${tag}.json" "os-report-${base_name}-${tag}.json" filtered-requirements.txt Dockerfile.temp installed.json requirements.txt - -env: - DOCKER_CLI_EXPERIMENTAL: enabled diff --git a/.github/workflows/release-glance.yml b/.github/workflows/release-glance.yml deleted file mode 100644 index a92b36f1a..000000000 --- a/.github/workflows/release-glance.yml +++ /dev/null @@ -1,118 +0,0 @@ -# -name: Create and publish a Glance compatible image - -# Configures this workflow to run every time a change is pushed to the branch called `release`. -on: - workflow_dispatch: - inputs: - imageTag: - description: 'Set tag for the image' - required: true - default: 'master-ubuntu_jammy' - type: choice - options: - - master-ubuntu_jammy - - 2023.1-ubuntu_jammy - - 2023.2-ubuntu_jammy - - 2024.1-ubuntu_jammy - pluginTag: - description: 'Set release used for the build environment' - required: true - default: 'master' - type: choice - options: - - "master" - - "2023.1" - - "2023.2" - - "2024.1" - -# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - build-and-push-image: - outputs: - MY_DATE: ${{ steps.mydate.outputs.MY_DATE }} - MY_CONTAINER: ${{ steps.mycontainer.outputs.MY_CONTAINER }} - runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. - # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. - # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - - name: Dynamically set MY_DATE environment variable - run: echo "MY_DATE=$(date +%s)" >> $GITHUB_ENV - - name: Build and push Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: Containerfiles/Glance-Containerfile - push: true - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/glance:${{ github.event.inputs.imageTag }} - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/glance:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - VERSION=${{ github.event.inputs.imageTag }} - PLUGIN_VERSION=${{ github.event.inputs.pluginTag }} - - name: Dynamically set MY_CONTAINER output option - id: mycontainer - run: echo "MY_CONTAINER=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/glance:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }}" >> $GITHUB_OUTPUT - - name: Dynamically set MY_DATE output option - id: mydate - run: echo "MY_DATE=${{ env.MY_DATE }}" >> $GITHUB_OUTPUT - - change-original-images: - runs-on: ubuntu-latest - needs: [build-and-push-image] - permissions: - contents: write - pull-requests: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Dynamically update the original images file - run: jq '. + ["${{ needs.build-and-push-image.outputs.MY_CONTAINER }}"] | sort' .original-images.json | tee .original-images.json.new - - name: Rewrite original images file - run: mv .original-images.json.new .original-images.json - - name: Create Pull Request - id: cpr - uses: peter-evans/create-pull-request@v7 - with: - commit-message: Update original images with new container - committer: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> - author: ${{ github.actor }} <${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com> - signoff: false - branch: ${{ needs.build-and-push-image.outputs.MY_DATE }} - sign-commits: true - delete-branch: true - title: 'chore: Update original images' - body: | - Update container image - - Updated original image file with container ${{needs.build-and-push-image.outputs.MY_CONTAINER}} - change request Auto-generated - labels: | - container images - automated pr - draft: false diff --git a/.github/workflows/release-heat-rxt.yml b/.github/workflows/release-heat-rxt.yml deleted file mode 100644 index 4b1a2af45..000000000 --- a/.github/workflows/release-heat-rxt.yml +++ /dev/null @@ -1,115 +0,0 @@ -# -name: Create and Publish a Heat Image - -on: - push: - paths: - - '.github/workflows/release-heat-rxt.yml' - - 'Containerfiles/HeatRXT-Containerfile' - branches: - - development - - main - workflow_dispatch: - inputs: - imageTag: - description: 'Set tag for the image' - required: true - default: '2024.1-ubuntu_jammy' - type: choice - options: - - 2024.1-ubuntu_jammy - -# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - DEF_TAG_NAME: 2024.1-ubuntu_jammy - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - build-and-push-image: - outputs: - MY_DATE: ${{ steps.mydate.outputs.MY_DATE }} - MY_CONTAINER: ${{ steps.mycontainer.outputs.MY_CONTAINER }} - runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - # ghcr only allows lowercase repository names - - name: lowercase repo name - run: | - echo "IMAGE_NAME=${GITHUB_REPOSITORY,,}" >>${GITHUB_ENV} - # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. - # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. - # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - - name: Dynamically set MY_DATE environment variable - run: echo "MY_DATE=$(date +%s)" >> $GITHUB_ENV - - name: Build and push Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: Containerfiles/HeatRXT-Containerfile - push: true - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/heat:${{ github.event.inputs.imageTag || env.DEF_TAG_NAME }} - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/heat:${{ github.event.inputs.imageTag || env.DEF_TAG_NAME }}-${{ env.MY_DATE }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - VERSION=${{ github.event.inputs.imageTag || env.DEF_TAG_NAME }} - - name: Dynamically set MY_CONTAINER output option - id: mycontainer - run: echo "MY_CONTAINER=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/heat:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }}" >> $GITHUB_OUTPUT - - name: Dynamically set MY_DATE output option - id: mydate - run: echo "MY_DATE=${{ env.MY_DATE }}" >> $GITHUB_OUTPUT - - change-original-images: - runs-on: ubuntu-latest - needs: [build-and-push-image] - permissions: - contents: write - pull-requests: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Dynamically update the original images file - run: jq '. + ["${{ needs.build-and-push-image.outputs.MY_CONTAINER }}"] | sort' .original-images.json | tee .original-images.json.new - - name: Rewrite original images file - run: mv .original-images.json.new .original-images.json - - name: Create Pull Request - id: cpr - uses: peter-evans/create-pull-request@v7 - with: - commit-message: Update original images with new container - committer: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> - author: ${{ github.actor }} <${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com> - signoff: false - branch: ${{ needs.build-and-push-image.outputs.MY_DATE }} - sign-commits: true - delete-branch: true - title: 'chore: Update original images' - body: | - Update container image - - Updated original image file with container ${{needs.build-and-push-image.outputs.MY_CONTAINER}} - change request Auto-generated - labels: | - container images - automated pr - draft: false diff --git a/.github/workflows/release-neutron-oslodb.yaml b/.github/workflows/release-neutron-oslodb.yaml deleted file mode 100644 index 3386e9221..000000000 --- a/.github/workflows/release-neutron-oslodb.yaml +++ /dev/null @@ -1,123 +0,0 @@ -# -name: Create and publish a Neutron oslodb patched image - -# Configures this workflow to run every time a change is pushed to the branch called `release`. -on: - workflow_dispatch: - inputs: - imageTag: - description: 'Set tag for the image' - required: true - default: 'master-ubuntu_jammy' - type: choice - options: - - master-ubuntu_jammy - - 2024.1-ubuntu_jammy - pluginTag: - description: 'Set release used for the build environment' - required: true - default: 'master' - type: choice - options: - - "master" - - "2024.1" - NeutronTag: - description: 'Set Neutron version' - required: true - default: 'sync-add-mode' - type: choice - options: - - 'sync-add-mode' - - 'sync-add-mode-2024.1' - -# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - build-and-push-image: - outputs: - MY_DATE: ${{ steps.mydate.outputs.MY_DATE }} - MY_CONTAINER: ${{ steps.mycontainer.outputs.MY_CONTAINER }} - runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. - # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. - # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - - name: Dynamically set MY_DATE environment variable - run: echo "MY_DATE=$(date +%s)" >> $GITHUB_ENV - - name: Build and push Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: Containerfiles/Neutron-oslo_db-Containerfile - push: true - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/neutron-oslodb:${{ github.event.inputs.imageTag }} - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/neutron-oslodb:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - VERSION=${{ github.event.inputs.imageTag }} - PLUGIN_VERSION=${{ github.event.inputs.pluginTag }} - NEUTRON_VERSION=${{ github.event.inputs.NeutronTag }} - - name: Dynamically set MY_CONTAINER output option - id: mycontainer - run: echo "MY_CONTAINER=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/neutron-oslodb:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }}" >> $GITHUB_OUTPUT - - name: Dynamically set MY_DATE output option - id: mydate - run: echo "MY_DATE=${{ env.MY_DATE }}" >> $GITHUB_OUTPUT - - change-original-images: - runs-on: ubuntu-latest - needs: [build-and-push-image] - permissions: - contents: write - pull-requests: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Dynamically update the original images file - run: jq '. + ["${{ needs.build-and-push-image.outputs.MY_CONTAINER }}"] | sort' .original-images.json | tee .original-images.json.new - - name: Rewrite original images file - run: mv .original-images.json.new .original-images.json - - name: Create Pull Request - id: cpr - uses: peter-evans/create-pull-request@v7 - with: - commit-message: Update original images with new container - committer: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> - author: ${{ github.actor }} <${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com> - signoff: false - branch: ${{ needs.build-and-push-image.outputs.MY_DATE }} - sign-commits: true - delete-branch: true - title: 'chore: Update original images' - body: | - Update container image - - Updated original image file with container ${{needs.build-and-push-image.outputs.MY_CONTAINER}} - change request Auto-generated - labels: | - container images - automated pr - draft: false diff --git a/.github/workflows/release-nova-uefi.yml b/.github/workflows/release-nova-uefi.yml deleted file mode 100644 index 388f5e1ac..000000000 --- a/.github/workflows/release-nova-uefi.yml +++ /dev/null @@ -1,116 +0,0 @@ -# -name: Create and publish a the Nova EFI compatible image - -# Configures this workflow to run every time a change is pushed to the branch called `release`. -on: - workflow_dispatch: - inputs: - imageTag: - description: 'Set tag for the image' - required: true - default: 'master-ubuntu_jammy' - type: choice - options: - - master-ubuntu_jammy - - 2025.1-ubuntu_jammy - - 2024.1-ubuntu_jammy - pluginTag: - description: 'Set release used for the build environment' - required: true - default: 'master' - type: choice - options: - - "master" - - "2025.1" - - "2024.1" - -# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - build-and-push-image: - outputs: - MY_DATE: ${{ steps.mydate.outputs.MY_DATE }} - MY_CONTAINER: ${{ steps.mycontainer.outputs.MY_CONTAINER }} - runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. - # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. - # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - - name: Dynamically set MY_DATE environment variable - run: echo "MY_DATE=$(date +%s)" >> $GITHUB_ENV - - name: Build and push Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: Containerfiles/NovaEFI-Containerfile - push: true - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/nova-efi:${{ github.event.inputs.imageTag }} - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/nova-efi:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - VERSION=${{ github.event.inputs.imageTag }} - PLUGIN_VERSION=${{ github.event.inputs.pluginTag }} - - name: Dynamically set MY_CONTAINER output option - id: mycontainer - run: echo "MY_CONTAINER=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/nova-efi:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }}" >> $GITHUB_OUTPUT - - name: Dynamically set MY_DATE output option - id: mydate - run: echo "MY_DATE=${{ env.MY_DATE }}" >> $GITHUB_OUTPUT - - change-original-images: - runs-on: ubuntu-latest - needs: [build-and-push-image] - permissions: - contents: write - pull-requests: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Dynamically update the original images file - run: jq '. + ["${{ needs.build-and-push-image.outputs.MY_CONTAINER }}"] | sort' .original-images.json | tee .original-images.json.new - - name: Rewrite original images file - run: mv .original-images.json.new .original-images.json - - name: Create Pull Request - id: cpr - uses: peter-evans/create-pull-request@v7 - with: - commit-message: Update original images with new container - committer: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> - author: ${{ github.actor }} <${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com> - signoff: false - branch: ${{ needs.build-and-push-image.outputs.MY_DATE }} - sign-commits: true - delete-branch: true - title: 'chore: Update original images' - body: | - Update container image - - Updated original image file with container ${{needs.build-and-push-image.outputs.MY_CONTAINER}} - change request Auto-generated - labels: | - container images - automated pr - draft: false diff --git a/.github/workflows/release-octavia-ovn.yml b/.github/workflows/release-octavia-ovn.yml deleted file mode 100644 index 1221fa84b..000000000 --- a/.github/workflows/release-octavia-ovn.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: Create and publish an Octavia compatible image - -# Configures this workflow to run every time a change is pushed to the branch called `release`. -on: - workflow_dispatch: - inputs: - imageTag: - description: 'Set tag for the image' - required: true - default: 'master-ubuntu_jammy' - type: choice - options: - - master-ubuntu_jammy - - 2024.1-ubuntu_jammy - pluginTag: - description: 'Set release used for the build environment' - required: true - default: 'master' - type: choice - options: - - "master" - - "2024.1" - ovnPluginTag: - description: 'Set OVN plugin version' - required: true - default: 'master' - type: choice - options: - - 'master' - - '5.0.0' - - '6.0.0' - - '7.0.0' - -# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - build-and-push-image: - outputs: - MY_DATE: ${{ steps.mydate.outputs.MY_DATE }} - MY_CONTAINER: ${{ steps.mycontainer.outputs.MY_CONTAINER }} - runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. - # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. - # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - - name: Dynamically set MY_DATE environment variable - run: echo "MY_DATE=$(date +%s)" >> $GITHUB_ENV - - name: Build and push Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: Containerfiles/OctaviaOVN-Containerfile - push: true - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/octavia-ovn:${{ github.event.inputs.imageTag }} - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/octavia-ovn:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - VERSION=${{ github.event.inputs.imageTag }} - PLUGIN_VERSION=${{ github.event.inputs.pluginTag }} - OVN_PLUGIN_VERSION=${{ github.event.inputs.ovnPluginTag }} - - name: Dynamically set MY_CONTAINER output option - id: mycontainer - run: echo "MY_CONTAINER=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/nova-efi:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }}" >> $GITHUB_OUTPUT - - name: Dynamically set MY_DATE output option - id: mydate - run: echo "MY_DATE=${{ env.MY_DATE }}" >> $GITHUB_OUTPUT - - change-original-images: - runs-on: ubuntu-latest - needs: [build-and-push-image] - permissions: - contents: write - pull-requests: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Dynamically update the original images file - run: jq '. + ["${{ needs.build-and-push-image.outputs.MY_CONTAINER }}"] | sort' .original-images.json | tee .original-images.json.new - - name: Rewrite original images file - run: mv .original-images.json.new .original-images.json - - name: Create Pull Request - id: cpr - uses: peter-evans/create-pull-request@v7 - with: - commit-message: Update original images with new container - committer: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> - author: ${{ github.actor }} <${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com> - signoff: false - branch: ${{ needs.build-and-push-image.outputs.MY_DATE }} - sign-commits: true - delete-branch: true - title: 'chore: Update original images' - body: | - Update container image - - Updated original image file with container ${{needs.build-and-push-image.outputs.MY_CONTAINER}} - change request Auto-generated - labels: | - container images - automated pr - draft: false diff --git a/.github/workflows/smoke-glance.yml b/.github/workflows/smoke-glance.yml deleted file mode 100644 index 7d0567f6f..000000000 --- a/.github/workflows/smoke-glance.yml +++ /dev/null @@ -1,42 +0,0 @@ -# -name: Run build check for the Glance compatible image - -on: - pull_request: - paths: - - Containerfiles/Glance-Containerfile - -# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - build-and-push-image: - runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: read - steps: - - name: Checkout repository - uses: actions/checkout@v4 - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: Containerfiles/Glance-Containerfile - push: false - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/glance:master-ubuntu_jammy - build-args: | - VERSION=master-ubuntu_jammy - PLUGIN_VERSION=master diff --git a/.github/workflows/smoke-heat-rxt.yml b/.github/workflows/smoke-heat-rxt.yml deleted file mode 100644 index f038661d0..000000000 --- a/.github/workflows/smoke-heat-rxt.yml +++ /dev/null @@ -1,41 +0,0 @@ -# -name: Run build check for the Heat image - -on: - pull_request: - paths: - - Containerfiles/HeatRXT-Containerfile - -# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - build-and-push-image: - runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: read - steps: - - name: Checkout repository - uses: actions/checkout@v4 - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: Containerfiles/HeatRXT-Containerfile - push: false - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/heat:2024.1-ubuntu_jammy - build-args: | - VERSION=2024.1-ubuntu_jammy diff --git a/.github/workflows/smoke-nova-uefi.yml b/.github/workflows/smoke-nova-uefi.yml deleted file mode 100644 index f9aa31d8d..000000000 --- a/.github/workflows/smoke-nova-uefi.yml +++ /dev/null @@ -1,42 +0,0 @@ -# -name: Run build check for the Nova EFI compatible image - -on: - pull_request: - paths: - - Containerfiles/NovaEFI-Containerfile - -# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - build-and-push-image: - runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: read - steps: - - name: Checkout repository - uses: actions/checkout@v4 - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: Containerfiles/NovaEFI-Containerfile - push: false - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/nova-efi:master-ubuntu_jammy - build-args: | - VERSION=master-ubuntu_jammy - PLUGIN_VERSION=master diff --git a/.github/workflows/smoke-octavia-ovn.yml b/.github/workflows/smoke-octavia-ovn.yml deleted file mode 100644 index ffebf1def..000000000 --- a/.github/workflows/smoke-octavia-ovn.yml +++ /dev/null @@ -1,42 +0,0 @@ -# -name: Run build check for the Octavia OVN compatible image - -on: - pull_request: - paths: - - Containerfiles/OctaviaOVN-Containerfile - -# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - build-and-push-image: - runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: read - steps: - - name: Checkout repository - uses: actions/checkout@v4 - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: Containerfiles/OctaviaOVN-Containerfile - push: false - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/octavia-ovn:master-ubuntu_jammy - build-args: | - VERSION=master-ubuntu_jammy - PLUGIN_VERSION=master diff --git a/.github/workflows/testing-deploy-openstack.yaml b/.github/workflows/testing-deploy-openstack.yaml deleted file mode 100644 index 60517178f..000000000 --- a/.github/workflows/testing-deploy-openstack.yaml +++ /dev/null @@ -1,46 +0,0 @@ -name: testing-openstack-deploy - -on: - workflow_run: - workflows: - - Migrate Images to QUAY - types: - - completed - workflow_dispatch: - -jobs: - deploy: - - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Set up Python environment - run: sudo apt-get update && sudo apt-get install -y python3-pip - - - name: Install requirements - run: pip3 install -r testing/requirements.txt - - - name: Create OpenStack config directory - run: mkdir -p ~/.config/openstack - - - name: Retrieve clouds.yaml from GitHub secrets - env: - CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} - run: | - echo "$CLOUDS_YAML" > ~/.config/openstack/clouds.yaml - - - name: Retrieve env.yaml from GitHub secrets - env: - ENV_YAML: ${{ secrets.ENV_YAML }} - run: | - echo "$ENV_YAML" > ~/env.yaml - - - name: Run deployment script - run: bash testing/doit.sh - - - name: Cleanup - if: always() - run: bash testing/cleanup.sh diff --git a/Containerfiles/Glance-Containerfile b/Containerfiles/Glance-Containerfile deleted file mode 100644 index a710b16b7..000000000 --- a/Containerfiles/Glance-Containerfile +++ /dev/null @@ -1,15 +0,0 @@ -# Patch oslo_db to help with deadlocks -ARG VERSION=master-ubuntu_jammy -FROM openstackhelm/glance:$VERSION as build -ARG PLUGIN_VERSION=master -RUN apt update && apt install -y git -RUN export ORIG_PLUGIN_VERSION="${PLUGIN_VERSION}"; \ -if [ "${PLUGIN_VERSION}" != 'master' ]; then export PLUGIN_VERSION=stable/${PLUGIN_VERSION}; fi; \ -. /var/lib/openstack/bin/activate; \ -/var/lib/openstack/bin/pip install boto3 os-brick \ - git+https://github.com/openstack/python-cinderclient@${PLUGIN_VERSION}#egg=python-cinderclient \ - git+https://github.com/openstack/oslo.db@${PLUGIN_VERSION}#egg=oslo_db \ - git+https://github.com/openstack/glance@${PLUGIN_VERSION}#egg=glance - -FROM openstackhelm/glance:${VERSION} -COPY --from=build /var/lib/openstack/. /var/lib/openstack/ diff --git a/Containerfiles/HeatRXT-Containerfile b/Containerfiles/HeatRXT-Containerfile deleted file mode 100644 index 3a4d47fe7..000000000 --- a/Containerfiles/HeatRXT-Containerfile +++ /dev/null @@ -1,13 +0,0 @@ -ARG VERSION=master-ubuntu_jammy -FROM openstackhelm/heat:${VERSION} as build -RUN apt-get update && apt-get install -y git && apt clean -RUN /var/lib/openstack/bin/pip install git+https://opendev.org/openstack/heat.git@stable/2024.1 -RUN /var/lib/openstack/bin/pip install --upgrade --force-reinstall pip -RUN find /var/lib/openstack -regex '^.*\(__pycache__\|\.py[co]\)$' -delete - -FROM openstackhelm/heat:${VERSION} -COPY --from=build /var/lib/openstack/. /var/lib/openstack/ -COPY Containerfiles/patches/heat_keystone_v3_patch.diff /tmp/heat_keystone_v3_patch.diff -RUN apt-get update && apt-get install -y git -RUN cd /var/lib/openstack/lib/python3.10/site-packages/ && git apply /tmp/heat_keystone_v3_patch.diff -RUN rm /tmp/heat_keystone_v3_patch.diff diff --git a/Containerfiles/Neutron-oslo_db-Containerfile b/Containerfiles/Neutron-oslo_db-Containerfile deleted file mode 100644 index 45a9647cd..000000000 --- a/Containerfiles/Neutron-oslo_db-Containerfile +++ /dev/null @@ -1,15 +0,0 @@ -# Patch oslo_db to help with deadlocks -ARG VERSION=master-ubuntu_jammy -FROM openstackhelm/neutron:$VERSION as build -ARG PLUGIN_VERSION=master -ARG NEUTRON_VERSION=master -RUN apt update && apt install -y git -RUN export ORIG_PLUGIN_VERSION="${PLUGIN_VERSION}"; \ -if [ "${PLUGIN_VERSION}" != 'master' ]; then export PLUGIN_VERSION=stable/${PLUGIN_VERSION}; fi; \ -. /var/lib/openstack/bin/activate; \ -/var/lib/openstack/bin/pip install --upgrade \ -git+https://github.com/openstack/oslo.db@${PLUGIN_VERSION}#egg=oslo_db \ -git+https://github.com/rackerlabs/neutron@${NEUTRON_VERSION}#egg=neutron - -FROM openstackhelm/neutron:${VERSION} -COPY --from=build /var/lib/openstack/. /var/lib/openstack/ diff --git a/Containerfiles/NovaEFI-Containerfile b/Containerfiles/NovaEFI-Containerfile deleted file mode 100644 index 5a82d2803..000000000 --- a/Containerfiles/NovaEFI-Containerfile +++ /dev/null @@ -1,22 +0,0 @@ -ARG VERSION=master-ubuntu_jammy -FROM quay.io/airshipit/nova:$VERSION as build -ARG PLUGIN_VERSION=master -RUN apt update && apt install -y git -RUN export ORIG_PLUGIN_VERSION="${PLUGIN_VERSION}"; \ -if [ "${PLUGIN_VERSION}" != 'master' ]; then export PLUGIN_VERSION=stable/${PLUGIN_VERSION}; fi; \ -. /var/lib/openstack/bin/activate; \ -/var/lib/openstack/bin/pip install git+https://github.com/openstack/oslo.db@${PLUGIN_VERSION}#egg=oslo_db && \ -/var/lib/openstack/bin/pip install python-barbicanclient - -FROM quay.io/airshipit/nova:${VERSION} -COPY --from=build /var/lib/openstack/. /var/lib/openstack/ -# Packages for the following features: -# - Nova: EFI -# - Nova: iSCSI -# Py Packages for the following features: -# - Nova: Libosinfo -RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y ovmf open-iscsi multipath-tools libgirepository-1.0-1 libgirepository1.0-dev \ - libcairo2-dev python3-dev gcc libosinfo-bin gir1.2-libosinfo-1.0 nfs-common cryptsetup nvme-cli; \ - rm -rf /var/cache/apt/archives /var/lib/apt/lists; \ - apt clean; /var/lib/openstack/bin/pip install pygobject; \ - find /var/lib/openstack -regex '^.*\(__pycache__\|\.py[co]\)$' -delete diff --git a/Containerfiles/OctaviaOVN-Containerfile b/Containerfiles/OctaviaOVN-Containerfile deleted file mode 100644 index c0d618a8c..000000000 --- a/Containerfiles/OctaviaOVN-Containerfile +++ /dev/null @@ -1,18 +0,0 @@ -ARG VERSION=master-ubuntu_jammy -FROM openstackhelm/octavia:$VERSION as build -ARG PLUGIN_VERSION=master -ARG OVN_PLUGIN_VERSION=master -RUN apt update && apt install -y git -RUN /var/lib/openstack/bin/pip install --index-url https://pypi.python.org/simple --upgrade pip -RUN if [ "${PLUGIN_VERSION}" != 'master' ]; then export PLUGIN_VERSION=stable/${PLUGIN_VERSION}; fi; \ -. /var/lib/openstack/bin/activate; \ -/var/lib/openstack/bin/pip install git+https://github.com/openstack/oslo.db@${PLUGIN_VERSION}#egg=oslo_db -RUN . /var/lib/openstack/bin/activate; \ -if [ "${OVN_PLUGIN_VERSION}" = 'master' ]; then \ -/var/lib/openstack/bin/pip install git+https://github.com/openstack/ovn-octavia-provider@${OVN_PLUGIN_VERSION}#egg=ovn_octavia_provider; \ -else \ -/var/lib/openstack/bin/pip install --index-url https://pypi.python.org/simple ovn-octavia-provider==${OVN_PLUGIN_VERSION}; \ -fi - -FROM openstackhelm/octavia:${VERSION} -COPY --from=build /var/lib/openstack/. /var/lib/openstack/ diff --git a/base-helm-configs/barbican/barbican-helm-overrides.yaml b/base-helm-configs/barbican/barbican-helm-overrides.yaml index 543ef7074..7ad2479cc 100644 --- a/base-helm-configs/barbican/barbican-helm-overrides.yaml +++ b/base-helm-configs/barbican/barbican-helm-overrides.yaml @@ -1,18 +1,18 @@ --- images: tags: - barbican_api: "quay.io/rackspace/rackerlabs-barbican:2024.1-ubuntu_jammy" - barbican_db_sync: "quay.io/rackspace/rackerlabs-barbican:2024.1-ubuntu_jammy" - bootstrap: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - db_drop: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - db_init: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - dep_check: "quay.io/rackspace/rackerlabs-kubernetes-entrypoint:latest-ubuntu_jammy" + barbican_api: "ghcr.io/rackerlabs/genestack-images/barbican:2024.1-latest" + barbican_db_sync: "ghcr.io/rackerlabs/genestack-images/barbican:2024.1-latest" + bootstrap: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" - ks_endpoints: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_service: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_user: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" + ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" - scripted_test: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" + scripted_test: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" # NOTE: (brew) CPU requests values based on a three node # hyperconverged lab (/scripts/hyperconverged-lab.sh). diff --git a/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml b/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml index 34a66a2cf..19e8efd5d 100644 --- a/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml +++ b/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml @@ -4,13 +4,13 @@ images: test: "quay.io/rackspace/rackerlabs-xrally-openstack:2.0.0" ceilometer_db_sync: "quay.io/rackspace/rackerlabs-ceilometer:2024.1-ubuntu_jammy" rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" - ks_user: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_service: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" + ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ceilometer_central: "quay.io/rackspace/rackerlabs-ceilometer:2024.1-ubuntu_jammy" ceilometer_compute: "quay.io/rackspace/rackerlabs-ceilometer:2024.1-ubuntu_jammy" ceilometer_ipmi: "quay.io/rackspace/rackerlabs-ceilometer:2024.1-ubuntu_jammy" ceilometer_notification: "quay.io/rackspace/rackerlabs-ceilometer:2024.1-ubuntu_jammy" - dep_check: "quay.io/rackspace/rackerlabs-kubernetes-entrypoint:latest-ubuntu_jammy" + dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" pull_policy: "Always" diff --git a/base-helm-configs/cinder/cinder-helm-overrides.yaml b/base-helm-configs/cinder/cinder-helm-overrides.yaml index 3b9df2bdc..0b991e40c 100644 --- a/base-helm-configs/cinder/cinder-helm-overrides.yaml +++ b/base-helm-configs/cinder/cinder-helm-overrides.yaml @@ -7,7 +7,7 @@ labels: images: tags: - bootstrap: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" + bootstrap: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" cinder_api: "quay.io/rackspace/rackerlabs-cinder:2024.1-ubuntu_jammy" cinder_backup: "quay.io/rackspace/rackerlabs-cinder:2024.1-ubuntu_jammy" cinder_backup_storage_init: "quay.io/rackspace/rackerlabs-ceph-config-helper:latest-ubuntu_jammy" @@ -16,13 +16,13 @@ images: cinder_storage_init: "quay.io/rackspace/rackerlabs-ceph-config-helper:latest-ubuntu_jammy" cinder_volume: "quay.io/rackspace/rackerlabs-cinder:2024.1-ubuntu_jammy" cinder_volume_usage_audit: "quay.io/rackspace/rackerlabs-cinder:2024.1-ubuntu_jammy" - db_drop: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - db_init: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - dep_check: "quay.io/rackspace/rackerlabs-kubernetes-entrypoint:latest-ubuntu_jammy" + db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" - ks_endpoints: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_service: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_user: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" + ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" test: "quay.io/rackspace/rackerlabs-xrally-openstack:2.0.0" diff --git a/base-helm-configs/designate/designate-helm-overrides.yaml b/base-helm-configs/designate/designate-helm-overrides.yaml index f7bfd57b5..83ee64287 100644 --- a/base-helm-configs/designate/designate-helm-overrides.yaml +++ b/base-helm-configs/designate/designate-helm-overrides.yaml @@ -43,14 +43,14 @@ labels: images: tags: - bootstrap: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - db_init: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - db_drop: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" + bootstrap: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" - ks_user: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_service: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_endpoints: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - dep_check: "quay.io/rackspace/rackerlabs-kubernetes-entrypoint:latest-ubuntu_jammy" + ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" designate_db_sync: "quay.io/rackspace/rackerlabs-designate:2024.1-ubuntu_jammy" designate_api: "quay.io/rackspace/rackerlabs-designate:2024.1-ubuntu_jammy" designate_central: "quay.io/rackspace/rackerlabs-designate:2024.1-ubuntu_jammy" diff --git a/base-helm-configs/glance/glance-helm-overrides.yaml b/base-helm-configs/glance/glance-helm-overrides.yaml index 7c09c62ed..6e4612707 100644 --- a/base-helm-configs/glance/glance-helm-overrides.yaml +++ b/base-helm-configs/glance/glance-helm-overrides.yaml @@ -6,18 +6,18 @@ images: tags: test: "quay.io/rackspace/rackerlabs-xrally-openstack:2.0.0" glance_storage_init: "quay.io/rackspace/rackerlabs-ceph-config-helper:latest-ubuntu_jammy" - glance_metadefs_load: "quay.io/rackspace/rackerlabs-glance:2024.1-ubuntu_jammy" - db_init: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - glance_db_sync: "quay.io/rackspace/rackerlabs-glance:2024.1-ubuntu_jammy" - db_drop: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_user: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_service: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_endpoints: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" + glance_metadefs_load: "ghcr.io/rackerlabs/genestack-images/glance:2024.1-latest" + db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + glance_db_sync: "ghcr.io/rackerlabs/genestack-images/glance:2024.1-latest" + db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" - glance_api: "quay.io/rackspace/rackerlabs-glance:2024.1-ubuntu_jammy" + glance_api: "ghcr.io/rackerlabs/genestack-images/glance:2024.1-latest" # Bootstrap image requires curl - bootstrap: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - dep_check: "quay.io/rackspace/rackerlabs-kubernetes-entrypoint:latest-ubuntu_jammy" + bootstrap: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" bootstrap: diff --git a/base-helm-configs/gnocchi/gnocchi-helm-overrides.yaml b/base-helm-configs/gnocchi/gnocchi-helm-overrides.yaml index d3adf7d9c..b78923eeb 100644 --- a/base-helm-configs/gnocchi/gnocchi-helm-overrides.yaml +++ b/base-helm-configs/gnocchi/gnocchi-helm-overrides.yaml @@ -4,16 +4,16 @@ images: db_init: "quay.io/rackspace/rackerlabs-gnocchi:2024.1-ubuntu_jammy" db_init_indexer: "quay.io/rackspace/rackerlabs-postgres:14.5" db_sync: "quay.io/rackspace/rackerlabs-gnocchi:2024.1-ubuntu_jammy" - dep_check: "quay.io/rackspace/rackerlabs-kubernetes-entrypoint:latest-ubuntu_jammy" + dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" gnocchi_api: "quay.io/rackspace/rackerlabs-gnocchi:2024.1-ubuntu_jammy" gnocchi_metricd: "quay.io/rackspace/rackerlabs-gnocchi:2024.1-ubuntu_jammy" gnocchi_resources_cleaner: "quay.io/rackspace/rackerlabs-gnocchi:2024.1-ubuntu_jammy" gnocchi_statsd: "quay.io/rackspace/rackerlabs-gnocchi:2024.1-ubuntu_jammy" gnocchi_storage_init: "quay.io/rackspace/rackerlabs-ceph-config-helper:latest-ubuntu_jammy" image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" - ks_endpoints: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_service: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_user: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" + ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ceph_client: user_secret_name: gnocchi-temp-keyring diff --git a/base-helm-configs/heat/heat-helm-overrides.yaml b/base-helm-configs/heat/heat-helm-overrides.yaml index 31c5d1f31..63611b83f 100644 --- a/base-helm-configs/heat/heat-helm-overrides.yaml +++ b/base-helm-configs/heat/heat-helm-overrides.yaml @@ -1,21 +1,21 @@ --- images: tags: - bootstrap: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - db_drop: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - db_init: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - dep_check: "quay.io/rackspace/rackerlabs-kubernetes-entrypoint:latest-ubuntu_jammy" - heat_api: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - heat_cfn: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - heat_cloudwatch: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - heat_db_sync: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - heat_engine: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - heat_engine_cleaner: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - heat_purge_deleted: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" + bootstrap: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" + heat_api: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + heat_cfn: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + heat_cloudwatch: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + heat_db_sync: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + heat_engine: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + heat_engine_cleaner: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + heat_purge_deleted: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" - ks_endpoints: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_service: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_user: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" + ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" test: "quay.io/rackspace/rackerlabs-xrally-openstack:2.0.0" diff --git a/base-helm-configs/horizon/horizon-helm-overrides.yaml b/base-helm-configs/horizon/horizon-helm-overrides.yaml index f543a492f..93ab4843d 100644 --- a/base-helm-configs/horizon/horizon-helm-overrides.yaml +++ b/base-helm-configs/horizon/horizon-helm-overrides.yaml @@ -1,12 +1,12 @@ --- images: tags: - db_init: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - db_drop: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" + db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" horizon_db_sync: "quay.io/rackspace/rackerlabs-horizon:2024.1-ubuntu_jammy" horizon: "quay.io/rackspace/rackerlabs-horizon:2024.1-ubuntu_jammy" test: "quay.io/rackspace/rackerlabs-osh-selenium:latest-ubuntu_jammy" - dep_check: "quay.io/rackspace/rackerlabs-kubernetes-entrypoint:latest-ubuntu_jammy" + dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" # NOTE: (brew) requests cpu/mem values based on a three node diff --git a/base-helm-configs/ironic/ironic-helm-overrides.yaml b/base-helm-configs/ironic/ironic-helm-overrides.yaml index 249d89cc1..3fcb4fa60 100644 --- a/base-helm-configs/ironic/ironic-helm-overrides.yaml +++ b/base-helm-configs/ironic/ironic-helm-overrides.yaml @@ -6,16 +6,16 @@ --- images: tags: - ironic_manage_cleaning_network: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ironic_retrive_cleaning_network: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ironic_retrive_swift_config: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - bootstrap: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - db_drop: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - db_init: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" + ironic_manage_cleaning_network: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ironic_retrive_cleaning_network: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ironic_retrive_swift_config: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + bootstrap: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ironic_db_sync: "quay.io/rackspace/rackerlabs-ironic:2024.1-ubuntu_jammy" - ks_user: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_service: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_endpoints: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" + ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" ironic_api: "quay.io/rackspace/rackerlabs-ironic:2024.1-ubuntu_jammy" ironic_conductor: "quay.io/rackspace/rackerlabs-ironic:2024.1-ubuntu_jammy" @@ -24,7 +24,7 @@ images: ironic_pxe_http: "docker.io/nginx:1.13.3" # Retained from openstack-helm default ironic_inspector: "quay.io/rackspace/rackerlabs-ironic-inspector:2024.1-ubuntu_jammy" ironic_inspector_db_sync: "quay.io/rackspace/rackerlabs-ironic-inspector:2024.1-ubuntu_jammy" - dep_check: "quay.io/rackspace/rackerlabs-kubernetes-entrypoint:latest-ubuntu_jammy" + dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" pull_policy: "IfNotPresent" diff --git a/base-helm-configs/keystone/keystone-helm-overrides.yaml b/base-helm-configs/keystone/keystone-helm-overrides.yaml index f2c34a22d..2e1a1600d 100644 --- a/base-helm-configs/keystone/keystone-helm-overrides.yaml +++ b/base-helm-configs/keystone/keystone-helm-overrides.yaml @@ -1,20 +1,20 @@ --- images: tags: - bootstrap: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - db_drop: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - db_init: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - dep_check: "quay.io/rackspace/rackerlabs-kubernetes-entrypoint:latest-ubuntu_jammy" + bootstrap: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" - keystone_api: "quay.io/rackspace/rackerlabs-keystone-rxt:2024.1-ubuntu_jammy-1747958291" - keystone_credential_cleanup: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - keystone_credential_rotate: "quay.io/rackspace/rackerlabs-keystone-rxt:2024.1-ubuntu_jammy-1747958291" - keystone_credential_setup: "quay.io/rackspace/rackerlabs-keystone-rxt:2024.1-ubuntu_jammy-1747958291" - keystone_db_sync: "quay.io/rackspace/rackerlabs-keystone-rxt:2024.1-ubuntu_jammy-1747958291" - keystone_domain_manage: "quay.io/rackspace/rackerlabs-keystone-rxt:2024.1-ubuntu_jammy-1747958291" - keystone_fernet_rotate: "quay.io/rackspace/rackerlabs-keystone-rxt:2024.1-ubuntu_jammy-1747958291" - keystone_fernet_setup: "quay.io/rackspace/rackerlabs-keystone-rxt:2024.1-ubuntu_jammy-1747958291" - ks_user: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" + keystone_api: "ghcr.io/rackerlabs/genestack-images/keystone:2024.1-latest" + keystone_credential_cleanup: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + keystone_credential_rotate: "ghcr.io/rackerlabs/genestack-images/keystone:2024.1-latest" + keystone_credential_setup: "ghcr.io/rackerlabs/genestack-images/keystone:2024.1-latest" + keystone_db_sync: "ghcr.io/rackerlabs/genestack-images/keystone:2024.1-latest" + keystone_domain_manage: "ghcr.io/rackerlabs/genestack-images/keystone:2024.1-latest" + keystone_fernet_rotate: "ghcr.io/rackerlabs/genestack-images/keystone:2024.1-latest" + keystone_fernet_setup: "ghcr.io/rackerlabs/genestack-images/keystone:2024.1-latest" + ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" test: "quay.io/rackspace/rackerlabs-xrally-openstack:2.0.0" diff --git a/base-helm-configs/libvirt/libvirt-helm-overrides.yaml b/base-helm-configs/libvirt/libvirt-helm-overrides.yaml index fb36e8157..4e6b8c1de 100644 --- a/base-helm-configs/libvirt/libvirt-helm-overrides.yaml +++ b/base-helm-configs/libvirt/libvirt-helm-overrides.yaml @@ -1,9 +1,9 @@ --- images: tags: - libvirt: docker.io/openstackhelm/libvirt:2024.1-ubuntu_jammy + libvirt: ghcr.io/rackerlabs/genestack-images/libvirt:latest ceph_config_helper: docker.io/openstackhelm/ceph-config-helper:ubuntu_jammy_18.2.2-1-20240312 - dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_jammy + dep_check: ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest network: backend: - ovn diff --git a/base-helm-configs/magnum/magnum-helm-overrides.yaml b/base-helm-configs/magnum/magnum-helm-overrides.yaml index 047bf3481..38308b567 100644 --- a/base-helm-configs/magnum/magnum-helm-overrides.yaml +++ b/base-helm-configs/magnum/magnum-helm-overrides.yaml @@ -1,14 +1,14 @@ --- images: tags: - bootstrap: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - db_drop: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - db_init: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - dep_check: "quay.io/rackspace/rackerlabs-kubernetes-entrypoint:latest-ubuntu_jammy" + bootstrap: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" - ks_endpoints: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_service: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_user: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" + ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" magnum_api: "quay.io/rackspace/rackerlabs-magnum:2024.1-ubuntu_jammy" magnum_conductor: "quay.io/rackspace/rackerlabs-magnum:2024.1-ubuntu_jammy" magnum_db_sync: "quay.io/rackspace/rackerlabs-magnum:2024.1-ubuntu_jammy" diff --git a/base-helm-configs/masakari/masakari-helm-overrides.yaml b/base-helm-configs/masakari/masakari-helm-overrides.yaml index 530fc4e9a..2626ed545 100644 --- a/base-helm-configs/masakari/masakari-helm-overrides.yaml +++ b/base-helm-configs/masakari/masakari-helm-overrides.yaml @@ -13,12 +13,12 @@ --- images: tags: - db_init: quay.io/airshipit/heat:2024.1-ubuntu_jammy + db_init: ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest db_sync: docker.io/openstackhelm/masakari:2024.1-ubuntu_jammy - db_drop: quay.io/airshipit/heat:2024.1-ubuntu_jammy - ks_endpoints: quay.io/airshipit/heat:2024.1-ubuntu_jammy - ks_service: quay.io/airshipit/heat:2024.1-ubuntu_jammy - ks_user: quay.io/airshipit/heat:2024.1-ubuntu_jammy + db_drop: ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest + ks_endpoints: ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest + ks_service: ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest + ks_user: ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest masakari_api: quay.io/rackspace/rackerlabs-masakari:2024.1-ubuntu_jammy masakari_engine: quay.io/rackspace/rackerlabs-masakari:2024.1-ubuntu_jammy # TEMP HOST-MONITOR IMAGE TO FIX: https://review.opendev.org/c/openstack/masakari-monitors/+/951336 @@ -26,7 +26,7 @@ images: masakari_process_monitor: quay.io/rackspace/rackerlabs-masakari-monitors:2024.1-ubuntu_jammy masakari_instance_monitor: quay.io/rackspace/rackerlabs-masakari-monitors:2024.1-ubuntu_jammy rabbit_init: docker.io/rabbitmq:3.13-management - dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + dep_check: ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest pull_policy: "IfNotPresent" # NOTE: (brew) requests cpu/mem values based on a three node diff --git a/base-helm-configs/neutron/neutron-helm-overrides.yaml b/base-helm-configs/neutron/neutron-helm-overrides.yaml index 2ccf08bd0..d3507addf 100644 --- a/base-helm-configs/neutron/neutron-helm-overrides.yaml +++ b/base-helm-configs/neutron/neutron-helm-overrides.yaml @@ -1,34 +1,34 @@ --- images: tags: - bootstrap: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - db_init: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - db_drop: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_user: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_service: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_endpoints: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - neutron_db_sync: "quay.io/rackspace/rackerlabs-neutron:2024.1-ubuntu_jammy" - neutron_dhcp: "quay.io/rackspace/rackerlabs-neutron:2024.1-ubuntu_jammy" - neutron_l3: "quay.io/rackspace/rackerlabs-neutron:2024.1-ubuntu_jammy" - neutron_l2gw: "quay.io/rackspace/rackerlabs-neutron:2024.1-ubuntu_jammy" - neutron_linuxbridge_agent: "quay.io/rackspace/rackerlabs-neutron:2024.1-ubuntu_jammy" - neutron_metadata: "quay.io/rackspace/rackerlabs-neutron:2024.1-ubuntu_jammy" - neutron_ovn_metadata: "quay.io/rackspace/rackerlabs-neutron:2024.1-ubuntu_jammy" - neutron_ovn_vpn: "quay.io/rackspace/rackerlabs-neutron:2024.1-ubuntu_jammy" - neutron_openvswitch_agent: "quay.io/rackspace/rackerlabs-neutron:2024.1-ubuntu_jammy" - neutron_server: "quay.io/rackspace/rackerlabs-neutron:2024.1-ubuntu_jammy" - neutron_rpc_server: "quay.io/rackspace/rackerlabs-neutron:2024.1-ubuntu_jammy" - neutron_bagpipe_bgp: "quay.io/rackspace/rackerlabs-neutron:2024.1-ubuntu_jammy" - neutron_netns_cleanup_cron: "quay.io/rackspace/rackerlabs-neutron:2024.1-ubuntu_jammy" + bootstrap: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + neutron_db_sync: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" + neutron_dhcp: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" + neutron_l3: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" + neutron_l2gw: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" + neutron_linuxbridge_agent: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" + neutron_metadata: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" + neutron_ovn_metadata: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" + neutron_ovn_vpn: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" + neutron_openvswitch_agent: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" + neutron_server: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" + neutron_rpc_server: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" + neutron_bagpipe_bgp: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" + neutron_netns_cleanup_cron: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" test: "quay.io/rackspace/rackerlabs-xrally-openstack:2.0.0" purge_test: "quay.io/rackspace/rackerlabs-ospurge:latest" rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" netoffload: "quay.io/rackspace/rackerlabs-netoffload:v1.0.1" - neutron_sriov_agent: "quay.io/rackspace/rackerlabs-neutron:2024.1-ubuntu_jammy" - neutron_sriov_agent_init: "quay.io/rackspace/rackerlabs-neutron:2024.1-ubuntu_jammy" - neutron_bgp_dragent: "quay.io/rackspace/rackerlabs-neutron:2024.1-ubuntu_jammy" - neutron_ironic_agent: "quay.io/rackspace/rackerlabs-neutron:2024.1-ubuntu_jammy" - dep_check: "quay.io/rackspace/rackerlabs-kubernetes-entrypoint:latest-ubuntu_jammy" + neutron_sriov_agent: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" + neutron_sriov_agent_init: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" + neutron_bgp_dragent: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" + neutron_ironic_agent: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" + dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" labels: diff --git a/base-helm-configs/nova/nova-helm-overrides.yaml b/base-helm-configs/nova/nova-helm-overrides.yaml index 94c29136d..f201671d4 100644 --- a/base-helm-configs/nova/nova-helm-overrides.yaml +++ b/base-helm-configs/nova/nova-helm-overrides.yaml @@ -1,29 +1,29 @@ --- images: tags: - bootstrap: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - db_drop: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - db_init: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - dep_check: "quay.io/rackspace/rackerlabs-kubernetes-entrypoint:latest-ubuntu_jammy" + bootstrap: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" - ks_endpoints: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_service: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_user: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - nova_api: "quay.io/rackspace/rackerlabs-nova-efi:2024.1-ubuntu_jammy-1737928811" - nova_archive_deleted_rows: "quay.io/rackspace/rackerlabs-nova-efi:2024.1-ubuntu_jammy-1737928811" - nova_cell_setup: "quay.io/rackspace/rackerlabs-nova-efi:2024.1-ubuntu_jammy-1737928811" - nova_cell_setup_init: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - nova_compute: "quay.io/rackspace/rackerlabs-nova-efi:2024.1-ubuntu_jammy-1737928811" + ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + nova_api: "ghcr.io/rackerlabs/genestack-images/nova:2024.1-latest" + nova_archive_deleted_rows: "ghcr.io/rackerlabs/genestack-images/nova:2024.1-latest" + nova_cell_setup: "ghcr.io/rackerlabs/genestack-images/nova:2024.1-latest" + nova_cell_setup_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + nova_compute: "ghcr.io/rackerlabs/genestack-images/nova:2024.1-latest" nova_compute_ironic: "docker.io/kolla/ubuntu-source-nova-compute-ironic:wallaby" - nova_compute_ssh: "quay.io/rackspace/rackerlabs-nova-efi:2024.1-ubuntu_jammy-1737928811" - nova_conductor: "quay.io/rackspace/rackerlabs-nova-efi:2024.1-ubuntu_jammy-1737928811" - nova_db_sync: "quay.io/rackspace/rackerlabs-nova-efi:2024.1-ubuntu_jammy-1737928811" - nova_novncproxy: "quay.io/rackspace/rackerlabs-nova-efi:2024.1-ubuntu_jammy-1737928811" - nova_novncproxy_assets: "quay.io/rackspace/rackerlabs-nova-efi:2024.1-ubuntu_jammy-1737928811" - nova_scheduler: "quay.io/rackspace/rackerlabs-nova-efi:2024.1-ubuntu_jammy-1737928811" + nova_compute_ssh: "ghcr.io/rackerlabs/genestack-images/nova:2024.1-latest" + nova_conductor: "ghcr.io/rackerlabs/genestack-images/nova:2024.1-latest" + nova_db_sync: "ghcr.io/rackerlabs/genestack-images/nova:2024.1-latest" + nova_novncproxy: "ghcr.io/rackerlabs/genestack-images/nova:2024.1-latest" + nova_novncproxy_assets: "ghcr.io/rackerlabs/genestack-images/nova:2024.1-latest" + nova_scheduler: "ghcr.io/rackerlabs/genestack-images/nova:2024.1-latest" nova_service_cleaner: "quay.io/rackspace/rackerlabs-ceph-config-helper:latest-ubuntu_jammy" - nova_spiceproxy: "quay.io/rackspace/rackerlabs-nova-efi:2024.1-ubuntu_jammy-1737928811" - nova_spiceproxy_assets: "quay.io/rackspace/rackerlabs-nova-efi:2024.1-ubuntu_jammy-1737928811" + nova_spiceproxy: "ghcr.io/rackerlabs/genestack-images/nova:2024.1-latest" + nova_spiceproxy_assets: "ghcr.io/rackerlabs/genestack-images/nova:2024.1-latest" nova_storage_init: "quay.io/rackspace/rackerlabs-ceph-config-helper:latest-ubuntu_jammy" nova_wait_for_computes_init: "quay.io/rackspace/rackerlabs-hyperkube-amd64:v1.11.6" rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" diff --git a/base-helm-configs/octavia/octavia-helm-overrides.yaml b/base-helm-configs/octavia/octavia-helm-overrides.yaml index d7d65b2f6..c3903a78b 100644 --- a/base-helm-configs/octavia/octavia-helm-overrides.yaml +++ b/base-helm-configs/octavia/octavia-helm-overrides.yaml @@ -1,20 +1,20 @@ --- images: tags: - bootstrap: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - db_drop: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - db_init: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - dep_check: "quay.io/rackspace/rackerlabs-kubernetes-entrypoint:latest-ubuntu_jammy" + bootstrap: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" - ks_endpoints: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_service: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_user: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - octavia_api: "quay.io/rackspace/rackerlabs-octavia-ovn:2024.1-ubuntu_jammy-1737651745" - octavia_db_sync: "quay.io/rackspace/rackerlabs-octavia-ovn:2024.1-ubuntu_jammy-1737651745" - octavia_health_manager: "quay.io/rackspace/rackerlabs-octavia-ovn:2024.1-ubuntu_jammy-1737651745" - octavia_health_manager_init: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - octavia_housekeeping: "quay.io/rackspace/rackerlabs-octavia-ovn:2024.1-ubuntu_jammy-1737651745" - octavia_worker: "quay.io/rackspace/rackerlabs-octavia-ovn:2024.1-ubuntu_jammy-1737651745" + ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + octavia_api: "ghcr.io/rackerlabs/genestack-images/octavia:2024.1-latest" + octavia_db_sync: "ghcr.io/rackerlabs/genestack-images/octavia:2024.1-latest" + octavia_health_manager: "ghcr.io/rackerlabs/genestack-images/octavia:2024.1-latest" + octavia_health_manager_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + octavia_housekeeping: "ghcr.io/rackerlabs/genestack-images/octavia:2024.1-latest" + octavia_worker: "ghcr.io/rackerlabs/genestack-images/octavia:2024.1-latest" openvswitch_vswitchd: "docker.io/kolla/centos-source-openvswitch-vswitchd:rocky" rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" test: "quay.io/rackspace/rackerlabs-xrally-openstack:2.0.0" diff --git a/base-helm-configs/placement/placement-helm-overrides.yaml b/base-helm-configs/placement/placement-helm-overrides.yaml index 8b9fa436b..0a5c3fb7b 100644 --- a/base-helm-configs/placement/placement-helm-overrides.yaml +++ b/base-helm-configs/placement/placement-helm-overrides.yaml @@ -1,15 +1,15 @@ --- images: tags: - db_drop: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - db_init: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - dep_check: "quay.io/rackspace/rackerlabs-kubernetes-entrypoint:latest-ubuntu_jammy" + db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" - ks_endpoints: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_service: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - ks_user: "quay.io/rackspace/rackerlabs-heat:2024.1-ubuntu_jammy" - placement: "quay.io/rackspace/rackerlabs-placement:2024.1-ubuntu_jammy" - placement_db_sync: "quay.io/rackspace/rackerlabs-placement:2024.1-ubuntu_jammy" + ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" + placement: "ghcr.io/rackerlabs/genestack-images/placement:2024.1-latest" + placement_db_sync: "ghcr.io/rackerlabs/genestack-images/placement:2024.1-latest" # NOTE: (brew) requests cpu/mem values based on a three node # hyperconverged lab (/scripts/hyperconverged-lab.sh). diff --git a/base-kustomize/cinder/netapp/kustomization.yaml b/base-kustomize/cinder/netapp/kustomization.yaml index 91f2e9c4d..eb6cddc10 100644 --- a/base-kustomize/cinder/netapp/kustomization.yaml +++ b/base-kustomize/cinder/netapp/kustomization.yaml @@ -1,10 +1,10 @@ images: - name: image-kubernetes-entrypoint-init - newName: quay.io/airshipit/kubernetes-entrypoint - newTag: v1.0.0 + newName: ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint + newTag: latest - name: image-heat-conf-init - newName: docker.io/openstackhelm/heat - newTag: 2024.1-ubuntu_jammy + newName: ghcr.io/rackerlabs/genestack-images/heat + newTag: 2024.1-latest - name: image-cinder-volume-netapp-init newName: ghcr.io/rackerlabs/genestack/cinder-volume-rxt newTag: 2024.1-ubuntu_jammy diff --git a/base-kustomize/octavia/base/kustomization.yaml b/base-kustomize/octavia/base/kustomization.yaml index e6c15ac33..0368e3d8c 100644 --- a/base-kustomize/octavia/base/kustomization.yaml +++ b/base-kustomize/octavia/base/kustomization.yaml @@ -4,11 +4,11 @@ sortOptions: images: - name: image-kubernetes-entrypoint-init - newName: quay.io/airshipit/kubernetes-entrypoint - newTag: v1.0.0 + newName: ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint + newTag: latest - name: image-octavia-ovn - newName: quay.io/rackspace/rackerlabs-octavia-ovn - newTag: 2024.1-ubuntu_jammy-1737651745 + newName: ghcr.io/rackerlabs/genestack-images/octavia + newTag: 2024.1-latest resources: - octavia-mariadb-database.yaml diff --git a/base-kustomize/ovn/base/ovn-setup.yaml b/base-kustomize/ovn/base/ovn-setup.yaml index bbb539b9e..94cf706e2 100644 --- a/base-kustomize/ovn/base/ovn-setup.yaml +++ b/base-kustomize/ovn/base/ovn-setup.yaml @@ -140,7 +140,7 @@ spec: emptyDir: {} initContainers: - name: init - image: "quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_jammy" + image: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" imagePullPolicy: IfNotPresent securityContext: allowPrivilegeEscalation: true @@ -213,7 +213,7 @@ spec: - 'echo "$$SCRIPT" > /tmp/script && ash /tmp/script' containers: - name: ovn-setup-exec - image: "docker.io/openstackhelm/ovn:ubuntu_jammy" + image: "ghcr.io/rackerlabs/genestack-images/ovs:v3.5.1-latest" imagePullPolicy: IfNotPresent command: - bash diff --git a/base-kustomize/skyline/base/deployment-apiserver.yaml b/base-kustomize/skyline/base/deployment-apiserver.yaml index 8145908b5..32784aef9 100644 --- a/base-kustomize/skyline/base/deployment-apiserver.yaml +++ b/base-kustomize/skyline/base/deployment-apiserver.yaml @@ -64,7 +64,7 @@ spec: defaultMode: 0555 initContainers: - name: init - image: "quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_jammy" + image: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" imagePullPolicy: IfNotPresent securityContext: allowPrivilegeEscalation: true @@ -101,7 +101,7 @@ spec: - kubernetes-entrypoint volumeMounts: [] - name: skyline-apiserver-service-init - image: "docker.io/openstackhelm/heat:2023.1-ubuntu_jammy" + image: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" imagePullPolicy: IfNotPresent resources: limits: diff --git a/manifests/utils/utils-openstack-client-admin.yaml b/manifests/utils/utils-openstack-client-admin.yaml index 4351d8529..b45d2cf51 100644 --- a/manifests/utils/utils-openstack-client-admin.yaml +++ b/manifests/utils/utils-openstack-client-admin.yaml @@ -7,7 +7,7 @@ spec: restartPolicy: Always containers: - name: "image-ks-service-registration" - image: docker.io/openstackhelm/heat:2023.1-ubuntu_jammy + image: ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest imagePullPolicy: IfNotPresent command: - sleep From 752c4cdbaa092302e99ac55efc37c1e2e1cf4f25 Mon Sep 17 00:00:00 2001 From: manojacloud <112391328+manojacloud@users.noreply.github.com> Date: Mon, 14 Jul 2025 17:08:20 +0530 Subject: [PATCH 16/97] chore: Add freezer images to genestack repo (#1070) Adding freezer images to genestack repo so that we can use these images to setup freezer. --- .original-images.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.original-images.json b/.original-images.json index ae10dc7e7..3961464ef 100644 --- a/.original-images.json +++ b/.original-images.json @@ -44,5 +44,7 @@ "ghcr.io/rackerlabs/skyline-rxt:master-ubuntu_jammy-1748595671", "ghcr.io/vexxhost/netoffload:v1.0.1", "quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_jammy", - "quay.io/airshipit/porthole-postgresql-utility:latest-ubuntu_bionic" + "quay.io/airshipit/porthole-postgresql-utility:latest-ubuntu_bionic", + "quay.io/airshipit/freezer:2025.1-ubuntu_jammy", + "quay.io/airshipit/freezer-api:2025.1-ubuntu_jammy" ] From 0e594faa3b05e3206d3ed4d734a18a4f22fc0208 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Tue, 15 Jul 2025 17:01:57 -0500 Subject: [PATCH 17/97] chore: update urls for federeated logins (#1071) Signed-off-by: Kevin Carter --- mkdocs.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index c6343fea2..dfeec73cd 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -346,12 +346,12 @@ nav: - Blog: https://blog.rackspacecloud.com/blog - Regions: - Availability: api-status.md - - SJC3: + - SJC: - "": https://status.api.sjc3.rackspacecloud.com - - "Control Panel": https://skyline.api.sjc3.rackspacecloud.com - - DFW3: + - "Control Panel": https://keystone.api.sjc3.rackspacecloud.com/v3/auth/OS-FEDERATION/websso/saml2?origin=https://skyline.api.sjc3.rackspacecloud.com/api/openstack/skyline/api/v1/websso + - DFW: - "": https://status.api.dfw3.rackspacecloud.com - - "Control Panel": https://skyline.api.dfw3.rackspacecloud.com - - IAD3: + - "Control Panel": https://keystone.api.dfw3.rackspacecloud.com/v3/auth/OS-FEDERATION/websso/saml2?origin=https://skyline.api.dfw3.rackspacecloud.com/api/openstack/skyline/api/v1/websso + - IAD: - "": https://status.api.iad3.rackspacecloud.com - - "Control Panel": https://skyline.api.iad3.rackspacecloud.com + - "Control Panel": https://keystone.api.iad3.rackspacecloud.com/v3/auth/OS-FEDERATION/websso/saml2?origin=https://skyline.api.iad3.rackspacecloud.com/api/openstack/skyline/api/v1/websso From d3e0ce3a62ed06f881472d6d431da05abbf124de Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Tue, 15 Jul 2025 17:40:19 -0500 Subject: [PATCH 18/97] chore: add lifecycle management for all of the openstack services (#1072) This will ensure that the lifecycle of the pods has a consistent install and upgrade experience. Signed-off-by: Kevin Carter --- .../barbican/barbican-helm-overrides.yaml | 8 +++ .../ceilometer/ceilometer-helm-overrides.yaml | 8 ++- .../cinder/cinder-helm-overrides.yaml | 14 +++++ .../designate/designate-helm-overrides.yaml | 6 +- .../glance/glance-helm-overrides.yaml | 4 +- .../gnocchi/gnocchi-helm-overrides.yaml | 20 ++++++- .../heat/heat-helm-overrides.yaml | 24 ++++++++ .../keystone/keystone-helm-overrides.yaml | 14 +++++ .../libvirt/libvirt-helm-overrides.yaml | 9 +++ .../magnum/magnum-helm-overrides.yaml | 14 +++++ .../masakari/masakari-helm-overrides.yaml | 24 ++++++++ .../neutron/neutron-helm-overrides.yaml | 56 +++++++++++++++++++ .../nova/nova-helm-overrides.yaml | 18 ++++++ .../octavia/octavia-helm-overrides.yaml | 20 +++++++ .../placement/placement-helm-overrides.yaml | 14 +++++ 15 files changed, 244 insertions(+), 9 deletions(-) diff --git a/base-helm-configs/barbican/barbican-helm-overrides.yaml b/base-helm-configs/barbican/barbican-helm-overrides.yaml index 7ad2479cc..4cf368007 100644 --- a/base-helm-configs/barbican/barbican-helm-overrides.yaml +++ b/base-helm-configs/barbican/barbican-helm-overrides.yaml @@ -25,6 +25,14 @@ pod: memory: "256Mi" cpu: "100m" limits: {} + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 20% + max_surge: 3 dependencies: static: diff --git a/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml b/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml index 19e8efd5d..00ff62995 100644 --- a/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml +++ b/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml @@ -1813,7 +1813,7 @@ pod: revision_history: 3 pod_replacement_strategy: RollingUpdate rolling_update: - max_unavailable: 1 + max_unavailable: 20% max_surge: 3 daemonsets: pod_replacement_strategy: RollingUpdate @@ -1821,6 +1821,12 @@ pod: enabled: true min_ready_seconds: 0 max_unavailable: 20% + disruption_budget: + api: + min_available: 0 + termination_grace_period: + api: + timeout: 600 manifests: deployment_api: false diff --git a/base-helm-configs/cinder/cinder-helm-overrides.yaml b/base-helm-configs/cinder/cinder-helm-overrides.yaml index 0b991e40c..bdd0222dd 100644 --- a/base-helm-configs/cinder/cinder-helm-overrides.yaml +++ b/base-helm-configs/cinder/cinder-helm-overrides.yaml @@ -30,6 +30,20 @@ images: # hyperconverged lab (/scripts/hyperconverged-lab.sh). # limit values based on defaults from the openstack-helm charts unless defined pod: + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 20% + max_surge: 3 + disruption_budget: + api: + min_available: 0 + termination_grace_period: + api: + timeout: 60 resources: enabled: true api: diff --git a/base-helm-configs/designate/designate-helm-overrides.yaml b/base-helm-configs/designate/designate-helm-overrides.yaml index 83ee64287..341e04787 100644 --- a/base-helm-configs/designate/designate-helm-overrides.yaml +++ b/base-helm-configs/designate/designate-helm-overrides.yaml @@ -121,7 +121,7 @@ pod: revision_history: 3 pod_replacement_strategy: RollingUpdate rolling_update: - max_unavailable: 1 + max_unavailable: 20% max_surge: 3 disruption_budget: api: @@ -138,9 +138,9 @@ pod: min_available: 0 termination_grace_period: api: - timeout: 30 + timeout: 60 mdns: - timeout: 30 + timeout: 60 resources: enabled: true api: diff --git a/base-helm-configs/glance/glance-helm-overrides.yaml b/base-helm-configs/glance/glance-helm-overrides.yaml index 6e4612707..a6791b836 100644 --- a/base-helm-configs/glance/glance-helm-overrides.yaml +++ b/base-helm-configs/glance/glance-helm-overrides.yaml @@ -255,14 +255,14 @@ pod: revision_history: 3 pod_replacement_strategy: RollingUpdate rolling_update: - max_unavailable: 1 + max_unavailable: 20% max_surge: 3 disruption_budget: api: min_available: 0 termination_grace_period: api: - timeout: 30 + timeout: 60 probes: api: glance-api: diff --git a/base-helm-configs/gnocchi/gnocchi-helm-overrides.yaml b/base-helm-configs/gnocchi/gnocchi-helm-overrides.yaml index b78923eeb..d49a4618c 100644 --- a/base-helm-configs/gnocchi/gnocchi-helm-overrides.yaml +++ b/base-helm-configs/gnocchi/gnocchi-helm-overrides.yaml @@ -100,14 +100,28 @@ pod: limits: {} lifecycle: upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 20% + max_surge: 3 daemonsets: + pod_replacement_strategy: RollingUpdate metricd: - enabled: true + enabled: false + min_ready_seconds: 0 max_unavailable: 20% - pod_replacement_strategy: RollingUpdate statsd: - enabled: true + enabled: false + min_ready_seconds: 0 max_unavailable: 20% + disruption_budget: + api: + min_available: 0 + termination_grace_period: + api: + timeout: 60 endpoints: fluentd: diff --git a/base-helm-configs/heat/heat-helm-overrides.yaml b/base-helm-configs/heat/heat-helm-overrides.yaml index 63611b83f..47e4b0688 100644 --- a/base-helm-configs/heat/heat-helm-overrides.yaml +++ b/base-helm-configs/heat/heat-helm-overrides.yaml @@ -23,6 +23,30 @@ images: # hyperconverged lab (/scripts/hyperconverged-lab.sh). # limit values based on defaults from the openstack-helm charts unless defined pod: + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 20% + max_surge: 3 + disruption_budget: + api: + min_available: 0 + cfn: + min_available: 0 + cloudwatch: + min_available: 0 + termination_grace_period: + api: + timeout: 60 + cfn: + timeout: 60 + cloudwatch: + timeout: 60 + engine: + timeout: 60 resources: enabled: true api: diff --git a/base-helm-configs/keystone/keystone-helm-overrides.yaml b/base-helm-configs/keystone/keystone-helm-overrides.yaml index 2e1a1600d..28e3537e9 100644 --- a/base-helm-configs/keystone/keystone-helm-overrides.yaml +++ b/base-helm-configs/keystone/keystone-helm-overrides.yaml @@ -22,6 +22,20 @@ images: # hyperconverged lab (/scripts/hyperconverged-lab.sh). # limit values based on defaults from the openstack-helm charts unless defined pod: + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 20% + max_surge: 3 + disruption_budget: + api: + min_available: 0 + termination_grace_period: + api: + timeout: 60 resources: enabled: true api: diff --git a/base-helm-configs/libvirt/libvirt-helm-overrides.yaml b/base-helm-configs/libvirt/libvirt-helm-overrides.yaml index 4e6b8c1de..1ad90d51d 100644 --- a/base-helm-configs/libvirt/libvirt-helm-overrides.yaml +++ b/base-helm-configs/libvirt/libvirt-helm-overrides.yaml @@ -18,3 +18,12 @@ dependencies: ovn: libvirt: pod: [] # In a hybrid deployment, we don't want to run ovn-controller on the same node as libvirt +pod: + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + libvirt: + enabled: true + min_ready_seconds: 0 + max_unavailable: 20% diff --git a/base-helm-configs/magnum/magnum-helm-overrides.yaml b/base-helm-configs/magnum/magnum-helm-overrides.yaml index 38308b567..2f22d89e7 100644 --- a/base-helm-configs/magnum/magnum-helm-overrides.yaml +++ b/base-helm-configs/magnum/magnum-helm-overrides.yaml @@ -18,6 +18,20 @@ images: # hyperconverged lab (/scripts/hyperconverged-lab.sh). # limit values based on defaults from the openstack-helm charts unless defined pod: + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 20% + max_surge: 3 + disruption_budget: + api: + min_available: 0 + termination_grace_period: + api: + timeout: 60 resources: enabled: true api: diff --git a/base-helm-configs/masakari/masakari-helm-overrides.yaml b/base-helm-configs/masakari/masakari-helm-overrides.yaml index 2626ed545..4c4323058 100644 --- a/base-helm-configs/masakari/masakari-helm-overrides.yaml +++ b/base-helm-configs/masakari/masakari-helm-overrides.yaml @@ -33,6 +33,30 @@ images: # hyperconverged lab (/scripts/hyperconverged-lab.sh). # limit values based on defaults from the openstack-helm charts unless defined pod: + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 20% + max_surge: 3 + daemonsets: + pod_replacement_strategy: RollingUpdate + compute: + enabled: true + min_ready_seconds: 0 + max_unavailable: 20% + disruption_budget: + masakari_api: + min_available: 0 + masakari_engine: + min_available: 0 + termination_grace_period: + masakari_api: + timeout: 60 + masakari_engine: + timeout: 60 resources: enabled: true masakari_api: diff --git a/base-helm-configs/neutron/neutron-helm-overrides.yaml b/base-helm-configs/neutron/neutron-helm-overrides.yaml index d3507addf..7d3b714ff 100644 --- a/base-helm-configs/neutron/neutron-helm-overrides.yaml +++ b/base-helm-configs/neutron/neutron-helm-overrides.yaml @@ -86,6 +86,62 @@ pod: cpu: "3000m" use_fqdn: neutron_agent: false + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 20% + max_surge: 3 + daemonsets: + pod_replacement_strategy: RollingUpdate + dhcp_agent: + enabled: true + min_ready_seconds: 0 + max_unavailable: 20% + l3_agent: + enabled: true + min_ready_seconds: 0 + max_unavailable: 20% + lb_agent: + enabled: true + min_ready_seconds: 0 + max_unavailable: 20% + metadata_agent: + enabled: true + min_ready_seconds: 0 + max_unavailable: 20% + ovn_metadata_agent: + enabled: true + min_ready_seconds: 0 + max_unavailable: 20% + ovn_vpn_agent: + enabled: true + min_ready_seconds: 0 + max_unavailable: 20% + ovs_agent: + enabled: true + min_ready_seconds: 0 + max_unavailable: 20% + sriov_agent: + enabled: true + min_ready_seconds: 0 + max_unavailable: 20% + netns_cleanup_cron: + enabled: true + min_ready_seconds: 0 + max_unavailable: 20% + disruption_budget: + server: + min_available: 0 + termination_grace_period: + server: + timeout: 60 + rpc_server: + timeout: 60 + ironic_agent: + timeout: 60 conf: dhcp_agent: diff --git a/base-helm-configs/nova/nova-helm-overrides.yaml b/base-helm-configs/nova/nova-helm-overrides.yaml index f201671d4..2912fd1c8 100644 --- a/base-helm-configs/nova/nova-helm-overrides.yaml +++ b/base-helm-configs/nova/nova-helm-overrides.yaml @@ -34,10 +34,28 @@ network: - ovn lifecycle: upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 20% + max_surge: 3 daemonsets: + pod_replacement_strategy: RollingUpdate compute: enabled: true + min_ready_seconds: 0 max_unavailable: 20% + disruption_budget: + metadata: + min_available: 0 + osapi: + min_available: 0 + termination_grace_period: + metadata: + timeout: 60 + osapi: + timeout: 60 ssh: enabled: true diff --git a/base-helm-configs/octavia/octavia-helm-overrides.yaml b/base-helm-configs/octavia/octavia-helm-overrides.yaml index c3903a78b..efd338a00 100644 --- a/base-helm-configs/octavia/octavia-helm-overrides.yaml +++ b/base-helm-configs/octavia/octavia-helm-overrides.yaml @@ -181,6 +181,26 @@ endpoints: # hyperconverged lab (/scripts/hyperconverged-lab.sh). # limit values based on defaults from the openstack-helm charts unless defined pod: + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 20% + max_surge: 3 + daemonsets: + pod_replacement_strategy: RollingUpdate + health_manager: + enabled: true + min_ready_seconds: 0 + max_unavailable: 20% + disruption_budget: + api: + min_available: 0 + termination_grace_period: + api: + timeout: 60 resources: enabled: true api: diff --git a/base-helm-configs/placement/placement-helm-overrides.yaml b/base-helm-configs/placement/placement-helm-overrides.yaml index 0a5c3fb7b..541cf81dc 100644 --- a/base-helm-configs/placement/placement-helm-overrides.yaml +++ b/base-helm-configs/placement/placement-helm-overrides.yaml @@ -15,6 +15,20 @@ images: # hyperconverged lab (/scripts/hyperconverged-lab.sh). # limit values based on defaults from the openstack-helm charts unless defined pod: + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 20% + max_surge: 3 + disruption_budget: + api: + min_available: 0 + termination_grace_period: + api: + timeout: 60 resources: enabled: true api: From 85128c870cb32cd2001693d504e44d41153e17f8 Mon Sep 17 00:00:00 2001 From: Bjoern Teipel Date: Tue, 15 Jul 2025 17:41:55 -0500 Subject: [PATCH 19/97] Fixing horizon secret key (#1068) The shared secret for horizon is now properly configured and referenced --- bin/create-secrets.sh | 5 ++--- bin/install-horizon.sh | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/bin/create-secrets.sh b/bin/create-secrets.sh index dae05fb2b..e0c73c2fc 100755 --- a/bin/create-secrets.sh +++ b/bin/create-secrets.sh @@ -66,7 +66,7 @@ designate_admin_password=$(generate_password 32) neutron_rabbitmq_password=$(generate_password 64) neutron_db_password=$(generate_password 32) neutron_admin_password=$(generate_password 32) -horizon_secret_key_password=$(generate_password 64) +horizon_secret_key=$(generate_password 64) horizon_db_password=$(generate_password 32) skyline_service_password=$(generate_password 32) skyline_db_password=$(generate_password 32) @@ -390,8 +390,7 @@ metadata: namespace: openstack type: Opaque data: - username: $(echo -n "horizon" | base64) - password: $(echo -n $horizon_secret_key_password | base64 -w0) + horizon_secret_key: $(echo -n $horizon_secret_key | base64 -w0) --- apiVersion: v1 kind: Secret diff --git a/bin/install-horizon.sh b/bin/install-horizon.sh index be49ad4e4..2cd9968da 100755 --- a/bin/install-horizon.sh +++ b/bin/install-horizon.sh @@ -23,7 +23,7 @@ done HELM_CMD+=" --set endpoints.identity.auth.admin.password=\"\$(kubectl --namespace openstack get secret keystone-admin -o jsonpath='{.data.password}' | base64 -d)\"" HELM_CMD+=" --set endpoints.oslo_cache.auth.memcache_secret_key=\"\$(kubectl --namespace openstack get secret os-memcached -o jsonpath='{.data.memcache_secret_key}' | base64 -d)\"" -HELM_CMD+=" --set conf.horizon.local_settings.config.horizon_secret_key=\"\$(kubectl --namespace openstack get secret horizon-secret-key -o jsonpath='{.data.root-password}' | base64 -d)\"" +HELM_CMD+=" --set conf.horizon.local_settings.config.horizon_secret_key=\"\$(kubectl --namespace openstack get secret horizon-secret-key -o jsonpath='{.data.horizon_secret_key}' | base64 -d)\"" HELM_CMD+=" --set endpoints.oslo_db.auth.admin.password=\"\$(kubectl --namespace openstack get secret mariadb -o jsonpath='{.data.root-password}' | base64 -d)\"" HELM_CMD+=" --set endpoints.oslo_db.auth.horizon.password=\"\$(kubectl --namespace openstack get secret horizon-db-password -o jsonpath='{.data.password}' | base64 -d)\"" From 1550f7594fbc6630e95ffe526d6ff99f023c38bc Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Tue, 15 Jul 2025 17:44:20 -0500 Subject: [PATCH 20/97] fix: nova schedule does not equal nova scheduler (#1073) --- base-helm-configs/nova/nova-helm-overrides.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base-helm-configs/nova/nova-helm-overrides.yaml b/base-helm-configs/nova/nova-helm-overrides.yaml index 2912fd1c8..fb13c1642 100644 --- a/base-helm-configs/nova/nova-helm-overrides.yaml +++ b/base-helm-configs/nova/nova-helm-overrides.yaml @@ -190,7 +190,7 @@ conf: # https://lists.openstack.org/pipermail/openstack-discuss/2023-April/033314.html # https://review.opendev.org/c/openstack/oslo.messaging/+/866617 kombu_reconnect_delay: 0.5 - schedule: + scheduler: workers: 2 workarounds: skip_cpu_compare_at_startup: false From 3a468b7e40f32feab73e59ca8b4c32746b137488 Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Tue, 15 Jul 2025 19:03:04 -0500 Subject: [PATCH 21/97] fix: correct gnocci baseline (#1074) --- base-helm-configs/gnocchi/gnocchi-helm-overrides.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base-helm-configs/gnocchi/gnocchi-helm-overrides.yaml b/base-helm-configs/gnocchi/gnocchi-helm-overrides.yaml index d49a4618c..a1e1bf343 100644 --- a/base-helm-configs/gnocchi/gnocchi-helm-overrides.yaml +++ b/base-helm-configs/gnocchi/gnocchi-helm-overrides.yaml @@ -45,7 +45,7 @@ conf: gnocchi_api_wsgi: wsgi: processes: 2 - threads: 4 + threads: 1 paste: "app:gnocchiv1": paste.app_factory: "gnocchi.rest.app:app_factory" From b6ba574a2ff2f68c3e3e85580f4c74d842583032 Mon Sep 17 00:00:00 2001 From: Jake Briggs Date: Tue, 15 Jul 2025 19:45:08 -0500 Subject: [PATCH 22/97] Update install-envoy-gateway.sh (#1075) --- bin/install-envoy-gateway.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/install-envoy-gateway.sh b/bin/install-envoy-gateway.sh index f3102550a..ead616d5a 100755 --- a/bin/install-envoy-gateway.sh +++ b/bin/install-envoy-gateway.sh @@ -4,7 +4,7 @@ GLOBAL_OVERRIDES_DIR="/etc/genestack/helm-configs/global_overrides" SERVICE_CONFIG_DIR="/etc/genestack/helm-configs/envoyproxy-gateway" BASE_OVERRIDES="/opt/genestack/base-helm-configs/envoyproxy-gateway/envoy-gateway-helm-overrides.yaml" -ENVOY_VERSION="v1.3.0" +ENVOY_VERSION="v1.4.2" HELM_CMD="helm upgrade --install envoyproxy-gateway oci://docker.io/envoyproxy/gateway-helm \ --version ${ENVOY_VERSION} \ --namespace envoyproxy-gateway-system \ From 942dee5070bd28f6a66d5e8b937c812611fa30c4 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Wed, 16 Jul 2025 18:51:24 -0500 Subject: [PATCH 23/97] feat: Add new RHEL OS to Docs Signed-off-by: Kevin Carter --- docs/openstack-glance-images.md | 39 ++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/docs/openstack-glance-images.md b/docs/openstack-glance-images.md index 90c64cf9d..0606a3553 100644 --- a/docs/openstack-glance-images.md +++ b/docs/openstack-glance-images.md @@ -369,11 +369,40 @@ openstack --os-cloud default image create \ SLES15-SP6 ``` -## Get RHEL +## Get Red Hat Enterprise Linux (RHEL) + +### RHEL 9 + +!!! note + + Make sure you download the latest available image from [here](https://access.redhat.com/downloads/content/479/ver=/rhel---9/9.6/x86_64/product-software). We used the `rhel-9.6-x86_64-kvm.qcow2` image. + +``` shell +openstack --os-cloud default image create \ + --progress \ + --disk-format qcow2 \ + --container-format bare \ + --public \ + --file rhel-9.6-x86_64-kvm.qcow2 \ + --property hw_vif_multiqueue_enabled=true \ + --property hw_qemu_guest_agent=yes \ + --property hypervisor_type=kvm \ + --property img_config_drive=optional \ + --property hw_machine_type=q35 \ + --property hw_firmware_type=uefi \ + --property os_require_quiesce=yes \ + --property os_type=linux \ + --property os_admin_user=cloud-user \ + --property os_distro=rhel \ + --property os_version=9.6 \ + RHEL-9.6 +``` + +### RHEL 10 !!! note - Make sure you download the latest available image from [here](https://access.redhat.com/downloads/content/479/ver=/rhel---9/9.4/x86_64/product-software). We used the rhel-9.4-x86_64-kvm.qcow2 image. + Make sure you download the latest available image from [here](https://access.redhat.com/downloads/content/479/ver=/rhel---10/10.0/x86_64/product-software). We used the `rhel-10.0-x86_64-kvm.qcow2` image. ``` shell openstack --os-cloud default image create \ @@ -381,7 +410,7 @@ openstack --os-cloud default image create \ --disk-format qcow2 \ --container-format bare \ --public \ - --file rhel-9.4-x86_64-kvm.qcow2 \ + --file rhel-10.0-x86_64-kvm.qcow2 \ --property hw_vif_multiqueue_enabled=true \ --property hw_qemu_guest_agent=yes \ --property hypervisor_type=kvm \ @@ -392,8 +421,8 @@ openstack --os-cloud default image create \ --property os_type=linux \ --property os_admin_user=cloud-user \ --property os_distro=rhel \ - --property os_version=9.4 \ - RHEL-9.4 + --property os_version=10.0 \ + RHEL-10.0 ``` ## Get Windows From 321a0607112faffa83ab6f2ad541f930e3f31b0d Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Fri, 18 Jul 2025 09:17:16 -0500 Subject: [PATCH 24/97] fix: only define values we are overriding (#1077) * fix: only define values we are overriding * fix: whitespace --- .../ceilometer/ceilometer-helm-overrides.yaml | 29 +++---------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml b/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml index 00ff62995..4965c4a2c 100644 --- a/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml +++ b/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml @@ -1776,34 +1776,13 @@ endpoints: pod: resources: enabled: true - compute: - requests: - memory: {} - cpu: {} - limits: - memory: {} - cpu: {} notification: requests: - memory: {} - cpu: {} - limits: - memory: {} - cpu: {} - central: - requests: - memory: {} - cpu: {} - limits: - memory: {} - cpu: {} - ipmi: - requests: - memory: {} - cpu: {} + memory: 256 + cpu: 500 limits: - memory: {} - cpu: {} + memory: 2Gi + cpu: "2000m" replicas: central: 1 notification: 1 From faff884e133f63273773edcda1206ebc01c2f11c Mon Sep 17 00:00:00 2001 From: "phillip.toohill" Date: Fri, 18 Jul 2025 13:44:19 -0500 Subject: [PATCH 25/97] fix: Updating mysql-exporter for correct secret config and updating docs (#1079) --- .../prometheus-mysql-exporter/values.yaml | 28 +++++++++---------- docs/prometheus-mysql-exporter.md | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/base-helm-configs/prometheus-mysql-exporter/values.yaml b/base-helm-configs/prometheus-mysql-exporter/values.yaml index b6718430e..5d2205c69 100644 --- a/base-helm-configs/prometheus-mysql-exporter/values.yaml +++ b/base-helm-configs/prometheus-mysql-exporter/values.yaml @@ -137,36 +137,36 @@ config: {} # logFormat: "logger:stderr" collectors: - # auto_increment.columns: false - # binlog_size: false + auto_increment.columns: true + binlog_size: true # engine_innodb_status: false # engine_tokudb_status: false # global_status: true # global_variables: true # info_schema.clientstats: false - # info_schema.innodb_metrics: false + info_schema.innodb_metrics: true # info_schema.innodb_tablespaces: false # info_schema.innodb_cmp: false # info_schema.innodb_cmpmem: false - # info_schema.processlist: false + info_schema.processlist: true # info_schema.processlist.min_time: 0 - # info_schema.query_response_time: false + info_schema.query_response_time: true # info_schema.tables: true # info_schema.tables.databases: '*' - # info_schema.tablestats: false + info_schema.tablestats: true # info_schema.schemastats: false - # info_schema.userstats: false + info_schema.userstats: true # perf_schema.eventsstatements: false # perf_schema.eventsstatements.digest_text_limit: 120 # perf_schema.eventsstatements.limit: false # perf_schema.eventsstatements.timelimit: 86400 - # perf_schema.eventswaits: false - # perf_schema.file_events: false + perf_schema.eventswaits: true + perf_schema.file_events: true # perf_schema.file_instances: false - # perf_schema.indexiowaits: false - # perf_schema.tableiowaits: false - # perf_schema.tablelocks: false - perf_schema.replication_group_member_stats: true + perf_schema.indexiowaits: true + perf_schema.tableiowaits: true + perf_schema.tablelocks: true + # perf_schema.replication_group_member_stats: true # slave_status: true # slave_hosts: false # heartbeat: false @@ -188,7 +188,7 @@ mysql: # secret with full config my.cnf existingConfigSecret: name: "mariadb-monitor" - key: "my.conf" + key: "my.cnf" # secret only containing the password existingPasswordSecret: name: "mariadb-monitoring" diff --git a/docs/prometheus-mysql-exporter.md b/docs/prometheus-mysql-exporter.md index 57ccb3e37..3281de942 100644 --- a/docs/prometheus-mysql-exporter.md +++ b/docs/prometheus-mysql-exporter.md @@ -21,7 +21,7 @@ kubectl --namespace openstack \ Then add the config to a secret that'll be used within the container for our shared services ``` shell -kubectl -n openstack create secret generic mariadb-monitor --type Opaque --from-literal=my.cnf="[client.monitoring] +kubectl -n openstack create secret generic mariadb-monitor --type Opaque --from-literal=my.cnf="[client.mariadb-monitor] user=monitoring password=$(kubectl --namespace openstack get secret mariadb-monitoring -o jsonpath='{.data.password}' | base64 -d)" ``` From e2c2eab1e71d36273717dcb49859976ee51cab4d Mon Sep 17 00:00:00 2001 From: James Denton Date: Mon, 21 Jul 2025 09:14:37 -0500 Subject: [PATCH 26/97] Exempted wlp wireless interfaces from rx/tx queue changes (#1080) * Exempted wlp wireless interfaces from rx/tx queue changes * Doc fix for envoyproxy --- ansible/roles/host_setup/files/queue_max.sh | 2 +- docs/infrastructure-envoy-gateway-api.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/host_setup/files/queue_max.sh b/ansible/roles/host_setup/files/queue_max.sh index 3f593c64b..f341e3f1e 100644 --- a/ansible/roles/host_setup/files/queue_max.sh +++ b/ansible/roles/host_setup/files/queue_max.sh @@ -16,7 +16,7 @@ set -e function ethernetDevs () { # Returns all physical devices ip -details -json link show | jq -r '.[] | - if .linkinfo.info_kind // .link_type == "loopback" or (.ifname | test("idrac+")) then + if .linkinfo.info_kind // .link_type == "loopback" or (.ifname | test("idrac+")) or (.ifname | test("wlp+")) then empty else .ifname diff --git a/docs/infrastructure-envoy-gateway-api.md b/docs/infrastructure-envoy-gateway-api.md index 60617dbcb..20ed138be 100644 --- a/docs/infrastructure-envoy-gateway-api.md +++ b/docs/infrastructure-envoy-gateway-api.md @@ -60,5 +60,5 @@ kubectl -n envoy-gateway get gateways.gateway.networking.k8s.io flex-gateway If you encounter any issues, check the logs of the `envoy-gateway` deployment. ``` shell -kubectl logs -n envoygateway-system deployment/envoy-gateway +kubectl logs -n envoyproxy-gateway-system deployment/envoy-gateway ``` From 193777a52bb564af498ded90f46dcb31177aaade Mon Sep 17 00:00:00 2001 From: Sowmya Nethi Date: Wed, 23 Jul 2025 09:18:21 +0530 Subject: [PATCH 27/97] feat(policy): allow readers to retrieve flavor profiles in Octavia (#1081) * fix: cinder service mapping in skyline configuration * feat(policy): allow readers to retrieve flavor profiles in Octavia --- base-kustomize/octavia/base/kustomization.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/base-kustomize/octavia/base/kustomization.yaml b/base-kustomize/octavia/base/kustomization.yaml index 0368e3d8c..3377cd48b 100644 --- a/base-kustomize/octavia/base/kustomization.yaml +++ b/base-kustomize/octavia/base/kustomization.yaml @@ -20,6 +20,24 @@ resources: # To run the OVN driver, the octavia-api container must have an agent container within the same pod. patches: + - target: + kind: Secret + name: octavia-etc + patch: |- + - op: add + path: /data/policy.yaml + value: b3NfbG9hZC1iYWxhbmNlcl9hcGk6Zmxhdm9yLXByb2ZpbGU6Z2V0X29uZTogcnVsZTpsb2FkLWJhbGFuY2VyOnJlYWQKb3NfbG9hZC1iYWxhbmNlcl9hcGk6Zmxhdm9yLXByb2ZpbGU6Z2V0X2FsbDogcnVsZTpsb2FkLWJhbGFuY2VyOnJlYWQ= + - target: + kind: Deployment + name: octavia-api + patch: |- + - op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + name: octavia-etc + mountPath: /etc/octavia/policy.yaml + subPath: policy.yaml + readOnly: true - target: kind: Deployment name: octavia-api From 57fd9b4d34552a05d5d09dbbc0776938bb954132 Mon Sep 17 00:00:00 2001 From: Sulochan Acharya Date: Wed, 23 Jul 2025 18:57:55 +0545 Subject: [PATCH 28/97] Update docs for fernet sync (#1082) We were using push secret which was not deleting old keys so swited back to fernet-sync. --- docs/sync-fernet-keys.md | 80 ++++++++++------------------------------ 1 file changed, 19 insertions(+), 61 deletions(-) diff --git a/docs/sync-fernet-keys.md b/docs/sync-fernet-keys.md index d1848ee34..f0d6bdb34 100644 --- a/docs/sync-fernet-keys.md +++ b/docs/sync-fernet-keys.md @@ -42,20 +42,14 @@ Main K8s Cluster | ──> API ──> | Remote K8s Cluster | ## How can we sync keys? - Ensure that each cluster has the correct permissions to read and write Kubernetes Secrets. -- Use tools such as [External Secret](https://external-secrets.io/latest/api/pushsecret/) to sync the keystone-ferent-keys. +- We are using [fernet-sync](https://github.com/rackerlabs/fernet-sync) to sync the keystone-ferent-keys. - Make sure to have service account token by reading the above secret. -## Using PushSecret (external-secrets) to sync secrets +## Setup fernet-sync deploymet -Lets look at how we can setup pushsecrets to sync fernet keys between two or more clusters. -First, install external-secrets operator +Lets look at how we can setup to sync fernet keys between two or more clusters. -```shell -helm repo add external-secrets https://charts.external-secrets.io -helm install external-secrets external-secrets/external-secrets -n external-secrets --create-namespace -``` - -Now, in order for secrets to sync, we will use a serviceaccount token with access only to a particular secret. +First, in order for secrets to sync, we will use a serviceaccount token with access only to a particular secret. So, in the target cluster create a new service account and give it appropriate permissions ``` apiVersion: v1 @@ -113,64 +107,28 @@ kubectl get secret keystone-sync-external-secret -o yaml -n openstack next, on the source cluster create credentials for the target ``` -apiVersion: v1 -kind: Secret -metadata: - name: target-credentials - namespace: openstack -data: - token: +git clone https://github.com/rackerlabs/fernet-sync.git +cd fernet-sync +vim create-secret.sh + +then make sure to have your cluser and token defined in the format +TOKENS = {"https://cluster1.example.com": "token for cluster1", "https://cluster2.example.com": "token for cluster2"} ``` -next, create a secret store for pushsecret to use +next, create the secret ``` -apiVersion: external-secrets.io/v1beta1 -kind: SecretStore -metadata: - name: target-store - namespace: openstack -spec: - provider: - kubernetes: - remoteNamespace: openstack - server: - url: - caBundle: - auth: - token: - bearerToken: - name: target-credentials - key: token +cd fernet-sync +./create-secret.sh + ``` -Now, you can create a pushsecret to sync (any secret but in our case we have restricted to) keystone-fernet-keys. +Now, you can create a deployment to sync secret -Lets create that pushsecret definition -``` -apiVersion: external-secrets.io/v1alpha1 -kind: PushSecret -metadata: - name: pushsecret-target-store - namespace: openstack -spec: - # Replace existing secrets in provider - updatePolicy: Replace - # Resync interval - refreshInterval: 300s - # SecretStore to push secrets to - secretStoreRefs: - - name: target-store - kind: SecretStore - # Target Secret - selector: - secret: - name: keystone-fernet-keys # Source cluster Secret name - data: - - match: - remoteRef: - remoteKey: keystone-fernet-keys # Target cluster Secret name +```shell +kubectl apply -f deployment.yaml ``` -This will sync keystone-fernet-keys from source to destination and refresh it every 300sec. +This will create a deployment that will listen to any change in `keystone-fernet-keys` secret and sync it to the +clusters defined the create-secret.sh script. From 94fa198bb84afa6c92bece1c304112b25a6876ef Mon Sep 17 00:00:00 2001 From: "phillip.toohill" Date: Wed, 23 Jul 2025 16:44:17 -0500 Subject: [PATCH 29/97] chore: Adding envoy servicemonitor and documentation (#1084) --- .../base/envoy-service-monitor.yaml | 15 + .../base/kustomization.yaml | 1 + docs/monitoring-info.md | 7 +- docs/prometheus-envoy-gateway.md | 14 + etc/grafana-dashboards/envoy-gateway.json | 3108 +++++++++++++++++ .../envoy-resource-monitor.json | 249 ++ 6 files changed, 3393 insertions(+), 1 deletion(-) create mode 100644 base-kustomize/envoyproxy-gateway/base/envoy-service-monitor.yaml create mode 100644 docs/prometheus-envoy-gateway.md create mode 100644 etc/grafana-dashboards/envoy-gateway.json create mode 100644 etc/grafana-dashboards/envoy-resource-monitor.json diff --git a/base-kustomize/envoyproxy-gateway/base/envoy-service-monitor.yaml b/base-kustomize/envoyproxy-gateway/base/envoy-service-monitor.yaml new file mode 100644 index 000000000..a2e67f338 --- /dev/null +++ b/base-kustomize/envoyproxy-gateway/base/envoy-service-monitor.yaml @@ -0,0 +1,15 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: envoy-gateway-monitor + namespace: prometheus +spec: + endpoints: + - interval: 15s + port: metrics + namespaceSelector: + matchNames: + - envoyproxy-gateway-system + selector: + matchLabels: + app.kubernetes.io/instance: envoyproxy-gateway diff --git a/base-kustomize/envoyproxy-gateway/base/kustomization.yaml b/base-kustomize/envoyproxy-gateway/base/kustomization.yaml index 9befd997c..aa708aa44 100644 --- a/base-kustomize/envoyproxy-gateway/base/kustomization.yaml +++ b/base-kustomize/envoyproxy-gateway/base/kustomization.yaml @@ -7,3 +7,4 @@ resources: - envoy-gatewayclass.yaml - envoy-gateway.yaml - envoy-endpoint-policies.yaml + - envoy-service-monitor.yaml diff --git a/docs/monitoring-info.md b/docs/monitoring-info.md index bee1a6035..3a49ac836 100644 --- a/docs/monitoring-info.md +++ b/docs/monitoring-info.md @@ -110,11 +110,16 @@ Once we've ran the apply command we will have installed ServiceMonitors for Kube You can view more information about OVN monitoring in the [OVN Monitoring Introduction Docs](ovn-monitoring-introduction.md). * ### Nginx Gateway Monitoring: -Genestack makes use of the Gateway API for its implementation of [Kubernetes Gateway API](https://gateway-api.sigs.k8s.io/). Genestack deploys this as part of its infrastructure, view the [Gateway Deployment Doc](infrastructure-gateway-api.md) for more information. +Genestack can make use of the NGINX Gateway API for its implementation of [Kubernetes Gateway API](https://gateway-api.sigs.k8s.io/). Genestack deploys this as part of its infrastructure, view the [Gateway Deployment Doc](infrastructure-gateway-api.md) for more information. Nginx Gateway does expose important metrics for us to gather but it does not do so via a service. Instead we must make use another Prometheus CRD the [PodMonitor](https://prometheus-operator.dev/docs/getting-started/design/#podmonitor). The install is similar to the above OVN monitoring as you can see in the [Nginx Gateway Exporter Deployment Doc](prometheus-nginx-gateway.md). The primary difference is the need to target and match on a pod that's exposing the metrics rather than a service. You can view more information about the metrics exposed by the Nginx Gateway by viewing the [Nginx Gateway Fabric Docs](https://docs.nginx.com/nginx-gateway-fabric/how-to/monitoring/prometheus/). +* ### Envoy Gateway Monitoring: +Genestack makes use of the Envoy Gateway API for its implementation of [Kubernetes Gateway API](https://gateway-api.sigs.k8s.io/). Genestack deploys the Envoy Gateway as part of its infrastructure, view the [Envoy Gateway Deployment Doc](infrastructure-envoy-gateway-api.md) for more information. +Envoy Gateway is a Kubernetes-native API Gateway and reverse proxy control plane. It simplifies deploying and operating Envoy Proxy as a data plane by using the standard Gateway API and its own extensible APIs. For more information about Envoy Gateway in general view the [Envoy Gateway Documentation](https://gateway.envoyproxy.io/docs/concepts/introduction/). +The Envoy Gateway serves Prometheus metrics by default and list of the metrics collected can be found at [Envoy Gateway Exported Metrics](https://gateway.envoyproxy.io/docs/tasks/observability/gateway-exported-metrics/). + * ### OpenStack Metrics: OpenStack Metrics are a bit different compared to the rest of the exporters as there's no single service, pod or deployment that exposes Prometheus metrics for collection. Instead, Genestack uses the [OpenStack Exporter](https://github.com/openstack-exporter/openstack-exporter) to gather the metrics for us. The OpenStack exporter reaches out to all the configured OpenStack services, queries their API for stats and packages them as metrics Prometheus can then process. The OpenStack exporter is configurable and can collect metrics from just about every OpenStack service such as Keystone, Nova, Octavia etc.. diff --git a/docs/prometheus-envoy-gateway.md b/docs/prometheus-envoy-gateway.md new file mode 100644 index 000000000..cb84fb988 --- /dev/null +++ b/docs/prometheus-envoy-gateway.md @@ -0,0 +1,14 @@ +# Envoy Gateway Monitoring + +Envoy Gateway exposes metrics that can be used to monitor the behavior and health of the Envoy Gateway. + +Following the deployment of the [Envoy Gateway](infrastructure-envoy-gateway-api.md) the metrics will be served and the service monitor will be created. + +If you need to deploy the service monitor independently you may apply the file directly with the following directions. + +## Installation + +``` shell +kubectl apply -f /etc/genestack/kustomize/envoyproxy-gateway/base/envoy-service-monitor.yaml +``` + diff --git a/etc/grafana-dashboards/envoy-gateway.json b/etc/grafana-dashboards/envoy-gateway.json new file mode 100644 index 000000000..355c39f56 --- /dev/null +++ b/etc/grafana-dashboards/envoy-gateway.json @@ -0,0 +1,3108 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Envoy Gateway monitoring Dashboard with exported metrics.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "title": "Watching Components", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "How long in seconds a subscribed watchable is handled.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 0, + "y": 1 + }, + "id": 1, + "maxPerRow": 3, + "options": { + "displayMode": "basic", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.0.0", + "repeat": "Runner", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(le) (watchable_subscribe_duration_seconds_bucket{runner=~\"$Runner\", namespace=\"$Namespace\"})", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Duration Bucket: $Runner", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 7, + "y": 1 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": false + }, + "pluginVersion": "11.0.0", + "repeat": "Runner", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "avg by(runner) (watchable_subscribe_duration_seconds_sum{runner=~\"$Runner\", namespace=\"$Namespace\"})", + "format": "time_series", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Avg", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "max by(runner) (watchable_subscribe_duration_seconds_sum{runner=~\"$Runner\", namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Max", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Duration Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Current depth of watchable map.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-blue", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 10, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": false + }, + "pluginVersion": "11.0.0", + "repeat": "Runner", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(runner) (watchable_depth{runner=~\"$Runner\", namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Depth", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-blue", + "mode": "shades" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Success" + }, + "properties": [ + { + "id": "displayName", + "value": "Success" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 12, + "y": 1 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "repeat": "Runner", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(status) (watchable_subscribe_total{runner=\"$Runner\", namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(watchable_subscribe_total{runner=\"$Runner\", namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Total", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Statistics", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 35, + "panels": [], + "title": "Status Updater", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of panics recovered in the system.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 0, + "y": 8 + }, + "id": 25, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": false + }, + "pluginVersion": "11.0.0", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(watchable_panics_recovered_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Recovered Panics", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Recovered Panics", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "How long a status update takes to finish for all Kind.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-blue", + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 37 + }, + "id": 61, + "options": { + "displayMode": "basic", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(le) (status_update_duration_seconds_bucket{namespace=\"$Namespace\"})", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Duration Bucket", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 0.2 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 37 + }, + "id": 82, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "status_update_duration_seconds_sum{namespace=\"$Namespace\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Avg Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 0.1 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 12, + "y": 37 + }, + "id": 83, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "status_update_duration_seconds_sum{namespace=\"$Namespace\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Max Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlPu" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.01 + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 37 + }, + "id": 84, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "logmin" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "status_update_duration_seconds_sum{namespace=\"$Namespace\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Min Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of status updates by object kind.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 0, + "y": 46 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(kind) (status_update_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{kind}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 10, + "y": 46 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(status) (status_update_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Status", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 54 + }, + "id": 126, + "panels": [], + "title": "xDS Server", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "super-light-green", + "mode": "shades" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 0, + "y": 55 + }, + "id": 127, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": false + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(xds_snapshot_create_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Total", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(status) (xds_snapshot_create_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{status}}", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Snapshot Creation Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 10, + "y": 55 + }, + "id": 149, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "max(xds_stream_duration_seconds_bucket{namespace=\"$Namespace\", isDeltaStream=\"true\"})", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Finished Stream", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Maximum duration seconds for finished xDS delta stream connection.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 15, + "y": 55 + }, + "id": 150, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "xds_stream_duration_seconds_sum{namespace=\"$Namespace\", isDeltaStream=\"true\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Max Duration", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Minimum duration seconds for finished xDS delta stream connection.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-green", + "mode": "shades" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 15, + "y": 59 + }, + "id": 151, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "min" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "xds_stream_duration_seconds_sum{namespace=\"$Namespace\", isDeltaStream=\"true\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Min Duration", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of xds snapshot cache updates by node id.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 0, + "y": 63 + }, + "id": 152, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(nodeID) (xds_snapshot_update_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Update Total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 10, + "y": 63 + }, + "id": 153, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(status) (xds_snapshot_update_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{nodeID}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Update Status", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 71 + }, + "id": 156, + "panels": [], + "title": "Infrastructure Manager", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 72 + }, + "id": 199, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(le) (resource_apply_duration_seconds_bucket{namespace=\"$Namespace\"})", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Apply Duration Bucket", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.3 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 72 + }, + "id": 220, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(kind) (resource_apply_duration_seconds_sum{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Avg Apply Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.3 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 72 + }, + "id": 221, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(kind) (resource_apply_duration_seconds_sum{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Max Apply Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlPu" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.3 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 72 + }, + "id": 222, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "logmin" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(kind) (resource_apply_duration_seconds_sum{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Min Apply Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of applied resources sumed by kind.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 80 + }, + "id": 157, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(kind) (resource_apply_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Applied Resources", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of applied resources that succeed sumed by kind.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 80 + }, + "id": 229, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(status) (resource_apply_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Applied Resources Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of applied resources sumed by infra name.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 80 + }, + "id": 178, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(name) (resource_apply_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Applied Infrastructures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 87 + }, + "id": 223, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(le) (resource_delete_duration_seconds_bucket{namespace=\"$Namespace\"})", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Delete Duration Bucket", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 0.3 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 87 + }, + "id": 224, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(kind) (resource_delete_duration_seconds_sum{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Avg Delete Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 0.3 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 87 + }, + "id": 225, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(kind) (resource_delete_duration_seconds_sum{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Max Delete Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlPu" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 0.3 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 87 + }, + "id": 226, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "logmin" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(kind) (resource_delete_duration_seconds_sum{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Min Delete Duration", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of deleted resources sumed by kind.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 95 + }, + "id": 227, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(kind) (resource_delete_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Deleted Resources", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of deleted resources that succeed sumed by kind.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 95 + }, + "id": 232, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(status) (resource_delete_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Deleted Resources Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of deleted resources sumed by infra name.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 95 + }, + "id": 228, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(name) (resource_delete_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Total Deleted Infrastructures", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 102 + }, + "id": 249, + "panels": [], + "title": "Wasm", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 103 + }, + "id": 250, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "wasm_cache_entries{namespace=\"$Namespace\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Cache Entries", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of Wasm remote fetch cache lookups.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 4, + "y": 103 + }, + "id": 251, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(hit) (wasm_cache_lookup_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "hit={{hit}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Cache Lookups", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total number of Wasm remote fetches and results.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 11, + "y": 103 + }, + "id": 252, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(status) (wasm_remote_fetch_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{status}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(wasm_remote_fetch_total{namespace=\"$Namespace\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Total", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Cache Remote Fetches", + "type": "stat" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "Control Plane" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "envoy-gateway-system", + "value": "envoy-gateway-system" + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "label_values(watchable_depth,namespace)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "Namespace", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(watchable_depth,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "label_values(watchable_depth,runner)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "Runner", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(watchable_depth,runner)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "browser", + "title": "Envoy Gateway Global", + "uid": "bdn8lriao7myoa", + "version": 1, + "weekStart": "" +} diff --git a/etc/grafana-dashboards/envoy-resource-monitor.json b/etc/grafana-dashboards/envoy-resource-monitor.json new file mode 100644 index 000000000..1a4353aef --- /dev/null +++ b/etc/grafana-dashboards/envoy-resource-monitor.json @@ -0,0 +1,249 @@ +{ + "description": "Memory and CPU Usage Monitor for Envoy Gateway and Envoy Proxy.\n", + "graphTooltip": 1, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [ ], + "title": "Envoy Gateway", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "never" + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 2, + "interval": "1m", + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table" + } + }, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum by (namespace) (\n rate(\n container_cpu_usage_seconds_total{\n container=\"envoy-gateway\"\n }\n [$__rate_interval])\n)\n", + "intervalFactor": 2, + "legendFormat": "{{namespace}}\n" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never" + }, + "unit": "bytes" + } + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 3, + "interval": "1m", + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table" + } + }, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum by (namespace) (\n container_memory_working_set_bytes{container=\"envoy-gateway\"}\n)\n", + "intervalFactor": 2, + "legendFormat": "{{namespace}}\n" + } + ], + "title": "Memory Usage", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 4, + "panels": [ ], + "title": "Envoy Proxy", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "never" + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 5, + "interval": "1m", + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table" + } + }, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum by (pod) (\n rate(\n container_cpu_usage_seconds_total{\n container=\"envoy\"\n }\n [$__rate_interval])\n)\n", + "intervalFactor": 2, + "legendFormat": "{{pod}}\n" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "custom": { + "fillOpacity": 10, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never" + }, + "unit": "bytes" + } + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 6, + "interval": "1m", + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table" + } + }, + "pluginVersion": "v11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum by (pod) (\n container_memory_working_set_bytes{container=\"envoy\"}\n)\n", + "intervalFactor": 2, + "legendFormat": "{{pod}}\n" + } + ], + "title": "Memory Usage", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "templating": { + "list": [ + { + "name": "datasource", + "query": "prometheus", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timezone": "utc", + "title": "Envoy Resources Monitor", + "uid": "f7aeb41676b7865cf31ae49691325f91" +} From 4ad8f70eccc3d6e0a5064569490e0d5acb8ff20a Mon Sep 17 00:00:00 2001 From: Jorge Perez Date: Wed, 23 Jul 2025 16:45:19 -0500 Subject: [PATCH 30/97] docs: add how-to for enovy security policies (#1083) Signed-off-by: Jorge Perez --- ...frastructure-envoy-gateway-api-security.md | 127 ++++++++++++++++++ mkdocs.yml | 1 + 2 files changed, 128 insertions(+) create mode 100644 docs/infrastructure-envoy-gateway-api-security.md diff --git a/docs/infrastructure-envoy-gateway-api-security.md b/docs/infrastructure-envoy-gateway-api-security.md new file mode 100644 index 000000000..d2d85424c --- /dev/null +++ b/docs/infrastructure-envoy-gateway-api-security.md @@ -0,0 +1,127 @@ +# Security Policies + +From [Envoy documentation](https://gateway.envoyproxy.io/docs/concepts/introduction/gateway_api_extensions/security-policy/): + +SecurityPolicy is an Envoy Gateway extension to the Kubernetes Gateway API that allows you to define authentication and authorization requirements for traffic entering your gateway. It acts as a security layer that only properly authenticated and authorized requests are allowed through your backend services. + +In this section we will be implementing [oidc](https://gateway.envoyproxy.io/docs/tasks/security/oidc/) authentication to auth using Azure AD. + +!!! note "You must have deployed Envoy Gateway already and installed the CRDs before this will work" + +## Create the HTTPRoute + +!!! note "The examples used here reference alertmanager. You will change the settings as necessary for your application/s" + +``` yaml title="alertmanager-gw-route.yaml" +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + annotations: + name: alertmanager-gateway-route + namespace: prometheus +spec: + hostnames: + - alertmanager.example.com + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: flex-internal-gateway + namespace: internal + sectionName: am-https + rules: + - backendRefs: + - group: "" + kind: Service + name: kube-prometheus-stack-alertmanager + port: 9093 + weight: 1 + matches: + - path: + type: PathPrefix + value: / +``` + +`kubectl apply -f alertmanager-gw-route.yaml` + +### Check/update your listener + +Make sure you have a listener configured on your gateway for the HTTPRoute you created. As an example, you should have something like the following in your gateway configuration: + +``` yaml + - allowedRoutes: + namespaces: + from: All + hostname: alertmanager.example.com + name: am-https + port: 443 + protocol: HTTPS + tls: + certificateRefs: + - group: "" + kind: Secret + name: alertmanager-envoy-secret + mode: Terminate +``` + + +## Register an OIDC application + +Registering the Azure OIDC application is beyond the scope of this article. You will need to add a redirect url and you will need to know your client and tenant ids as well as your client secret. Once you have all that information, you may proceed to configuring the Kubernetes secret and security policy. + +## Kubernetes secret + +You will need to create a kubernetes secret that contains the client secret for your Azure application. You can either use a yaml file or paste the secret on the command line. + +=== "CLI" + ``` shell + read -s CLIENT_SECRET + read -p "Please enter the application namespace: " APP_NAMESPACE + read -p "Please enter the application name: " APP_NAME + kubectl -n ${APP_NAMESPACE} create secret generic azuread-client-secret-${APP_NAME} --from-literal=client-secret=${CLIENT_SECRET} + ``` + +=== "YAML" + ``` yaml title="azuread-client-secret-APP_NAME.yaml" + apiVersion: v1 + data: + client-secret: + kind: Secret + metadata: + name: azuread-client-secret + namespace: + type: Opaque + ``` + + `kubectl apply -f azuread-client-secret-` + + +## Create the Security Policy + +``` yaml title="alertmanager-sp.yaml" +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: SecurityPolicy +metadata: + annotations: + generation: 1 + name: azuread-oidc-policy + namespace: +spec: + oidc: + clientID: + clientSecret: + group: "" + kind: Secret + name: azuread-client-secret- + logoutPath: //logout + provider: + issuer: https://login.microsoftonline.com//v2.0 + redirectURL: https://alertmanager.example.com//oauth2/callback + targetRefs: + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: +``` + +`kubectl -f apply alertmanager-sp.yaml` + +!!! note "Your redirect URL in the SecurityPolicy must match what you configured in your OIDC application" diff --git a/mkdocs.yml b/mkdocs.yml index dfeec73cd..ad444975c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -276,6 +276,7 @@ nav: - Custom Routes: infrastructure-nginx-gateway-api-custom.md - Rackspace Example Gateway Overview: rackspace-infrastructure-nginx-gateway-api.md - Creating self-signed CA issuer for Gateway API: infrastructure-nginx-gateway-api-ca-issuer.md + - Creating Security Policies: infrastructure-envoy-gateway-api-security.md - Observability: - Observability Overview: observability-info.md - Monitoring Overview: monitoring-info.md From a6dc314d99f62e87d25a0cb5be567a89762efd66 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Wed, 23 Jul 2025 17:25:22 -0500 Subject: [PATCH 31/97] chore: cleans up the routing Signed-off-by: Kevin Carter --- mkdocs.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index ad444975c..d3906f224 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -272,11 +272,10 @@ nav: - MariaDB: - Operations: infrastructure-mariadb-ops.md - Gateway API: - - NGINX Gateway: - - Custom Routes: infrastructure-nginx-gateway-api-custom.md - - Rackspace Example Gateway Overview: rackspace-infrastructure-nginx-gateway-api.md - - Creating self-signed CA issuer for Gateway API: infrastructure-nginx-gateway-api-ca-issuer.md - - Creating Security Policies: infrastructure-envoy-gateway-api-security.md + - Custom Routes: infrastructure-nginx-gateway-api-custom.md + - Rackspace Example Gateway Overview: rackspace-infrastructure-nginx-gateway-api.md + - Creating self-signed CA issuer for Gateway API: infrastructure-nginx-gateway-api-ca-issuer.md + - Creating Security Policies: infrastructure-envoy-gateway-api-security.md - Observability: - Observability Overview: observability-info.md - Monitoring Overview: monitoring-info.md From 6eb81c67c7d4d54f0f2f9c003c58e04e5c09ab63 Mon Sep 17 00:00:00 2001 From: Jorge Perez Date: Thu, 24 Jul 2025 14:36:20 -0500 Subject: [PATCH 32/97] Jorge doc envoy security policy (#1089) * docs: add how-to for enovy security policies Signed-off-by: Jorge Perez * fix: updated docs to correct bad yaml formatting in yaml example Signed-off-by: Jorge Perez --------- Signed-off-by: Jorge Perez --- docs/infrastructure-envoy-gateway-api-security.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/infrastructure-envoy-gateway-api-security.md b/docs/infrastructure-envoy-gateway-api-security.md index d2d85424c..829d792dd 100644 --- a/docs/infrastructure-envoy-gateway-api-security.md +++ b/docs/infrastructure-envoy-gateway-api-security.md @@ -84,11 +84,11 @@ You will need to create a kubernetes secret that contains the client secret for ``` yaml title="azuread-client-secret-APP_NAME.yaml" apiVersion: v1 data: - client-secret: + client-secret: kind: Secret metadata: - name: azuread-client-secret - namespace: + name: azuread-client-secret + namespace: type: Opaque ``` From 1d9264b50190359c59e9edd13e5a3f418f9c0132 Mon Sep 17 00:00:00 2001 From: Gerald Williams Date: Thu, 24 Jul 2025 14:37:52 -0500 Subject: [PATCH 33/97] feat: add router to flex-billing metrics. (#1087) Modified Ceilometer overrides to send router events to Gnocchi for processing by Flex-Billing. --- .../ceilometer/ceilometer-helm-overrides.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml b/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml index 4965c4a2c..cd9cda971 100644 --- a/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml +++ b/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml @@ -504,7 +504,9 @@ conf: name: fields: payload.router.name resource_id: - fields: ["payload.router.id", "payload.id"] + fields: payload.router.id + event_type: + fields: event_type - event_type: floatingip.* traits: <<: *network_traits @@ -1051,15 +1053,20 @@ conf: ip.floating: event_create: - floatingip.create.end - event_delete: floatingip.delete.end + - router.create.end + event_delete: + - floatingip.delete.end + - router.delete.end event_update: - floatingip.update.end + - router.update.end event_attributes: id: resource_id user_id: user_id project_id: project_id floating_ip_address: ip_address event_type: event_type + display_name: name - resource_type: stack metrics: From 92fc9928ebcd9c6d02c3dc55b59f9af28ca699c6 Mon Sep 17 00:00:00 2001 From: "phillip.toohill" Date: Thu, 24 Jul 2025 14:38:15 -0500 Subject: [PATCH 34/97] chore: Updating alert/monitor docs with default alerts (#1088) --- docs/alerting-info.md | 10 +- docs/genestack-alerts.md | 428 +++++++++++++++++++++++++++++++++++++++ docs/monitoring-info.md | 6 + 3 files changed, 436 insertions(+), 8 deletions(-) create mode 100644 docs/genestack-alerts.md diff --git a/docs/alerting-info.md b/docs/alerting-info.md index c79944dc7..c768a697d 100644 --- a/docs/alerting-info.md +++ b/docs/alerting-info.md @@ -81,11 +81,5 @@ We can now take all this information and build out an alerting workflow that sui ## Genestack alerts -This section contains some information on individual Genestack alert. - -### MariaDB backup alert - -Based on a schedule of 6 hours by default, it allows 1 hour to upload and -alerts when MySQL doesn't successfully complete a backup. - -It alerts at warning level the first time this happens, and at critical level the second time this happens. +Genestack supplies default alerts, some of which are configured as part of the prometheus install and some of them come from the exporters deployments directly and are not controlled by Genestack. +View the list of currently defined alerts supplied by genestack at [Genestack Alerts](genestack-alerts.md). diff --git a/docs/genestack-alerts.md b/docs/genestack-alerts.md new file mode 100644 index 000000000..03ce9d130 --- /dev/null +++ b/docs/genestack-alerts.md @@ -0,0 +1,428 @@ + +

Genestack Prometheus Alerts

+ +## Blackbox Alerts +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **Service Down** | Service probe has failed for more than two minutes on (instance {{ $labels.instance }}) | Service probe has failed for more than two minutes.
LABELS = {{ $labels }}
| critical | +| **TLS certificate expiring** | SSL certificate will expire soon on (instance {{ $labels.instance }}) | SSL certificate expires within 30 days.
VALUE = {{ $value }}
LABELS = {{ $labels }}
| warning | +| **TLS certificate expiring** | SSL certificate will expire soon on (instance {{ $labels.instance }}) | SSL certificate expires within 15 days.
VALUE = {{ $value }}
LABELS = {{ $labels }}
| critical | +

🔝 Back to Top

+ +--- + +## Compute Resource Alerts +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **AbnormalInstanceFailures** | Instance build failure rate is abnormally high | This indicates a major problem building compute instances.
View logs and take action to resolve the build failures.
| critical | +| **InstancesStuckInFailureState** | Instances stuck in failure state for a prolonged period | There are instances stuck in a building or error state for a prolonged period
that need to be cleaned up.
| warning | +

🔝 Back to Top

+ +--- + +## Image Resource Alerts +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **AbnormalImageFailures** | Image create failure rate is abnormally high | This indicates a major problem creating images.
View logs and take action to resolve the build failures.
| critical | +| **ImagesStuckInFailureState** | Images stuck in failure state for a prolonged period | There are images stuck in a failure state for a prolonged period
that need to be cleaned up.
| warning | +

🔝 Back to Top

+ +--- + +## Linux MDM device and RAID alerts +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **NodeMdInfoFailedDeviceCritical** | NVME device on Linux software RAID failure info | {{ $labels.name }}
Number MD Failed:{{ $labels.FailedDevices }}
LABELS: {{ $labels }} | critical | +| **NodeMdInfoStateCritical** | Linux software MD RAID State is NOT active\|clean | {{ $labels.name }}
State:{{ $labels.State }}
LABELS: {{ $labels }} | critical | +| **NodeMdInfoSuperblockPersistenceCritical** | Linux software MD Superblock is NOT persistent | {{ $labels.name }}
Persistence:{{ $labels.Persistence }}
LABELS: {{ $labels }} | critical | +| **NodeMdStateCritical** | Linux MDM RAID State is {{ $labels.state }} | {{ $labels.name }}
MD RAID status:{{ $value }}
MD RAID device:{{ $labels.device }}
LABELS: {{ $labels }} | critical | +

🔝 Back to Top

+ +--- + +## MariaDB backup alerts +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **mariadbBackupCritical** | Second successive MariaDB backup not successful within 1 hour of scheduled run | Second successive MariaDB backup not successful within 1 hour of scheduled run.
| critical | +| **mariadbBackupWarning** | Last MariaDB backup not successful within 1 hour of scheduled run | Last MariaDB backup not successful within 1 hour of scheduled run.
| warning | +

🔝 Back to Top

+ +--- + +## Multipath path checker alerts +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **NodeDmpathInfoMultipathCritical** | Multipathd paths are NOT active\|ready and paths are likely orphaned | {{ $labels.name }}
labels: {{ $labels }} | critical | +

🔝 Back to Top

+ +--- + +## Mysql Alerts +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **MysqlDown** | MariaDB down (instance {{ $labels.instance }}) | MariaDB instance is down on {{ $labels.instance }}
VALUE = {{ $value }}
LABELS = {{ $labels }}
| critical | +| **MysqlRestarted** | MySQL restarted (instance {{ $labels.instance }}) | MySQL has just been restarted, less than one minute ago on {{ $labels.instance }}.
VALUE = {{ $value }}
LABELS = {{ $labels }}
| info | +| **MysqlSlowQueries** | MySQL slow queries (instance {{ $labels.instance }}) | MySQL server has some new slow queries.
VALUE = {{ $value }}
LABELS = {{ $labels }}
| warning | +| **MysqlTooManyConnections(>80%)** | Database too many connections (> 90%) (instance {{ $labels.instance }}) | More than 90% of MySQL connections are in use on {{ $labels.instance }}
VALUE = {{ $value }}
LABELS = {{ $labels }}
| warning | +

🔝 Back to Top

+ +--- + +## OVN backup alerts +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **ovnBackupDiskUsageCritical** | OVN backup volume >= 90% disk usage | OVN backup volume >= 90% disk usage.
| critical | +| **ovnBackupDiskUsageWarning** | OVN backup volume >= 80% disk usage | OVN backup volume >= 80% disk usage.
| warning | +| **ovnBackupUploadCritical** | Second successive OVN backup not uploaded within 1 hour of scheduled run | Second successive OVN backup not uploaded within 1 hour of scheduled run.
| critical | +| **ovnBackupUploadWarning** | Last OVN backup not uploaded within 1 hour of scheduled run | Last OVN backup not uploaded within 1 hour of scheduled run.
| warning | +

🔝 Back to Top

+ +--- + +## Octavia Resource Alerts +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **LoadbalancersInError** | Loadbalancer stuck in error state for a prolonged period | This may indicate a potential problem with failover and/or health manager services.
This could also indicate other problems building load balancers in general.
| critical | +

🔝 Back to Top

+ +--- + +## Volume Alerts +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **KubernetesVolumeOutOfDiskSpace** | Kubernetes Volume out of disk space (instance {{ $labels.instance }}) | Volume is almost full (< 20% left).
VALUE = {{ $value }}
LABELS = {{ $labels }}
| warning | +

🔝 Back to Top

+ +--- + +## alertmanager.rules +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **AlertmanagerClusterCrashlooping** | Half or more of the Alertmanager instances within the same cluster are crashlooping. | {{ $value \| humanizePercentage }} of Alertmanager instances within the {{$labels.job}} cluster have restarted at least 5 times in the last 10m. | critical | +| **AlertmanagerClusterDown** | Half or more of the Alertmanager instances within the same cluster are down. | {{ $value \| humanizePercentage }} of Alertmanager instances within the {{$labels.job}} cluster have been up for less than half of the last 5m. | critical | +| **AlertmanagerClusterFailedToSendAlerts** | All Alertmanager instances in a cluster failed to send notifications to a critical integration. | The minimum notification failure rate to {{ $labels.integration }} sent from any instance in the {{$labels.job}} cluster is {{ $value \| humanizePercentage }}. | critical | +| **AlertmanagerClusterFailedToSendAlerts** | All Alertmanager instances in a cluster failed to send notifications to a non-critical integration. | The minimum notification failure rate to {{ $labels.integration }} sent from any instance in the {{$labels.job}} cluster is {{ $value \| humanizePercentage }}. | warning | +| **AlertmanagerConfigInconsistent** | Alertmanager instances within the same cluster have different configurations. | Alertmanager instances within the {{$labels.job}} cluster have different configurations. | critical | +| **AlertmanagerFailedReload** | Reloading an Alertmanager configuration has failed. | Configuration has failed to load for {{ $labels.namespace }}/{{ $labels.pod}}. | critical | +| **AlertmanagerFailedToSendAlerts** | An Alertmanager instance failed to send notifications. | Alertmanager {{ $labels.namespace }}/{{ $labels.pod}} failed to send {{ $value \| humanizePercentage }} of notifications to {{ $labels.integration }}. | warning | +| **AlertmanagerMembersInconsistent** | A member of an Alertmanager cluster has not found all other cluster members. | Alertmanager {{ $labels.namespace }}/{{ $labels.pod}} has only found {{ $value }} members of the {{$labels.job}} cluster. | critical | +

🔝 Back to Top

+ +--- + +## config-reloaders +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **ConfigReloaderSidecarErrors** | config-reloader sidecar has not had a successful reload for 10m | Errors encountered while the {{$labels.pod}} config-reloader sidecar attempts to sync config in {{$labels.namespace}} namespace.
As a result, configuration for service running in {{$labels.pod}} may be stale and cannot be updated anymore. | warning | +

🔝 Back to Top

+ +--- + +## etcd +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **etcdDatabaseHighFragmentationRatio** | etcd database size in use is less than 50% of the actual allocated storage. | etcd cluster "{{ $labels.job }}": database size in use on instance {{ $labels.instance }} is {{ $value \| humanizePercentage }} of the actual allocated disk space, please run defragmentation (e.g. etcdctl defrag) to retrieve the unused fragmented disk space. | warning | +| **etcdDatabaseQuotaLowSpace** | etcd cluster database is running full. | etcd cluster "{{ $labels.job }}": database size exceeds the defined quota on etcd instance {{ $labels.instance }}, please defrag or increase the quota as the writes to etcd will be disabled when it is full. | critical | +| **etcdExcessiveDatabaseGrowth** | etcd cluster database growing very fast. | etcd cluster "{{ $labels.job }}": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance {{ $labels.instance }}, please check as it might be disruptive. | warning | +| **etcdGRPCRequestsSlow** | etcd grpc requests are slow | etcd cluster "{{ $labels.job }}": 99th percentile of gRPC requests is {{ $value }}s on etcd instance {{ $labels.instance }} for {{ $labels.grpc_method }} method. | critical | +| **etcdHighCommitDurations** | etcd cluster 99th percentile commit durations are too high. | etcd cluster "{{ $labels.job }}": 99th percentile commit durations {{ $value }}s on etcd instance {{ $labels.instance }}. | warning | +| **etcdHighFsyncDurations** | etcd cluster 99th percentile fsync durations are too high. | etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}. | warning | +| **etcdHighFsyncDurations** | etcd cluster 99th percentile fsync durations are too high. | etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}. | critical | +| **etcdHighNumberOfFailedGRPCRequests** | etcd cluster has high number of failed grpc requests. | etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}. | warning | +| **etcdHighNumberOfFailedGRPCRequests** | etcd cluster has high number of failed grpc requests. | etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}. | critical | +| **etcdHighNumberOfFailedProposals** | etcd cluster has high number of proposal failures. | etcd cluster "{{ $labels.job }}": {{ $value }} proposal failures within the last 30 minutes on etcd instance {{ $labels.instance }}. | warning | +| **etcdHighNumberOfLeaderChanges** | etcd cluster has high number of leader changes. | etcd cluster "{{ $labels.job }}": {{ $value }} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated. | warning | +| **etcdInsufficientMembers** | etcd cluster has insufficient number of members. | etcd cluster "{{ $labels.job }}": insufficient members ({{ $value }}). | critical | +| **etcdMemberCommunicationSlow** | etcd cluster member communication is slow. | etcd cluster "{{ $labels.job }}": member communication with {{ $labels.To }} is taking {{ $value }}s on etcd instance {{ $labels.instance }}. | warning | +| **etcdMembersDown** | etcd cluster members are down. | etcd cluster "{{ $labels.job }}": members are down ({{ $value }}). | warning | +| **etcdNoLeader** | etcd cluster has no leader. | etcd cluster "{{ $labels.job }}": member {{ $labels.instance }} has no leader. | critical | +

🔝 Back to Top

+ +--- + +## fluentbit serviceMonitor alert +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **MissingFluentbitServiceMonitor** | ServiceMonitor 'fluentbit-fluent-bit' is either down or missing. | Check if the Fluentbit ServiceMonitor is properly configured and deployed.
| critical | +

🔝 Back to Top

+ +--- + +## general.rules +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **InfoInhibitor** | Info-level alert inhibition. | This is an alert that is used to inhibit info alerts.
By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with
other alerts.
This alert fires whenever there's a severity="info" alert, and stops firing when another alert with a
severity of 'warning' or 'critical' starts firing on the same namespace.
This alert should be routed to a null receiver and configured to inhibit alerts with severity="info".
| none | +| **TargetDown** | One or more targets are unreachable. | {{ printf "%.4g" $value }}% of the {{ $labels.job }}/{{ $labels.service }} targets in {{ $labels.namespace }} namespace are down. | warning | +| **Watchdog** | An alert that should always be firing to certify that Alertmanager is working properly. | This is an alert meant to ensure that the entire alerting pipeline is functional.
This alert is always firing, therefore it should always be firing in Alertmanager
and always fire against a receiver. There are integrations with various notification
mechanisms that send a notification when this alert is not firing. For example the
"DeadMansSnitch" integration in PagerDuty.
| none | +

🔝 Back to Top

+ +--- + +## kube-apiserver-slos +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **KubeAPIErrorBudgetBurn** | The API server is burning too much error budget. | The API server is burning too much error budget on cluster {{ $labels.cluster }}. | critical | +| **KubeAPIErrorBudgetBurn** | The API server is burning too much error budget. | The API server is burning too much error budget on cluster {{ $labels.cluster }}. | critical | +| **KubeAPIErrorBudgetBurn** | The API server is burning too much error budget. | The API server is burning too much error budget on cluster {{ $labels.cluster }}. | warning | +| **KubeAPIErrorBudgetBurn** | The API server is burning too much error budget. | The API server is burning too much error budget on cluster {{ $labels.cluster }}. | warning | +

🔝 Back to Top

+ +--- + +## kube-state-metrics +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **KubeStateMetricsListErrors** | kube-state-metrics is experiencing errors in list operations. | kube-state-metrics is experiencing errors at an elevated rate in list operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. | critical | +| **KubeStateMetricsShardingMismatch** | kube-state-metrics sharding is misconfigured. | kube-state-metrics pods are running with different --total-shards configuration, some Kubernetes objects may be exposed multiple times or not exposed at all. | critical | +| **KubeStateMetricsShardsMissing** | kube-state-metrics shards are missing. | kube-state-metrics shards are missing, some Kubernetes objects are not being exposed. | critical | +| **KubeStateMetricsWatchErrors** | kube-state-metrics is experiencing errors in watch operations. | kube-state-metrics is experiencing errors at an elevated rate in watch operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. | critical | +

🔝 Back to Top

+ +--- + +## kubernetes-apps +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **KubeContainerWaiting** | Pod container waiting longer than 1 hour | pod/{{ $labels.pod }} in namespace {{ $labels.namespace }} on container {{ $labels.container}} has been in waiting state for longer than 1 hour. (reason: "{{ $labels.reason }}") on cluster {{ $labels.cluster }}. | warning | +| **KubeDaemonSetMisScheduled** | DaemonSet pods are misscheduled. | {{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run on cluster {{ $labels.cluster }}. | warning | +| **KubeDaemonSetNotScheduled** | DaemonSet pods are not scheduled. | {{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled on cluster {{ $labels.cluster }}. | warning | +| **KubeDaemonSetRolloutStuck** | DaemonSet rollout is stuck. | DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not finished or progressed for at least 15m on cluster {{ $labels.cluster }}. | warning | +| **KubeDeploymentGenerationMismatch** | Deployment generation mismatch due to possible roll-back | Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment }} does not match, this indicates that the Deployment has failed but has not been rolled back on cluster {{ $labels.cluster }}. | warning | +| **KubeDeploymentReplicasMismatch** | Deployment has not matched the expected number of replicas. | Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than 15 minutes on cluster {{ $labels.cluster }}. | warning | +| **KubeDeploymentRolloutStuck** | Deployment rollout is not progressing. | Rollout of deployment {{ $labels.namespace }}/{{ $labels.deployment }} is not progressing for longer than 15 minutes on cluster {{ $labels.cluster }}. | warning | +| **KubeHpaMaxedOut** | HPA is running at max replicas | HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has been running at max replicas for longer than 15 minutes on cluster {{ $labels.cluster }}. | warning | +| **KubeHpaReplicasMismatch** | HPA has not matched desired number of replicas. | HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has not matched the desired number of replicas for longer than 15 minutes on cluster {{ $labels.cluster }}. | warning | +| **KubeJobFailed** | Job failed to complete. | Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. Removing failed job after investigation should clear this alert on cluster {{ $labels.cluster }}. | warning | +| **KubeJobNotCompleted** | Job did not complete in time | Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than {{ "43200" \| humanizeDuration }} to complete on cluster {{ $labels.cluster }}. | warning | +| **KubePodCrashLooping** | Pod is crash looping. | Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is in waiting state (reason: "CrashLoopBackOff") on cluster {{ $labels.cluster }}. | warning | +| **KubePodNotReady** | Pod has been in a non-ready state for more than 15 minutes. | Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes on cluster {{ $labels.cluster }}. | warning | +| **KubeStatefulSetGenerationMismatch** | StatefulSet generation mismatch due to possible roll-back | StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset }} does not match, this indicates that the StatefulSet has failed but has not been rolled back on cluster {{ $labels.cluster }}. | warning | +| **KubeStatefulSetReplicasMismatch** | StatefulSet has not matched the expected number of replicas. | StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 15 minutes on cluster {{ $labels.cluster }}. | warning | +| **KubeStatefulSetUpdateNotRolledOut** | StatefulSet update has not been rolled out. | StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out on cluster {{ $labels.cluster }}. | warning | +

🔝 Back to Top

+ +--- + +## kubernetes-resources +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **CPUThrottlingHigh** | Processes experience elevated CPU throttling. | {{ $value \| humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }} on cluster {{ $labels.cluster }}. | info | +| **KubeCPUOvercommit** | Cluster has overcommitted CPU resource requests. | Cluster {{ $labels.cluster }} has overcommitted CPU resource requests for Pods by {{ $value }} CPU shares and cannot tolerate node failure. | warning | +| **KubeCPUQuotaOvercommit** | Cluster has overcommitted CPU resource requests. | Cluster {{ $labels.cluster }} has overcommitted CPU resource requests for Namespaces. | warning | +| **KubeMemoryOvercommit** | Cluster has overcommitted memory resource requests. | Cluster {{ $labels.cluster }} has overcommitted memory resource requests for Pods by {{ $value \| humanize }} bytes and cannot tolerate node failure. | warning | +| **KubeMemoryQuotaOvercommit** | Cluster has overcommitted memory resource requests. | Cluster {{ $labels.cluster }} has overcommitted memory resource requests for Namespaces. | warning | +| **KubeQuotaAlmostFull** | Namespace quota is going to be full. | Namespace {{ $labels.namespace }} is using {{ $value \| humanizePercentage }} of its {{ $labels.resource }} quota on cluster {{ $labels.cluster }}. | info | +| **KubeQuotaExceeded** | Namespace quota has exceeded the limits. | Namespace {{ $labels.namespace }} is using {{ $value \| humanizePercentage }} of its {{ $labels.resource }} quota on cluster {{ $labels.cluster }}. | warning | +| **KubeQuotaFullyUsed** | Namespace quota is fully used. | Namespace {{ $labels.namespace }} is using {{ $value \| humanizePercentage }} of its {{ $labels.resource }} quota on cluster {{ $labels.cluster }}. | info | +

🔝 Back to Top

+ +--- + +## kubernetes-storage +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **KubePersistentVolumeErrors** | PersistentVolume is having issues with provisioning. | The persistent volume {{ $labels.persistentvolume }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} has status {{ $labels.phase }}. | critical | +| **KubePersistentVolumeFillingUp** | PersistentVolume is filling up. | The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is only {{ $value \| humanizePercentage }} free. | critical | +| **KubePersistentVolumeFillingUp** | PersistentVolume is filling up. | Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is expected to fill up within four days. Currently {{ $value \| humanizePercentage }} is available. | warning | +| **KubePersistentVolumeInodesFillingUp** | PersistentVolumeInodes are filling up. | The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} only has {{ $value \| humanizePercentage }} free inodes. | critical | +| **KubePersistentVolumeInodesFillingUp** | PersistentVolumeInodes are filling up. | Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is expected to run out of inodes within four days. Currently {{ $value \| humanizePercentage }} of its inodes are free. | warning | +

🔝 Back to Top

+ +--- + +## kubernetes-system +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **KubeClientErrors** | Kubernetes API server client is experiencing errors. | Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ $value \| humanizePercentage }} errors on cluster {{ $labels.cluster }}. | warning | +| **KubeVersionMismatch** | Different semantic versions of Kubernetes components running. | There are {{ $value }} different semantic versions of Kubernetes components running on cluster {{ $labels.cluster }}. | warning | +

🔝 Back to Top

+ +--- + +## kubernetes-system-apiserver +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **KubeAPIDown** | Target disappeared from Prometheus target discovery. | KubeAPI has disappeared from Prometheus target discovery. | critical | +| **KubeAPITerminatedRequests** | The kubernetes apiserver has terminated {{ $value \| humanizePercentage }} of its incoming requests. | The kubernetes apiserver has terminated {{ $value \| humanizePercentage }} of its incoming requests on cluster {{ $labels.cluster }}. | warning | +| **KubeAggregatedAPIDown** | Kubernetes aggregated API is down. | Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value \| humanize }}% available over the last 10m on cluster {{ $labels.cluster }}. | warning | +| **KubeAggregatedAPIErrors** | Kubernetes aggregated API has reported errors. | Kubernetes aggregated API {{ $labels.instance }}/{{ $labels.name }} has reported {{ $labels.reason }} errors on cluster {{ $labels.cluster }}. | warning | +| **KubeClientCertificateExpiration** | Client certificate is about to expire. | A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days on cluster {{ $labels.cluster }}. | warning | +| **KubeClientCertificateExpiration** | Client certificate is about to expire. | A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours on cluster {{ $labels.cluster }}. | critical | +

🔝 Back to Top

+ +--- + +## kubernetes-system-controller-manager +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **KubeControllerManagerDown** | Target disappeared from Prometheus target discovery. | KubeControllerManager has disappeared from Prometheus target discovery. | critical | +

🔝 Back to Top

+ +--- + +## kubernetes-system-kube-proxy +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **KubeProxyDown** | Target disappeared from Prometheus target discovery. | KubeProxy has disappeared from Prometheus target discovery. | critical | +

🔝 Back to Top

+ +--- + +## kubernetes-system-kubelet +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **KubeNodeEviction** | Node is evicting pods. | Node {{ $labels.node }} on {{ $labels.cluster }} is evicting Pods due to {{ $labels.eviction_signal }}. Eviction occurs when eviction thresholds are crossed, typically caused by Pods exceeding RAM/ephemeral-storage limits. | info | +| **KubeNodeNotReady** | Node is not ready. | {{ $labels.node }} has been unready for more than 15 minutes on cluster {{ $labels.cluster }}. | warning | +| **KubeNodePressure** | Node has as active Condition. | {{ $labels.node }} on cluster {{ $labels.cluster }} has active Condition {{ $labels.condition }}. This is caused by resource usage exceeding eviction thresholds. | info | +| **KubeNodeReadinessFlapping** | Node readiness status is flapping. | The readiness status of node {{ $labels.node }} has changed {{ $value }} times in the last 15 minutes on cluster {{ $labels.cluster }}. | warning | +| **KubeNodeUnreachable** | Node is unreachable. | {{ $labels.node }} is unreachable and some workloads may be rescheduled on cluster {{ $labels.cluster }}. | warning | +| **KubeletClientCertificateExpiration** | Kubelet client certificate is about to expire. | Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value \| humanizeDuration }} on cluster {{ $labels.cluster }}. | warning | +| **KubeletClientCertificateExpiration** | Kubelet client certificate is about to expire. | Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value \| humanizeDuration }} on cluster {{ $labels.cluster }}. | critical | +| **KubeletClientCertificateRenewalErrors** | Kubelet has failed to renew its client certificate. | Kubelet on node {{ $labels.node }} has failed to renew its client certificate ({{ $value \| humanize }} errors in the last 5 minutes) on cluster {{ $labels.cluster }}. | warning | +| **KubeletDown** | Target disappeared from Prometheus target discovery. | Kubelet has disappeared from Prometheus target discovery. | critical | +| **KubeletPlegDurationHigh** | Kubelet Pod Lifecycle Event Generator is taking too long to relist. | The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }} on cluster {{ $labels.cluster }}. | warning | +| **KubeletPodStartUpLatencyHigh** | Kubelet Pod startup latency is too high. | Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }} on cluster {{ $labels.cluster }}. | warning | +| **KubeletServerCertificateExpiration** | Kubelet server certificate is about to expire. | Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value \| humanizeDuration }} on cluster {{ $labels.cluster }}. | warning | +| **KubeletServerCertificateExpiration** | Kubelet server certificate is about to expire. | Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value \| humanizeDuration }} on cluster {{ $labels.cluster }}. | critical | +| **KubeletServerCertificateRenewalErrors** | Kubelet has failed to renew its server certificate. | Kubelet on node {{ $labels.node }} has failed to renew its server certificate ({{ $value \| humanize }} errors in the last 5 minutes) on cluster {{ $labels.cluster }}. | warning | +| **KubeletTooManyPods** | Kubelet is running at capacity. | Kubelet '{{ $labels.node }}' is running at {{ $value \| humanizePercentage }} of its Pod capacity on cluster {{ $labels.cluster }}. | info | +

🔝 Back to Top

+ +--- + +## kubernetes-system-scheduler +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **KubeSchedulerDown** | Target disappeared from Prometheus target discovery. | KubeScheduler has disappeared from Prometheus target discovery. | critical | +

🔝 Back to Top

+ +--- + +## mariadb-alerts +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **MariaDBDown** | MariaDB not up and running, immediate attention is required. | MariaDB {{$labels.job}} on {{$labels.instance}} is not up. | critical | +| **MariaDBReplicationErrors** | MariaDB is reporting replication errors from {{$labels.instance}}, immediate attention is required. | MariaDB {{$labels.job}} on {{$labels.instance}} is reporting replication errors. | critical | +| **MysqlSlaveReplicationLag** | MySQL Slave replication lag (instance {{ $labels.instance }}) | MySQL replication lag on {{ $labels.instance }}
VALUE = {{ $value }}
LABELS = {{ $labels }} | critical | +| **MysqlTooManyConnections(>80%)** | MySQL too many connections (> 80%) (instance {{ $labels.instance }}) | More than 80% of MySQL connections are in use on {{ $labels.instance }}
VALUE = {{ $value }}
LABELS = {{ $labels }} | warning | +

🔝 Back to Top

+ +--- + +## node-exporter +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **NodeBondingDegraded** | Bonding interface is degraded | Bonding interface {{ $labels.master }} on {{ $labels.instance }} is in degraded state due to one or more slave failures. | warning | +| **NodeCPUHighUsage** | High CPU usage. | CPU usage at {{ $labels.instance }} has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.
| info | +| **NodeClockNotSynchronising** | Clock not synchronising. | Clock at {{ $labels.instance }} is not synchronising. Ensure NTP is configured on this host. | warning | +| **NodeClockSkewDetected** | Clock skew detected. | Clock at {{ $labels.instance }} is out of sync by more than 0.05s. Ensure NTP is configured correctly on this host. | warning | +| **NodeDiskIOSaturation** | Disk IO queue is high. | Disk IO queue (aqu-sq) is high on {{ $labels.device }} at {{ $labels.instance }}, has been above 10 for the last 30 minutes, is currently at {{ printf "%.2f" $value }}.
This symptom might indicate disk saturation.
| warning | +| **NodeFileDescriptorLimit** | Kernel is predicted to exhaust file descriptors limit soon. | File descriptors limit at {{ $labels.instance }} is currently at {{ printf "%.2f" $value }}%. | warning | +| **NodeFileDescriptorLimit** | Kernel is predicted to exhaust file descriptors limit soon. | File descriptors limit at {{ $labels.instance }} is currently at {{ printf "%.2f" $value }}%. | critical | +| **NodeFilesystemAlmostOutOfFiles** | Filesystem has less than 5% inodes left. | Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left. | warning | +| **NodeFilesystemAlmostOutOfFiles** | Filesystem has less than 3% inodes left. | Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left. | critical | +| **NodeFilesystemAlmostOutOfSpace** | Filesystem has less than 5% space left. | Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left. | warning | +| **NodeFilesystemAlmostOutOfSpace** | Filesystem has less than 3% space left. | Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left. | critical | +| **NodeFilesystemFilesFillingUp** | Filesystem is predicted to run out of inodes within the next 24 hours. | Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up. | warning | +| **NodeFilesystemFilesFillingUp** | Filesystem is predicted to run out of inodes within the next 4 hours. | Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up fast. | critical | +| **NodeFilesystemSpaceFillingUp** | Filesystem is predicted to run out of space within the next 24 hours. | Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left and is filling up. | warning | +| **NodeFilesystemSpaceFillingUp** | Filesystem is predicted to run out of space within the next 4 hours. | Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left and is filling up fast. | critical | +| **NodeHighNumberConntrackEntriesUsed** | Number of conntrack are getting close to the limit. | {{ $labels.instance }} {{ $value \| humanizePercentage }} of conntrack entries are used. | warning | +| **NodeMemoryHighUtilization** | Host is running out of memory. | Memory is filling up at {{ $labels.instance }}, has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.
| warning | +| **NodeMemoryMajorPagesFaults** | Memory major page faults are occurring at very high rate. | Memory major pages are occurring at very high rate at {{ $labels.instance }}, 500 major page faults per second for the last 15 minutes, is currently at {{ printf "%.2f" $value }}.
Please check that there is enough memory available at this instance.
| warning | +| **NodeNetworkReceiveErrs** | Network interface is reporting many receive errors. | {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} receive errors in the last two minutes. | warning | +| **NodeNetworkTransmitErrs** | Network interface is reporting many transmit errors. | {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} transmit errors in the last two minutes. | warning | +| **NodeRAIDDegraded** | RAID Array is degraded. | RAID array '{{ $labels.device }}' at {{ $labels.instance }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically. | critical | +| **NodeRAIDDiskFailure** | Failed device in RAID array. | At least one device in RAID array at {{ $labels.instance }} failed. Array '{{ $labels.device }}' needs attention and possibly a disk swap. | warning | +| **NodeSystemSaturation** | System saturated, load per core is very high. | System load per core at {{ $labels.instance }} has been above 2 for the last 15 minutes, is currently at {{ printf "%.2f" $value }}.
This might indicate this instance resources saturation and can cause it becoming unresponsive.
| warning | +| **NodeSystemdServiceCrashlooping** | Systemd service keeps restaring, possibly crash looping. | Systemd service {{ $labels.name }} has being restarted too many times at {{ $labels.instance }} for the last 15 minutes. Please check if service is crash looping. | warning | +| **NodeSystemdServiceFailed** | Systemd service has entered failed state. | Systemd service {{ $labels.name }} has entered failed state at {{ $labels.instance }} | warning | +| **NodeTextFileCollectorScrapeError** | Node Exporter text file collector failed to scrape. | Node Exporter text file collector on {{ $labels.instance }} failed to scrape. | warning | +

🔝 Back to Top

+ +--- + +## node-network +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **NodeNetworkInterfaceFlapping** | Network interface is often changing its status | Network interface "{{ $labels.device }}" changing its up status often on node-exporter {{ $labels.namespace }}/{{ $labels.pod }} | warning | +

🔝 Back to Top

+ +--- + +## pod-state-alerts +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **HighPodRestartRate** | High pod restart count detected | Pod {{ $labels.pod }} in namespace {{ $labels.namespace }} is restarting frequently, which may indicate network instability. | warning | +| **KubePodNotReadyCritical** | Pod has been in a non-ready state for more than 5 minutes. | Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 5 minutes. | critical | +| **TooManyContainerRestarts** | Container named {{ $labels.container }} in {{ $labels.pod }} in {{ $labels.namespace }} has restarted too many times in a short period and needs to be investigated. | Namespace: {{$labels.namespace}}
Pod name: {{$labels.pod}}
Container name: {{$labels.container}}
| critical | +

🔝 Back to Top

+ +--- + +## prometheus +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **PrometheusBadConfig** | Failed Prometheus configuration reload. | Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to reload its configuration. | critical | +| **PrometheusDuplicateTimestamps** | Prometheus is dropping samples with duplicate timestamps. | Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ printf "%.4g" $value }} samples/s with different values but duplicated timestamp. | warning | +| **PrometheusErrorSendingAlertsToAnyAlertmanager** | Prometheus encounters more than 3% errors sending alerts to any Alertmanager. | {{ printf "%.1f" $value }}% minimum errors while sending alerts from Prometheus {{$labels.namespace}}/{{$labels.pod}} to any Alertmanager. | critical | +| **PrometheusErrorSendingAlertsToSomeAlertmanagers** | More than 1% of alerts sent by Prometheus to a specific Alertmanager were affected by errors. | {{ printf "%.1f" $value }}% of alerts sent by Prometheus {{$labels.namespace}}/{{$labels.pod}} to Alertmanager {{$labels.alertmanager}} were affected by errors. | warning | +| **PrometheusHighQueryLoad** | Prometheus is reaching its maximum capacity serving concurrent requests. | Prometheus {{$labels.namespace}}/{{$labels.pod}} query API has less than 20% available capacity in its query engine for the last 15 minutes. | warning | +| **PrometheusKubernetesListWatchFailures** | Requests in Kubernetes SD are failing. | Kubernetes service discovery of Prometheus {{$labels.namespace}}/{{$labels.pod}} is experiencing {{ printf "%.0f" $value }} failures with LIST/WATCH requests to the Kubernetes API in the last 5 minutes. | warning | +| **PrometheusLabelLimitHit** | Prometheus has dropped targets because some scrape configs have exceeded the labels limit. | Prometheus {{$labels.namespace}}/{{$labels.pod}} has dropped {{ printf "%.0f" $value }} targets because some samples exceeded the configured label_limit, label_name_length_limit or label_value_length_limit. | warning | +| **PrometheusMissingRuleEvaluations** | Prometheus is missing rule evaluations due to slow rule group evaluation. | Prometheus {{$labels.namespace}}/{{$labels.pod}} has missed {{ printf "%.0f" $value }} rule group evaluations in the last 5m. | warning | +| **PrometheusNotConnectedToAlertmanagers** | Prometheus is not connected to any Alertmanagers. | Prometheus {{$labels.namespace}}/{{$labels.pod}} is not connected to any Alertmanagers. | warning | +| **PrometheusNotIngestingSamples** | Prometheus is not ingesting samples. | Prometheus {{$labels.namespace}}/{{$labels.pod}} is not ingesting samples. | warning | +| **PrometheusNotificationQueueRunningFull** | Prometheus alert notification queue predicted to run full in less than 30m. | Alert notification queue of Prometheus {{$labels.namespace}}/{{$labels.pod}} is running full. | warning | +| **PrometheusOutOfOrderTimestamps** | Prometheus drops samples with out-of-order timestamps. | Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ printf "%.4g" $value }} samples/s with timestamps arriving out of order. | warning | +| **PrometheusRemoteStorageFailures** | Prometheus fails to send samples to remote storage. | Prometheus {{$labels.namespace}}/{{$labels.pod}} failed to send {{ printf "%.1f" $value }}% of the samples to {{ $labels.remote_name}}:{{ $labels.url }} | critical | +| **PrometheusRemoteWriteBehind** | Prometheus remote write is behind. | Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write is {{ printf "%.1f" $value }}s behind for {{ $labels.remote_name}}:{{ $labels.url }}. | critical | +| **PrometheusRemoteWriteDesiredShards** | Prometheus remote write desired shards calculation wants to run more than configured max shards. | Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write desired shards calculation wants to run {{ $value }} shards for queue {{ $labels.remote_name}}:{{ $labels.url }}, which is more than the max of {{ printf `prometheus_remote_storage_shards_max{instance="%s",job="kube-prometheus-stack-prometheus",namespace="prometheus"}` $labels.instance \| query \| first \| value }}. | warning | +| **PrometheusRuleFailures** | Prometheus is failing rule evaluations. | Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to evaluate {{ printf "%.0f" $value }} rules in the last 5m. | critical | +| **PrometheusSDRefreshFailure** | Failed Prometheus SD refresh. | Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to refresh SD with mechanism {{$labels.mechanism}}. | warning | +| **PrometheusScrapeBodySizeLimitHit** | Prometheus has dropped some targets that exceeded body size limit. | Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed {{ printf "%.0f" $value }} scrapes in the last 5m because some targets exceeded the configured body_size_limit. | warning | +| **PrometheusScrapeSampleLimitHit** | Prometheus has failed scrapes that have exceeded the configured sample limit. | Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed {{ printf "%.0f" $value }} scrapes in the last 5m because some targets exceeded the configured sample_limit. | warning | +| **PrometheusTSDBCompactionsFailing** | Prometheus has issues compacting blocks. | Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value \| humanize}} compaction failures over the last 3h. | warning | +| **PrometheusTSDBReloadsFailing** | Prometheus has issues reloading blocks from disk. | Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value \| humanize}} reload failures over the last 3h. | warning | +| **PrometheusTargetLimitHit** | Prometheus has dropped targets because some scrape configs have exceeded the targets limit. | Prometheus {{$labels.namespace}}/{{$labels.pod}} has dropped {{ printf "%.0f" $value }} targets because the number of targets exceeded the configured target_limit. | warning | +| **PrometheusTargetSyncFailure** | Prometheus has failed to sync targets. | {{ printf "%.0f" $value }} targets in Prometheus {{$labels.namespace}}/{{$labels.pod}} have failed to sync because invalid configuration was supplied. | critical | +

🔝 Back to Top

+ +--- + +## prometheus-operator +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **PrometheusOperatorListErrors** | Errors while performing list operations in controller. | Errors while performing List operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace. | warning | +| **PrometheusOperatorNodeLookupErrors** | Errors while reconciling Prometheus. | Errors while reconciling Prometheus in {{ $labels.namespace }} Namespace. | warning | +| **PrometheusOperatorNotReady** | Prometheus operator not ready | Prometheus operator in {{ $labels.namespace }} namespace isn't ready to reconcile {{ $labels.controller }} resources. | warning | +| **PrometheusOperatorReconcileErrors** | Errors while reconciling objects. | {{ $value \| humanizePercentage }} of reconciling operations failed for {{ $labels.controller }} controller in {{ $labels.namespace }} namespace. | warning | +| **PrometheusOperatorRejectedResources** | Resources rejected by Prometheus operator | Prometheus operator in {{ $labels.namespace }} namespace rejected {{ printf "%0.0f" $value }} {{ $labels.controller }}/{{ $labels.resource }} resources. | warning | +| **PrometheusOperatorStatusUpdateErrors** | Errors while updating objects status. | {{ $value \| humanizePercentage }} of status update operations failed for {{ $labels.controller }} controller in {{ $labels.namespace }} namespace. | warning | +| **PrometheusOperatorSyncFailed** | Last controller reconciliation failed | Controller {{ $labels.controller }} in {{ $labels.namespace }} namespace fails to reconcile {{ $value }} objects. | warning | +| **PrometheusOperatorWatchErrors** | Errors while performing watch operations in controller. | Errors while performing watch operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace. | warning | +

🔝 Back to Top

+ +--- + +## rabbitmq +| Alert Name | Summary | Description | Severity | +| :--- | :--- | :--- | :--- | +| **ContainerRestarts** | Investigate why the container got restarted.
Check the logs of the current container: `kubectl -n {{ $labels.namespace }} logs {{ $labels.pod }}`
Check the logs of the previous container: `kubectl -n {{ $labels.namespace }} logs {{ $labels.pod }} --previous`
Check the last state of the container: `kubectl -n {{ $labels.namespace }} get pod {{ $labels.pod }} -o jsonpath='{.status.containerStatuses[].lastState}'`
| Over the last 10 minutes, container `{{ $labels.container }}`
restarted `{{ $value \| printf "%.0f" }}` times in pod `{{ $labels.pod }}` of RabbitMQ cluster
`{{ $labels.rabbitmq_cluster }}` in namespace `{{ $labels.namespace }}`.
| warning | +| **FileDescriptorsNearLimit** | More than 80% of file descriptors are used on the RabbitMQ node.
When this value reaches 100%, new connections will not be accepted and disk write operations may fail.
Client libraries, peer nodes and CLI tools will not be able to connect when the node runs out of available file descriptors.
See https://www.rabbitmq.com/production-checklist.html#resource-limits-file-handle-limit.
| `{{ $value \| humanizePercentage }}` file descriptors of file
descriptor limit are used in RabbitMQ node `{{ $labels.rabbitmq_node }}`,
pod `{{ $labels.pod }}`, RabbitMQ cluster `{{ $labels.rabbitmq_cluster }}`,
namespace `{{ $labels.namespace }}`.
| warning | +| **HighConnectionChurn** | More than 10% of total connections are churning.
This means that client application connections are short-lived instead of long-lived.
Read https://www.rabbitmq.com/connections.html#high-connection-churn to understand why this is an anti-pattern.
| Over the last 5 minutes, `{{ $value \| humanizePercentage }}`
of total connections are closed or opened per second in RabbitMQ cluster `{{ $labels.rabbitmq_cluster }}`
in namespace `{{ $labels.namespace }}`.
| warning | +| **InsufficientEstablishedErlangDistributionLinks** | RabbitMQ clusters have a full mesh topology.
All RabbitMQ nodes connect to all other RabbitMQ nodes in both directions.
The expected number of established Erlang distribution links is therefore `n*(n-1)` where `n` is the number of RabbitMQ nodes in the cluster.
Therefore, the expected number of distribution links are `0` for a 1-node cluster, `6` for a 3-node cluster, and `20` for a 5-node cluster.
This alert reports that the number of established distributions links is less than the expected number.
Some reasons for this alert include failed network links, network partitions, failed clustering (i.e. nodes can't join the cluster).
Check the panels `All distribution links`, `Established distribution links`, `Connecting distributions links`, `Waiting distribution links`, and `distribution links`
of the Grafana dashboard `Erlang-Distribution`.
Check the logs of the RabbitMQ nodes: `kubectl -n {{ $labels.namespace }} logs -l app.kubernetes.io/component=rabbitmq,app.kubernetes.io/name={{ $labels.rabbitmq_cluster }}`
| There are only `{{ $value }}` established Erlang distribution links
in RabbitMQ cluster `{{ $labels.rabbitmq_cluster }}` in namespace `{{ $labels.namespace }}`.
| warning | +| **LowDiskWatermarkPredicted** | Based on the trend of available disk space over the past 24 hours, it's predicted that, in 24 hours from now, a disk alarm will be triggered since the free disk space will drop below the free disk space limit.
This alert is reported for the partition where the RabbitMQ data directory is stored.
When the disk alarm will be triggered, all publishing connections across all cluster nodes will be blocked.
See
https://www.rabbitmq.com/alarms.html,
https://www.rabbitmq.com/disk-alarms.html,
https://www.rabbitmq.com/production-checklist.html#resource-limits-disk-space,
https://www.rabbitmq.com/persistence-conf.html,
https://www.rabbitmq.com/connection-blocked.html.
| The predicted free disk space in 24 hours from now is `{{ $value \| humanize1024 }}B`
in RabbitMQ node `{{ $labels.rabbitmq_node }}`, pod `{{ $labels.pod }}`,
RabbitMQ cluster `{{ $labels.rabbitmq_cluster }}`, namespace `{{ $labels.namespace }}`.
| warning | +| **MemoryAlarm** | A RabbitMQ node reached the `vm_memory_high_watermark` threshold.
See https://www.rabbitmq.com/docs/alarms#overview, https://www.rabbitmq.com/docs/memory.
| RabbitMQ cluster `{{ $labels.rabbitmq_cluster }}` memory alarm active. Publishers are blocked.
| warning | +| **NoMajorityOfNodesReady** | No majority of nodes have been ready for the last 5 minutes.
Check the details of the pods:
`kubectl -n {{ $labels.namespace }} describe pods -l app.kubernetes.io/component=rabbitmq,app.kubernetes.io/name={{ $labels.label_app_kubernetes_io_name }}`
| Only `{{ $value }}` replicas are ready in StatefulSet `{{ $labels.statefulset }}`
of RabbitMQ cluster `{{ $labels.label_app_kubernetes_io_name }}` in namespace `{{ $labels.namespace }}`.
| warning | +| **PersistentVolumeMissing** | RabbitMQ needs a PersistentVolume for its data.
However, there is no PersistentVolume bound to the PersistentVolumeClaim.
This means the requested storage could not be provisioned.
Check the status of the PersistentVolumeClaim: `kubectl -n {{ $labels.namespace }} describe pvc {{ $labels.persistentvolumeclaim }}`.
| PersistentVolumeClaim `{{ $labels.persistentvolumeclaim }}` of
RabbitMQ cluster `{{ $labels.label_app_kubernetes_io_name }}` in namespace
`{{ $labels.namespace }}` is not bound.
| critical | +| **QueueHasNoConsumers** | Messages are sitting idle in the queue, without any processing.
This alert is highly application specific (and e.g. doesn't make sense for stream queues).
| Over the last 10 minutes, non-empty queue `{{ $labels.queue }}` with {{ $value }} messages
in virtual host `{{ $labels.vhost }}` didn't have any consumers in
RabbitMQ cluster `{{ $labels.rabbitmq_cluster }}` in namespace `{{ $labels.namespace }}`.
| warning | +| **QueueIsGrowing** | Queue size is steadily growing over time.
| Over the last 10 minutes, queue `{{ $labels.queue }}` in virtual host `{{ $labels.vhost }}`
was growing. 10 minute moving average has grown by {{ $value }}.
This happens in RabbitMQ cluster `{{ $labels.rabbitmq_cluster }}` in namespace `{{ $labels.namespace }}`.
| warning | +| **RabbitmqDiskAlarm** | A RabbitMQ node reached the `disk_free_limit` threshold.
See https://www.rabbitmq.com/docs/alarms#overview, https://www.rabbitmq.com/docs/disk-alarms.
| RabbitMQ cluster `{{ $labels.rabbitmq_cluster }}` disk alarm active. Publishers are blocked.
| warning | +| **RabbitmqFileDescriptorAlarm** | A RabbitMQ node ran out of file descriptors.
See https://www.rabbitmq.com/docs/alarms#file-descriptors.
| RabbitMQ cluster `{{ $labels.rabbitmq_cluster }}` file descriptor alarm active. Publishers are blocked.
| warning | +| **TCPSocketsNearLimit** | More than 80% of TCP sockets are open on the RabbitMQ node.
When this value reaches 100%, new connections will not be accepted.
Client libraries, peer nodes and CLI tools will not be able to connect when the node runs out of available TCP sockets.
See https://www.rabbitmq.com/networking.html.
| `{{ $value \| humanizePercentage }}` TCP sockets of TCP socket
limit are open in RabbitMQ node `{{ $labels.rabbitmq_node }}`, pod `{{ $labels.pod }}`,
RabbitMQ cluster `{{ $labels.rabbitmq_cluster }}`, namespace `{{ $labels.namespace }}`.
| warning | +| **UnroutableMessages** | There are messages published into an exchange which cannot be routed and are either dropped silently, or returned to publishers.
Is your routing topology set up correctly?
Check your application code and bindings between exchanges and queues.
See
https://www.rabbitmq.com/publishers.html#unroutable,
https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed.
| There were `{{ $value \| printf "%.0f" }}` unroutable messages within the last
5 minutes in RabbitMQ cluster `{{ $labels.rabbitmq_cluster }}` in namespace
`{{ $labels.namespace }}`.
| warning | +

🔝 Back to Top

+ +--- + diff --git a/docs/monitoring-info.md b/docs/monitoring-info.md index 3a49ac836..f89796db0 100644 --- a/docs/monitoring-info.md +++ b/docs/monitoring-info.md @@ -170,6 +170,12 @@ Currently, in Genestack the textfile-collector is used to collect kernel-taint s This is currently the complete list of exporters and monitoring callouts deployed within the Genestack workflow. That said, Genestack is constantly evolving and list may grow or change entirely as we look to further improve our systems! With all these metrics available we need a way to visualize them to get a better picture of our systems and their health, we'll discuss that next! +* ### Kubernetes Event Exporter +Kubernetes clusters are constantly sending events that contain potentially important data that should be captured. +With the [Kubernetes Event Exporter](https://github.com/resmoio/kubernetes-event-exporter) we can capture these events to gain a better view of what our cluster is doing. +This exporter also includes built in alerting mechanisms for things like Slack and MSTeams that can be configured to send messages when specific events are seen. +View the [Kubernetes Event Exporter Install Instructions](prometheus-kube-event-exporter.md) to get this exporter installed. + ## Visualization In Genestack we deploy [Grafana](https://grafana.com/) as our default visualization tool. Grafana is open-sourced tooling which aligns well with the Genestack ethos while providing seamless visualization of the metrics generated by our systems. From cfd8547dba863514875a1450267fe11fa350299d Mon Sep 17 00:00:00 2001 From: manojacloud <112391328+manojacloud@users.noreply.github.com> Date: Fri, 25 Jul 2025 13:33:13 +0530 Subject: [PATCH 35/97] chore: Add blazar image to genestack repo (#1090) Adding blazer image to genestack repo so that we can use this image to setup blazer. --- .original-images.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.original-images.json b/.original-images.json index 3961464ef..3d1556f71 100644 --- a/.original-images.json +++ b/.original-images.json @@ -46,5 +46,6 @@ "quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_jammy", "quay.io/airshipit/porthole-postgresql-utility:latest-ubuntu_bionic", "quay.io/airshipit/freezer:2025.1-ubuntu_jammy", - "quay.io/airshipit/freezer-api:2025.1-ubuntu_jammy" + "quay.io/airshipit/freezer-api:2025.1-ubuntu_jammy", + "quay.io/airshipit/blazar:2025.1-ubuntu_jammy" ] From 521a2eca1d6fd883193282831da29701c6482130 Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Fri, 25 Jul 2025 08:49:40 -0500 Subject: [PATCH 36/97] feat: allow extra_specs to be read by admin_or_reader --- base-helm-configs/cinder/cinder-helm-overrides.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/base-helm-configs/cinder/cinder-helm-overrides.yaml b/base-helm-configs/cinder/cinder-helm-overrides.yaml index bdd0222dd..4c56603f7 100644 --- a/base-helm-configs/cinder/cinder-helm-overrides.yaml +++ b/base-helm-configs/cinder/cinder-helm-overrides.yaml @@ -103,6 +103,8 @@ conf: volume_clear: zero volume_driver: cinder_rxt.rackspace.RXTLVM volume_group: cinder-volumes-1 + policy: + "volume_extension:types_extra_specs:read_sensitive": "rule:xena_system_admin_or_project_reader" cinder: DEFAULT: allow_availability_zone_fallback: true From 096ff187e5a2e2a40fcd0298822f3e2d160243b6 Mon Sep 17 00:00:00 2001 From: Sulochan Acharya Date: Mon, 28 Jul 2025 19:45:12 +0545 Subject: [PATCH 37/97] Add magnum capi driver image to image list (#1095) --- .original-images.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.original-images.json b/.original-images.json index 3d1556f71..fa4d79a38 100644 --- a/.original-images.json +++ b/.original-images.json @@ -30,6 +30,7 @@ "ghcr.io/rackerlabs/genestack/glance:2024.1-ubuntu_jammy-1740121591", "ghcr.io/rackerlabs/genestack/gnocchi:2024.1-ubuntu_jammy-1738626728", "ghcr.io/rackerlabs/genestack/heat:2024.1-ubuntu_jammy-1738626724", + "ghcr.io/rackerlabs/genestack/magnum:2024.1-ubuntu_jammy-1742991496", "ghcr.io/rackerlabs/genestack/neutron-oslodb:2024.1-ubuntu_jammy-1738626982", "ghcr.io/rackerlabs/genestack/neutron-oslodb:2024.1-ubuntu_jammy-1739651767", "ghcr.io/rackerlabs/genestack/neutron-oslodb:2024.1-ubuntu_jammy-1742943886", From 6e1290630bb794c1a5e799ee35a41a58020880f2 Mon Sep 17 00:00:00 2001 From: phillip-toohill Date: Fri, 25 Jul 2025 11:10:22 -0500 Subject: [PATCH 38/97] fix: Adjusting monitoring docs kube events exporter --- docs/monitoring-info.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/monitoring-info.md b/docs/monitoring-info.md index f89796db0..9a8fa1787 100644 --- a/docs/monitoring-info.md +++ b/docs/monitoring-info.md @@ -68,6 +68,12 @@ View the [kube-state-docs](https://github.com/kubernetes/kube-state-metrics/tree Beyond those two highly important ones installed by default are many more equally important metric exporters that we install as part of Genestack's workflow that we'll go over next. +* ### Kubernetes Event Exporter +Kubernetes clusters are constantly sending events that contain potentially important data that should be captured. +With the [Kubernetes Event Exporter](https://github.com/resmoio/kubernetes-event-exporter) we can capture these events to gain a better view of what our cluster is doing. +This exporter also includes built in alerting mechanisms for things like Slack and MSTeams that can be configured to send messages when specific events are seen. +View the [Kubernetes Event Exporter Install Instructions](prometheus-kube-event-exporter.md) to get this exporter installed. + * ### MariaDB/MySQL Exporter: Genestack uses a couple different database solutions to run OpenStack or just for general storage capabilities, the most prominent of them is MySQL or more specifically within Genestack MariaDB and Galera. When installing [MariaDB](infrastructure-mariadb.md) as part of Genestack's workflow it is default to enable metrics which deploys its own service monitor as part of the [mariadb-operator](https://mariadb-operator.github.io/mariadb-operator/latest/) helm charts. @@ -170,12 +176,6 @@ Currently, in Genestack the textfile-collector is used to collect kernel-taint s This is currently the complete list of exporters and monitoring callouts deployed within the Genestack workflow. That said, Genestack is constantly evolving and list may grow or change entirely as we look to further improve our systems! With all these metrics available we need a way to visualize them to get a better picture of our systems and their health, we'll discuss that next! -* ### Kubernetes Event Exporter -Kubernetes clusters are constantly sending events that contain potentially important data that should be captured. -With the [Kubernetes Event Exporter](https://github.com/resmoio/kubernetes-event-exporter) we can capture these events to gain a better view of what our cluster is doing. -This exporter also includes built in alerting mechanisms for things like Slack and MSTeams that can be configured to send messages when specific events are seen. -View the [Kubernetes Event Exporter Install Instructions](prometheus-kube-event-exporter.md) to get this exporter installed. - ## Visualization In Genestack we deploy [Grafana](https://grafana.com/) as our default visualization tool. Grafana is open-sourced tooling which aligns well with the Genestack ethos while providing seamless visualization of the metrics generated by our systems. From bc0ade467f1763a2b09734b41aa311c9a88adbd6 Mon Sep 17 00:00:00 2001 From: Chanyeol Yoon <31232158+ycy1766@users.noreply.github.com> Date: Tue, 29 Jul 2025 03:16:42 +0900 Subject: [PATCH 39/97] fix(nova): Correct path for lifecycle overrides (#1093) Aligns override structure with the base chart by moving lifecycle settings to the `pod` section. This fixes an issue where overrides were not being applied. Signed-off-by: Chanyeol Yoon --- .../nova/nova-helm-overrides.yaml | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/base-helm-configs/nova/nova-helm-overrides.yaml b/base-helm-configs/nova/nova-helm-overrides.yaml index fb13c1642..d25a16c72 100644 --- a/base-helm-configs/nova/nova-helm-overrides.yaml +++ b/base-helm-configs/nova/nova-helm-overrides.yaml @@ -32,30 +32,6 @@ images: network: backend: - ovn - lifecycle: - upgrades: - deployments: - revision_history: 3 - pod_replacement_strategy: RollingUpdate - rolling_update: - max_unavailable: 20% - max_surge: 3 - daemonsets: - pod_replacement_strategy: RollingUpdate - compute: - enabled: true - min_ready_seconds: 0 - max_unavailable: 20% - disruption_budget: - metadata: - min_available: 0 - osapi: - min_available: 0 - termination_grace_period: - metadata: - timeout: 60 - osapi: - timeout: 60 ssh: enabled: true @@ -300,6 +276,30 @@ endpoints: # hyperconverged lab (/scripts/hyperconverged-lab.sh). # limit values based on defaults from the openstack-helm charts unless defined pod: + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 20% + max_surge: 3 + daemonsets: + pod_replacement_strategy: RollingUpdate + compute: + enabled: true + min_ready_seconds: 0 + max_unavailable: 20% + disruption_budget: + metadata: + min_available: 0 + osapi: + min_available: 0 + termination_grace_period: + metadata: + timeout: 60 + osapi: + timeout: 60 resources: enabled: true compute: From 46acf729218f3619f6ca64a39f842b3c9c77477b Mon Sep 17 00:00:00 2001 From: Gaurav-t Date: Tue, 29 Jul 2025 00:13:40 +0530 Subject: [PATCH 40/97] feat: Enables static vendordata (#1086) It enables static vendordata and adds related resources. --- .../nova/nova-helm-overrides.yaml | 16 +++++ base-kustomize/nova/base/kustomization.yaml | 1 + .../base/static-vendordata-configmap.yaml | 7 +++ docs/openstack-vendordata.md | 58 +++++++++++++++++++ mkdocs.yml | 1 + 5 files changed, 83 insertions(+) create mode 100644 base-kustomize/nova/base/static-vendordata-configmap.yaml create mode 100644 docs/openstack-vendordata.md diff --git a/base-helm-configs/nova/nova-helm-overrides.yaml b/base-helm-configs/nova/nova-helm-overrides.yaml index d25a16c72..c148babf3 100644 --- a/base-helm-configs/nova/nova-helm-overrides.yaml +++ b/base-helm-configs/nova/nova-helm-overrides.yaml @@ -100,6 +100,9 @@ conf: vif_plugging_timeout: 300 cross_az_attach: true network_allocate_retries: 3 + api: + vendordata_providers: ['StaticJSON'] + vendordata_jsonfile_path: /etc/nova/vendor_data.json api_database: connection_debug: 0 connection_recycle_time: 600 @@ -346,6 +349,19 @@ pod: readOnlyRootFilesystem: false use_fqdn: compute: false + mounts: + nova_api_metadata: + init_container: null + nova_api_metadata: + volumeMounts: + - name: metadata-api-static-vendordata + mountPath: /etc/nova/vendor_data.json + subPath: vendor_data.json + readOnly: true + volumes: + - name: metadata-api-static-vendordata + configMap: + name: static-vendor-data manifests: deployment_spiceproxy: false diff --git a/base-kustomize/nova/base/kustomization.yaml b/base-kustomize/nova/base/kustomization.yaml index 570c66098..9766f8f2c 100644 --- a/base-kustomize/nova/base/kustomization.yaml +++ b/base-kustomize/nova/base/kustomization.yaml @@ -11,3 +11,4 @@ resources: - hpa-nova-novncproxy.yaml - hpa-nova-scheduler.yaml - policies.yaml + - static-vendordata-configmap.yaml diff --git a/base-kustomize/nova/base/static-vendordata-configmap.yaml b/base-kustomize/nova/base/static-vendordata-configmap.yaml new file mode 100644 index 000000000..291f523ea --- /dev/null +++ b/base-kustomize/nova/base/static-vendordata-configmap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: static-vendor-data + namespace: openstack +data: + vendor_data.json: '{}' diff --git a/docs/openstack-vendordata.md b/docs/openstack-vendordata.md new file mode 100644 index 000000000..2da140ea4 --- /dev/null +++ b/docs/openstack-vendordata.md @@ -0,0 +1,58 @@ +# Openstack Vendordata + +To read more about Openstack Vendordata see [upstream docs](https://docs.openstack.org/nova/latest/admin/vendordata.html) + +## Overview + +It is a feature that provides way to pass vendor-specific data to the instances at boot-time. It can be accessed +with one of the following ways: + +* [Metadata Service](https://docs.openstack.org/nova/latest/admin/metadata-service.html) +* [Config Drives](https://docs.openstack.org/nova/latest/admin/config-drive.html) + +Also, Vendordata sources can be specified with two ways: + +* [StaticJSON](https://docs.openstack.org/nova/latest/admin/vendordata.html#staticjson) +* [DynamicJSON](https://docs.openstack.org/nova/latest/admin/vendordata.html#dynamicjson) + +*StaticJSON* collects data from a JSON file that exits locally and is suitable when data remains same for all instances. +On the other hand *DynamicJSON* can collect data from external REST service and works well when that data does change for +instances. + +## Vendordata in genestack + +Genestack use *Metadata Service* to access Vendordata. It has StaticJSON enabled in nova.conf as default provider: + +```yaml +api: + vendordata_providers: ['StaticJSON'] + vendordata_jsonfile_path: /etc/nova/vendor_data.json +``` + +You can override the default configmap `/opt/genestack/base-kustomize/nova/base/static-vendordata-configmap` to pass +static vendor-data against `vendor_data.json` key, which is mounted at `/etc/nova/vendor_data.json` in metadata service +resources. + +For DynamicJSON you need to enable it amongst providers and have to specify dynamic target URL(s) in nova.conf as follows: + +```yaml +api: + vendordata_providers: ['StaticJSON', 'DynamicJSON'] + vendordata_jsonfile_path: /etc/nova/vendor_data.json + vendordata_dynamic_targets: ['target/url1', 'target/url2'] +``` + +A POST request call will be made to these dynamic targets and you can expect the request body contains instance's context +e.g. *instance-id, image-id, hostname etc.* These targets should return a valid JSON in response. + +## Cloud-init and Vendordata + +Cloud-init instructions can be passed-in as string against key - `cloud-init` within Vendordata JSON as follows: + +```json +{ + "cloud-init": "#cloud-config\nruncmd:\n..." +} +``` + +See [cloud-init docs](https://cloudinit.readthedocs.io/en/latest/reference/datasources/openstack.html) for more details. diff --git a/mkdocs.yml b/mkdocs.yml index d3906f224..da500d83f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -315,6 +315,7 @@ nav: - Override Public Endpoint fqdn for service catalog: openstack-override-public-endpoint-fqdn.md - Service Overrides: openstack-service-overrides.md - Resource and Project Lookups: openstack-resource-lookups.md + - Vendordata: openstack-vendordata.md - Working locally With Docs: mkdocs-howto.md - Cloud Onboarding: - Cloud Onboarding Welcome: cloud-onboarding-welcome.md From d883a2a86996422959f51476609a337722d50bd0 Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Mon, 28 Jul 2025 13:44:49 -0500 Subject: [PATCH 41/97] fix: (docs) move vendordata to under compute --- mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index da500d83f..17c1f4f93 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -295,6 +295,7 @@ nav: - Nova PCI Passthrough: openstack-pci-passthrough.md - Host Aggregates: openstack-host-aggregates.md - Instance Data Recovery: openstack-data-disk-recovery.md + - Vendordata: openstack-vendordata.md - Quota Management: - Quota Management: openstack-quota-managment.md - Images: @@ -315,7 +316,6 @@ nav: - Override Public Endpoint fqdn for service catalog: openstack-override-public-endpoint-fqdn.md - Service Overrides: openstack-service-overrides.md - Resource and Project Lookups: openstack-resource-lookups.md - - Vendordata: openstack-vendordata.md - Working locally With Docs: mkdocs-howto.md - Cloud Onboarding: - Cloud Onboarding Welcome: cloud-onboarding-welcome.md From 6a4c56d96bd9f74299d13d7fc0eb75605ef7a7e6 Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Mon, 28 Jul 2025 16:45:35 -0500 Subject: [PATCH 42/97] Fix typo alertmanager route (#1100) * fix: Fix typo in Alertmanager route Renames 'custom-alertmanger-gateway-route' to 'custom-alertmanager-gateway-route'. Signed-off-by: Chanyeol Yoon * fix: (yamllint) correct indents --------- Signed-off-by: Chanyeol Yoon Co-authored-by: Chanyeol Yoon --- .../routes/custom-alertmanager-routes.yaml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/etc/gateway-api/routes/custom-alertmanager-routes.yaml b/etc/gateway-api/routes/custom-alertmanager-routes.yaml index 7c61de56f..caa3cd5ba 100644 --- a/etc/gateway-api/routes/custom-alertmanager-routes.yaml +++ b/etc/gateway-api/routes/custom-alertmanager-routes.yaml @@ -1,16 +1,17 @@ +--- apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: - name: custom-alertmanger-gateway-route + name: custom-alertmanager-gateway-route namespace: prometheus spec: parentRefs: - - name: flex-gateway - sectionName: http - namespace: nginx-gateway + - name: flex-gateway + sectionName: http + namespace: nginx-gateway hostnames: - - "alertmanager.your.domain.tld" + - "alertmanager.your.domain.tld" rules: - backendRefs: - - name: kube-prometheus-stack-alertmanager - port: 9093 + - name: kube-prometheus-stack-alertmanager + port: 9093 From f852b65ad128061bac855809fa62e819f768f2b9 Mon Sep 17 00:00:00 2001 From: Dan With Date: Tue, 29 Jul 2025 15:00:56 -0500 Subject: [PATCH 43/97] Fix: Ensure multipath disabled on block nodes. Only enable iscsid (#1078) This is a follow-up PR that further isolates host-setup multipath for compute nodes. It ensures multipath is disabled for block nodes and that no multipath.conf file is overwritten on block nodes. Additionally, documentation has been revised to reflect that custom_multipath only pertains to compute nodes in the inventory.yaml file. Any reference to using multipath for LVM should condition people to think about two interfaces on the storage node; hence, the inventory.yaml variable 'storage_network_multipath' being used in the block nodes section. Block nodes ONLY need iscsid running with tgtadm package installed. NO multipathd.service should be running/enabled on block nodes. A complimentary PR will be made to flex-host-baseline to ensure the correct variables are set in all inventory.yaml files. # Please enter the commit message for your changes. Lines starting --- .../host_setup/tasks/custom_multipath.yml | 32 +++++++++++++++++++ ansible/roles/host_setup/vars/debian.yml | 1 + ansible/roles/host_setup/vars/ubuntu.yml | 1 + docs/openstack-cinder-lvmisci.md | 6 ++-- ...nstack-cinder-volume-provisioning-specs.md | 21 ++++++++++++ scripts/hyperconverged-lab.sh | 2 +- 6 files changed, 59 insertions(+), 4 deletions(-) diff --git a/ansible/roles/host_setup/tasks/custom_multipath.yml b/ansible/roles/host_setup/tasks/custom_multipath.yml index 14c6aa96c..188225d3a 100644 --- a/ansible/roles/host_setup/tasks/custom_multipath.yml +++ b/ansible/roles/host_setup/tasks/custom_multipath.yml @@ -16,6 +16,7 @@ - name: Copy over multipath Round Robin configuration file when: - custom_multipath | default(false) | bool + - ('openstack_compute_nodes' in group_names) block: - name: Install Packages ansible.builtin.package: @@ -36,6 +37,8 @@ - name: Install open-iscsi and multipath on nova compute nodes when: - enable_iscsi | default(false) | bool + - custom_multipath | default(false) | bool + - ('openstack_compute_nodes' in group_names) block: - name: Install Packages ansible.builtin.package: @@ -61,3 +64,32 @@ notify: - Restart iscsid - Restart multipathd and multipath-tools service + +- name: Install open-iscsi on block nodes + when: + - enable_iscsi | default(false) | bool + - ('cinder_storage_nodes' in group_names) + block: + - name: Install Packages + ansible.builtin.package: + name: + - open-iscsi + state: "{{ iscsi_package_state | default('present') }}" + update_cache: true + - name: Determine initiator name + set_fact: + initiator_name: > + {% set _iqn = "iqn.2004-10.com." + ansible_distribution |lower() + ":" + ansible_hostname -%} + {% if ansible_iscsi_iqn is defined -%} + {% if (ansible_iscsi_iqn |length >= 15) -%} + {% set _iqn = ansible_iscsi_iqn -%} + {% endif -%} + {% endif -%} + {{ _iqn }} + - name: Set iscsi initiator name + ansible.builtin.lineinfile: + path: /etc/iscsi/initiatorname.iscsi + regexp: '^InitiatorName=.*|^GenerateName=.*' + line: "InitiatorName={{ initiator_name }}" + notify: + - Restart iscsid diff --git a/ansible/roles/host_setup/vars/debian.yml b/ansible/roles/host_setup/vars/debian.yml index 7cf625a0c..bb7a70fba 100644 --- a/ansible/roles/host_setup/vars/debian.yml +++ b/ansible/roles/host_setup/vars/debian.yml @@ -50,6 +50,7 @@ _host_distro_packages: - iptables - irqbalance - libkmod2 + - lsscsi - lvm2 - nfs-client - nvme-cli diff --git a/ansible/roles/host_setup/vars/ubuntu.yml b/ansible/roles/host_setup/vars/ubuntu.yml index 3e547cac9..9f19fb40a 100644 --- a/ansible/roles/host_setup/vars/ubuntu.yml +++ b/ansible/roles/host_setup/vars/ubuntu.yml @@ -49,6 +49,7 @@ _host_distro_packages: - iptables - irqbalance - libkmod2 + - lsscsi - lvm2 - nfs-client - nvme-cli diff --git a/docs/openstack-cinder-lvmisci.md b/docs/openstack-cinder-lvmisci.md index 864ffc865..946a687cd 100644 --- a/docs/openstack-cinder-lvmisci.md +++ b/docs/openstack-cinder-lvmisci.md @@ -79,7 +79,7 @@ Within the `inventory.yaml` file, ensure you have the following variables for yo openstack_compute_nodes: vars: enable_iscsi: true - storage_network_multipath: false # optional -- enable when running multipath + custom_multipath: false # optional -- enable when running multipath with custom multipath.conf storage_nodes: vars: enable_iscsi: true @@ -111,7 +111,7 @@ ansible-playbook -i inventory.yaml playbooks/deploy-cinder-volumes-reference.yam ``` console ansible-playbook -i /etc/genestack/inventory/inventory.yaml deploy-cinder-volumes-reference.yaml \ - -e "cinder_storage_network_interface=ansible_br_storage_a cinder_storage_network_interface_secondary=ansible_br_storage_b storage_network_multipath=true storage_network_multipath=true cinder_backend_name=lvmdriver-1" \ + -e "cinder_storage_network_interface=ansible_br_storage_a cinder_storage_network_interface_secondary=ansible_br_storage_b storage_network_multipath=true cinder_backend_name=lvmdriver-1" \ --user ubuntu \ --become 'cinder_storage_nodes' ``` @@ -277,7 +277,7 @@ storage: ## 7 Verify Multipath Operations -If multipath is enabled, you check the status of the multipath devices on the storage nodes. +If multipath is enabled on compute nodes, you can verify dual iscsi targets on the storage nodes. ``` bash tgtadm --mode target --op show diff --git a/docs/openstack-cinder-volume-provisioning-specs.md b/docs/openstack-cinder-volume-provisioning-specs.md index d19b148cf..5109be37e 100644 --- a/docs/openstack-cinder-volume-provisioning-specs.md +++ b/docs/openstack-cinder-volume-provisioning-specs.md @@ -10,3 +10,24 @@ These specifications are set in the volume type. The following commands constrai root@openstack-node-0:~# kubectl --namespace openstack exec -ti openstack-admin-client -- openstack volume type set --property provisioning:min_vol_size=10 6af6ade2-53ca-4260-8b79-1ba2f208c91d root@openstack-node-0:~# kubectl --namespace openstack exec -ti openstack-admin-client -- openstack volume type set --property provisioning:max_vol_size=2048 6af6ade2-53ca-4260-8b79-1ba2f208c91d ``` + +## Sample Provisioning Script + +``` shell +#!/bin/bash + +openstack volume type create --description 'Capacity with LUKS encryption' --encryption-provider luks --encryption-cipher aes-xts-plain64 --encryption-key-size 256 --encryption-control-location front-end --property volume_backend_name=LVM_iSCSI --property provisioning:max_vol_size='2048' --property provisioning:min_vol_size='100' Capacity +openstack volume type create --description 'Standard with LUKS encryption' --encryption-provider luks --encryption-cipher aes-xts-plain64 --encryption-key-size 256 --encryption-control-location front-end --property volume_backend_name=LVM_iSCSI --property provisioning:max_vol_size='2048' --property provisioning:min_vol_size='10' Standard +openstack volume type create --description 'Performance with LUKS encryption' --encryption-provider luks --encryption-cipher aes-xts-plain64 --encryption-key-size 256 --encryption-control-location front-end --property volume_backend_name=LVM_iSCSI --property provisioning:max_vol_size='2048' --property provisioning:min_vol_size='10' Performance + + +openstack volume qos create --property read_iops_sec_per_gb='1' --property write_iops_sec_per_gb='1' Capacity-Block +openstack volume qos create --property read_iops_sec_per_gb='5' --property write_iops_sec_per_gb='5' Standard-Block +openstack volume qos create --property read_iops_sec_per_gb='10' --property write_iops_sec_per_gb='10' Performance-Block + +openstack volume qos associate Capacity-Block Capacity +openstack volume qos associate Standard-Block Standard +openstack volume qos associate Performance-Block Performance + +openstack volume type set --private __DEFAULT__ +``` diff --git a/scripts/hyperconverged-lab.sh b/scripts/hyperconverged-lab.sh index fae950748..c18c76772 100755 --- a/scripts/hyperconverged-lab.sh +++ b/scripts/hyperconverged-lab.sh @@ -407,7 +407,7 @@ all: openstack_compute_nodes: vars: enable_iscsi: true - storage_network_multipath: false + custom_multipath: false hosts: ${LAB_NAME_PREFIX}-0.${GATEWAY_DOMAIN}: null ${LAB_NAME_PREFIX}-1.${GATEWAY_DOMAIN}: null From 1970875c052f0b77699fd88b3cfed779d2c8635f Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Wed, 30 Jul 2025 15:39:36 -0500 Subject: [PATCH 44/97] chore: port additional images into the new enterprise images (#1099) Signed-off-by: Kevin Carter --- .github/workflows/release-cinder-netapp.yml | 111 ---------------- .github/workflows/release-magnum-rxt.yml | 124 ------------------ .github/workflows/smoke-cinder-netapp.yml | 45 ------- .../barbican/barbican-helm-overrides.yaml | 4 +- .../ceilometer/ceilometer-helm-overrides.yaml | 6 +- .../cinder/cinder-helm-overrides.yaml | 18 +-- .../designate/designate-helm-overrides.yaml | 18 +-- .../glance/glance-helm-overrides.yaml | 6 +- .../gnocchi/gnocchi-helm-overrides.yaml | 2 +- .../heat/heat-helm-overrides.yaml | 6 +- .../horizon/horizon-helm-overrides.yaml | 3 +- .../ironic/ironic-helm-overrides.yaml | 18 +-- .../keystone/keystone-helm-overrides.yaml | 6 +- .../magnum/magnum-helm-overrides.yaml | 10 +- .../masakari/masakari-helm-overrides.yaml | 8 +- .../neutron/neutron-helm-overrides.yaml | 8 +- .../nova/nova-helm-overrides.yaml | 6 +- .../octavia/octavia-helm-overrides.yaml | 6 +- .../placement/placement-helm-overrides.yaml | 2 +- .../skyline/base/deployment-apiserver.yaml | 4 +- 20 files changed, 65 insertions(+), 346 deletions(-) delete mode 100644 .github/workflows/release-cinder-netapp.yml delete mode 100644 .github/workflows/release-magnum-rxt.yml delete mode 100644 .github/workflows/smoke-cinder-netapp.yml diff --git a/.github/workflows/release-cinder-netapp.yml b/.github/workflows/release-cinder-netapp.yml deleted file mode 100644 index abb3b491a..000000000 --- a/.github/workflows/release-cinder-netapp.yml +++ /dev/null @@ -1,111 +0,0 @@ -# -name: Create and publish a Cinder RXT compatible image - -# Configures this workflow to run every time a change is pushed to the branch called `release`. -on: - workflow_dispatch: - inputs: - imageTag: - description: 'Set tag for the image' - required: true - default: 'master-ubuntu_jammy' - type: choice - options: - - master-ubuntu_jammy - - 2023.1-ubuntu_jammy - - 2023.2-ubuntu_jammy - - 2024.1-ubuntu_jammy - -# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - build-and-push-image: - outputs: - MY_DATE: ${{ steps.mydate.outputs.MY_DATE }} - MY_CONTAINER: ${{ steps.mycontainer.outputs.MY_CONTAINER }} - runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - # ghcr only allows lowercase repository names - - name: lowercase repo name - run: | - echo "IMAGE_NAME=${GITHUB_REPOSITORY,,}" >>${GITHUB_ENV} - # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. - # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. - # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - - name: Dynamically set MY_DATE environment variable - run: echo "MY_DATE=$(date +%s)" >> $GITHUB_ENV - - name: Build and push Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: Containerfiles/Cinder-volume-netapp-Containerfile - push: true - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/cinder-volume-rxt:${{ github.event.inputs.imageTag }} - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/cinder-volume-rxt:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - VERSION=${{ github.event.inputs.imageTag }} - - name: Dynamically set MY_CONTAINER output option - id: mycontainer - run: echo "MY_CONTAINER=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/cinder:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }}" >> $GITHUB_OUTPUT - - name: Dynamically set MY_DATE output option - id: mydate - run: echo "MY_DATE=${{ env.MY_DATE }}" >> $GITHUB_OUTPUT - - change-original-images: - runs-on: ubuntu-latest - needs: [build-and-push-image] - permissions: - contents: write - pull-requests: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Dynamically update the original images file - run: jq '. + ["${{ needs.build-and-push-image.outputs.MY_CONTAINER }}"] | sort' .original-images.json | tee .original-images.json.new - - name: Rewrite original images file - run: mv .original-images.json.new .original-images.json - - name: Create Pull Request - id: cpr - uses: peter-evans/create-pull-request@v7 - with: - commit-message: Update original images with new container - committer: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> - author: ${{ github.actor }} <${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com> - signoff: false - branch: ${{ needs.build-and-push-image.outputs.MY_DATE }} - sign-commits: true - delete-branch: true - title: 'chore: Update original images' - body: | - Update container image - - Updated original image file with container ${{needs.build-and-push-image.outputs.MY_CONTAINER}} - change request Auto-generated - labels: | - container images - automated pr - draft: false diff --git a/.github/workflows/release-magnum-rxt.yml b/.github/workflows/release-magnum-rxt.yml deleted file mode 100644 index 36f58d782..000000000 --- a/.github/workflows/release-magnum-rxt.yml +++ /dev/null @@ -1,124 +0,0 @@ -# -name: Create and Publish a Magnum Image - -on: - push: - paths: - - .github/workflows/release-magnum-rxt.yml - - Containerfiles/MagnumRXT-Containerfile - branches: - - development - - main - workflow_dispatch: - inputs: - imageTag: - description: Set tag for the image - required: true - default: 2024.1-ubuntu_jammy - type: choice - options: - - master - - 2024.1-ubuntu_jammy - pluginTag: - description: 'Set release used for the build environment' - required: true - default: master - type: choice - options: - - master - -# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - DEF_TAG_NAME: 2024.1-ubuntu_jammy - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - build-and-push-image: - outputs: - MY_DATE: ${{ steps.mydate.outputs.MY_DATE }} - MY_CONTAINER: ${{ steps.mycontainer.outputs.MY_CONTAINER }} - runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - # ghcr only allows lowercase repository names - - name: lowercase repo name - run: | - echo "IMAGE_NAME=${GITHUB_REPOSITORY,,}" >>${GITHUB_ENV} - # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. - # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. - # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - - name: Dynamically set MY_DATE environment variable - run: echo "MY_DATE=$(date +%s)" >> $GITHUB_ENV - - name: Build and push Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: Containerfiles/MagnumRXT-Containerfile - push: true - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/magnum:${{ github.event.inputs.imageTag || env.DEF_TAG_NAME }} - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/magnum:${{ github.event.inputs.imageTag || env.DEF_TAG_NAME }}-${{ env.MY_DATE }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - VERSION=${{ github.event.inputs.imageTag || env.DEF_TAG_NAME }} - PLUGIN_VERSION=${{ github.event.inputs.pluginTag }} - - name: Dynamically set MY_CONTAINER output option - id: mycontainer - run: echo "MY_CONTAINER=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/magnum:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }}" >> $GITHUB_OUTPUT - - name: Dynamically set MY_DATE output option - id: mydate - run: echo "MY_DATE=${{ env.MY_DATE }}" >> $GITHUB_OUTPUT - - change-original-images: - runs-on: ubuntu-latest - needs: [build-and-push-image] - permissions: - contents: write - pull-requests: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Dynamically update the original images file - run: jq '. + ["${{ needs.build-and-push-image.outputs.MY_CONTAINER }}"] | sort' .original-images.json | tee .original-images.json.new - - name: Rewrite original images file - run: mv .original-images.json.new .original-images.json - - name: Create Pull Request - id: cpr - uses: peter-evans/create-pull-request@v7 - with: - commit-message: Update original images with new container - committer: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> - author: ${{ github.actor }} <${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com> - signoff: false - branch: ${{ needs.build-and-push-image.outputs.MY_DATE }} - sign-commits: true - delete-branch: true - title: 'chore: Update original images' - body: | - Update container image - - Updated original image file with container ${{needs.build-and-push-image.outputs.MY_CONTAINER}} - change request Auto-generated - labels: | - container images - automated pr - draft: false diff --git a/.github/workflows/smoke-cinder-netapp.yml b/.github/workflows/smoke-cinder-netapp.yml deleted file mode 100644 index 36dc23b92..000000000 --- a/.github/workflows/smoke-cinder-netapp.yml +++ /dev/null @@ -1,45 +0,0 @@ -# -name: Run build check for the Cinder Volume Netapp RXT compatible image - -on: - pull_request: - paths: - - Containerfiles/Cinder-volume-netapp-Containerfile - -# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - build-and-push-image: - runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: read - steps: - - name: Checkout repository - uses: actions/checkout@v4 - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - # ghcr only allows lowercase repository names - - name: lowercase repo name - run: | - echo "IMAGE_NAME=${GITHUB_REPOSITORY,,}" >>${GITHUB_ENV} - - name: Build Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: Containerfiles/Cinder-volume-netapp-Containerfile - push: false - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/keystone-rxt:master-ubuntu_jammy - build-args: | - VERSION=master-ubuntu_jammy diff --git a/base-helm-configs/barbican/barbican-helm-overrides.yaml b/base-helm-configs/barbican/barbican-helm-overrides.yaml index 4cf368007..5b7346e29 100644 --- a/base-helm-configs/barbican/barbican-helm-overrides.yaml +++ b/base-helm-configs/barbican/barbican-helm-overrides.yaml @@ -7,11 +7,11 @@ images: db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" - image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" + image_repo_sync: null ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" - rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" + rabbit_init: null scripted_test: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" # NOTE: (brew) CPU requests values based on a three node diff --git a/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml b/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml index cd9cda971..6eb0ef00e 100644 --- a/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml +++ b/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml @@ -1,9 +1,9 @@ --- images: tags: - test: "quay.io/rackspace/rackerlabs-xrally-openstack:2.0.0" + test: null ceilometer_db_sync: "quay.io/rackspace/rackerlabs-ceilometer:2024.1-ubuntu_jammy" - rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" + rabbit_init: null ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ceilometer_central: "quay.io/rackspace/rackerlabs-ceilometer:2024.1-ubuntu_jammy" @@ -11,7 +11,7 @@ images: ceilometer_ipmi: "quay.io/rackspace/rackerlabs-ceilometer:2024.1-ubuntu_jammy" ceilometer_notification: "quay.io/rackspace/rackerlabs-ceilometer:2024.1-ubuntu_jammy" dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" - image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" + image_repo_sync: null pull_policy: "Always" conf: diff --git a/base-helm-configs/cinder/cinder-helm-overrides.yaml b/base-helm-configs/cinder/cinder-helm-overrides.yaml index 4c56603f7..b1d3c70a7 100644 --- a/base-helm-configs/cinder/cinder-helm-overrides.yaml +++ b/base-helm-configs/cinder/cinder-helm-overrides.yaml @@ -8,23 +8,23 @@ labels: images: tags: bootstrap: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" - cinder_api: "quay.io/rackspace/rackerlabs-cinder:2024.1-ubuntu_jammy" - cinder_backup: "quay.io/rackspace/rackerlabs-cinder:2024.1-ubuntu_jammy" + cinder_api: "ghcr.io/rackerlabs/genestack-images/cinder:2024.1-latest" + cinder_backup: "ghcr.io/rackerlabs/genestack-images/cinder:2024.1-latest" cinder_backup_storage_init: "quay.io/rackspace/rackerlabs-ceph-config-helper:latest-ubuntu_jammy" - cinder_db_sync: "quay.io/rackspace/rackerlabs-cinder:2024.1-ubuntu_jammy" - cinder_scheduler: "quay.io/rackspace/rackerlabs-cinder:2024.1-ubuntu_jammy" + cinder_db_sync: "ghcr.io/rackerlabs/genestack-images/cinder:2024.1-latest" + cinder_scheduler: "ghcr.io/rackerlabs/genestack-images/cinder:2024.1-latest" cinder_storage_init: "quay.io/rackspace/rackerlabs-ceph-config-helper:latest-ubuntu_jammy" - cinder_volume: "quay.io/rackspace/rackerlabs-cinder:2024.1-ubuntu_jammy" - cinder_volume_usage_audit: "quay.io/rackspace/rackerlabs-cinder:2024.1-ubuntu_jammy" + cinder_volume: "ghcr.io/rackerlabs/genestack-images/cinder:2024.1-latest" + cinder_volume_usage_audit: "ghcr.io/rackerlabs/genestack-images/cinder:2024.1-latest" db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" - image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" + image_repo_sync: null ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" - rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" - test: "quay.io/rackspace/rackerlabs-xrally-openstack:2.0.0" + rabbit_init: null + test: null # NOTE: (brew) requests cpu/mem values based on a three node # hyperconverged lab (/scripts/hyperconverged-lab.sh). diff --git a/base-helm-configs/designate/designate-helm-overrides.yaml b/base-helm-configs/designate/designate-helm-overrides.yaml index 341e04787..8ca01d795 100644 --- a/base-helm-configs/designate/designate-helm-overrides.yaml +++ b/base-helm-configs/designate/designate-helm-overrides.yaml @@ -46,19 +46,19 @@ images: bootstrap: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" - rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" + rabbit_init: null ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" - designate_db_sync: "quay.io/rackspace/rackerlabs-designate:2024.1-ubuntu_jammy" - designate_api: "quay.io/rackspace/rackerlabs-designate:2024.1-ubuntu_jammy" - designate_central: "quay.io/rackspace/rackerlabs-designate:2024.1-ubuntu_jammy" - designate_mdns: "quay.io/rackspace/rackerlabs-designate:2024.1-ubuntu_jammy" - designate_worker: "quay.io/rackspace/rackerlabs-designate:2024.1-ubuntu_jammy" - designate_producer: "quay.io/rackspace/rackerlabs-designate:2024.1-ubuntu_jammy" - designate_sink: "quay.io/rackspace/rackerlabs-designate:2024.1-ubuntu_jammy" - image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" + designate_db_sync: "ghcr.io/rackerlabs/genestack-images/designate:2024.1-latest" + designate_api: "ghcr.io/rackerlabs/genestack-images/designate:2024.1-latest" + designate_central: "ghcr.io/rackerlabs/genestack-images/designate:2024.1-latest" + designate_mdns: "ghcr.io/rackerlabs/genestack-images/designate:2024.1-latest" + designate_worker: "ghcr.io/rackerlabs/genestack-images/designate:2024.1-latest" + designate_producer: "ghcr.io/rackerlabs/genestack-images/designate:2024.1-latest" + designate_sink: "ghcr.io/rackerlabs/genestack-images/designate:2024.1-latest" + image_repo_sync: null pull_policy: "IfNotPresent" local_registry: active: false diff --git a/base-helm-configs/glance/glance-helm-overrides.yaml b/base-helm-configs/glance/glance-helm-overrides.yaml index a6791b836..43314bbc7 100644 --- a/base-helm-configs/glance/glance-helm-overrides.yaml +++ b/base-helm-configs/glance/glance-helm-overrides.yaml @@ -4,7 +4,7 @@ storage: pvc images: tags: - test: "quay.io/rackspace/rackerlabs-xrally-openstack:2.0.0" + test: null glance_storage_init: "quay.io/rackspace/rackerlabs-ceph-config-helper:latest-ubuntu_jammy" glance_metadefs_load: "ghcr.io/rackerlabs/genestack-images/glance:2024.1-latest" db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" @@ -13,12 +13,12 @@ images: ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" - rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" + rabbit_init: null glance_api: "ghcr.io/rackerlabs/genestack-images/glance:2024.1-latest" # Bootstrap image requires curl bootstrap: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" - image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" + image_repo_sync: null bootstrap: enabled: true diff --git a/base-helm-configs/gnocchi/gnocchi-helm-overrides.yaml b/base-helm-configs/gnocchi/gnocchi-helm-overrides.yaml index a1e1bf343..1b413091e 100644 --- a/base-helm-configs/gnocchi/gnocchi-helm-overrides.yaml +++ b/base-helm-configs/gnocchi/gnocchi-helm-overrides.yaml @@ -10,7 +10,7 @@ images: gnocchi_resources_cleaner: "quay.io/rackspace/rackerlabs-gnocchi:2024.1-ubuntu_jammy" gnocchi_statsd: "quay.io/rackspace/rackerlabs-gnocchi:2024.1-ubuntu_jammy" gnocchi_storage_init: "quay.io/rackspace/rackerlabs-ceph-config-helper:latest-ubuntu_jammy" - image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" + image_repo_sync: null ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" diff --git a/base-helm-configs/heat/heat-helm-overrides.yaml b/base-helm-configs/heat/heat-helm-overrides.yaml index 47e4b0688..0fd78df91 100644 --- a/base-helm-configs/heat/heat-helm-overrides.yaml +++ b/base-helm-configs/heat/heat-helm-overrides.yaml @@ -12,12 +12,12 @@ images: heat_engine: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" heat_engine_cleaner: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" heat_purge_deleted: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" - image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" + image_repo_sync: null ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" - rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" - test: "quay.io/rackspace/rackerlabs-xrally-openstack:2.0.0" + rabbit_init: null + test: null # NOTE: (brew) requests cpu/mem values based on a three node # hyperconverged lab (/scripts/hyperconverged-lab.sh). diff --git a/base-helm-configs/horizon/horizon-helm-overrides.yaml b/base-helm-configs/horizon/horizon-helm-overrides.yaml index 93ab4843d..0e133cdd5 100644 --- a/base-helm-configs/horizon/horizon-helm-overrides.yaml +++ b/base-helm-configs/horizon/horizon-helm-overrides.yaml @@ -5,9 +5,8 @@ images: db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" horizon_db_sync: "quay.io/rackspace/rackerlabs-horizon:2024.1-ubuntu_jammy" horizon: "quay.io/rackspace/rackerlabs-horizon:2024.1-ubuntu_jammy" - test: "quay.io/rackspace/rackerlabs-osh-selenium:latest-ubuntu_jammy" dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" - image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" + image_repo_sync: null # NOTE: (brew) requests cpu/mem values based on a three node # hyperconverged lab (/scripts/hyperconverged-lab.sh). diff --git a/base-helm-configs/ironic/ironic-helm-overrides.yaml b/base-helm-configs/ironic/ironic-helm-overrides.yaml index 3fcb4fa60..0b6e181b5 100644 --- a/base-helm-configs/ironic/ironic-helm-overrides.yaml +++ b/base-helm-configs/ironic/ironic-helm-overrides.yaml @@ -12,20 +12,20 @@ images: bootstrap: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" - ironic_db_sync: "quay.io/rackspace/rackerlabs-ironic:2024.1-ubuntu_jammy" + ironic_db_sync: "ghcr.io/rackerlabs/genestack-images/ironic-api:2024.1-latest" ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" - rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" - ironic_api: "quay.io/rackspace/rackerlabs-ironic:2024.1-ubuntu_jammy" - ironic_conductor: "quay.io/rackspace/rackerlabs-ironic:2024.1-ubuntu_jammy" - ironic_pxe: "quay.io/rackspace/rackerlabs-ironic:2024.1-ubuntu_jammy" - ironic_pxe_init: "quay.io/rackspace/rackerlabs-ironic:2024.1-ubuntu_jammy" + rabbit_init: null + ironic_api: "ghcr.io/rackerlabs/genestack-images/ironic-api:2024.1-latest" + ironic_conductor: "ghcr.io/rackerlabs/genestack-images/ironic-api:2024.1-latest" + ironic_pxe: "ghcr.io/rackerlabs/genestack-images/ironic-api:2024.1-latest" + ironic_pxe_init: "ghcr.io/rackerlabs/genestack-images/ironic-api:2024.1-latest" ironic_pxe_http: "docker.io/nginx:1.13.3" # Retained from openstack-helm default - ironic_inspector: "quay.io/rackspace/rackerlabs-ironic-inspector:2024.1-ubuntu_jammy" - ironic_inspector_db_sync: "quay.io/rackspace/rackerlabs-ironic-inspector:2024.1-ubuntu_jammy" + ironic_inspector: "ghcr.io/rackerlabs/genestack-images/ironic-inspector:2024.1-latest" + ironic_inspector_db_sync: "ghcr.io/rackerlabs/genestack-images/ironic-inspector:2024.1-latest" dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" - image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" + image_repo_sync: null pull_policy: "IfNotPresent" conf: diff --git a/base-helm-configs/keystone/keystone-helm-overrides.yaml b/base-helm-configs/keystone/keystone-helm-overrides.yaml index 28e3537e9..3215f7d75 100644 --- a/base-helm-configs/keystone/keystone-helm-overrides.yaml +++ b/base-helm-configs/keystone/keystone-helm-overrides.yaml @@ -5,7 +5,7 @@ images: db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" - image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" + image_repo_sync: null keystone_api: "ghcr.io/rackerlabs/genestack-images/keystone:2024.1-latest" keystone_credential_cleanup: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" keystone_credential_rotate: "ghcr.io/rackerlabs/genestack-images/keystone:2024.1-latest" @@ -15,8 +15,8 @@ images: keystone_fernet_rotate: "ghcr.io/rackerlabs/genestack-images/keystone:2024.1-latest" keystone_fernet_setup: "ghcr.io/rackerlabs/genestack-images/keystone:2024.1-latest" ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" - rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" - test: "quay.io/rackspace/rackerlabs-xrally-openstack:2.0.0" + rabbit_init: null + test: null # NOTE: (brew) requests cpu/mem values based on a three node # hyperconverged lab (/scripts/hyperconverged-lab.sh). diff --git a/base-helm-configs/magnum/magnum-helm-overrides.yaml b/base-helm-configs/magnum/magnum-helm-overrides.yaml index 2f22d89e7..d2cc00d26 100644 --- a/base-helm-configs/magnum/magnum-helm-overrides.yaml +++ b/base-helm-configs/magnum/magnum-helm-overrides.yaml @@ -5,14 +5,14 @@ images: db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" - image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" + image_repo_sync: null ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" - magnum_api: "quay.io/rackspace/rackerlabs-magnum:2024.1-ubuntu_jammy" - magnum_conductor: "quay.io/rackspace/rackerlabs-magnum:2024.1-ubuntu_jammy" - magnum_db_sync: "quay.io/rackspace/rackerlabs-magnum:2024.1-ubuntu_jammy" - rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" + magnum_api: "ghcr.io/rackerlabs/genestack-images/magnum:2024.1-latest" + magnum_conductor: "ghcr.io/rackerlabs/genestack-images/magnum:2024.1-latest" + magnum_db_sync: "ghcr.io/rackerlabs/genestack-images/magnum:2024.1-latest" + rabbit_init: null # NOTE: (brew) requests cpu/mem values based on a three node # hyperconverged lab (/scripts/hyperconverged-lab.sh). diff --git a/base-helm-configs/masakari/masakari-helm-overrides.yaml b/base-helm-configs/masakari/masakari-helm-overrides.yaml index 4c4323058..78315026f 100644 --- a/base-helm-configs/masakari/masakari-helm-overrides.yaml +++ b/base-helm-configs/masakari/masakari-helm-overrides.yaml @@ -19,12 +19,12 @@ images: ks_endpoints: ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest ks_service: ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest ks_user: ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest - masakari_api: quay.io/rackspace/rackerlabs-masakari:2024.1-ubuntu_jammy - masakari_engine: quay.io/rackspace/rackerlabs-masakari:2024.1-ubuntu_jammy + masakari_api: ghcr.io/rackerlabs/genestack-images/masakari:2024.1-latest + masakari_engine: ghcr.io/rackerlabs/genestack-images/masakari:2024.1-latest # TEMP HOST-MONITOR IMAGE TO FIX: https://review.opendev.org/c/openstack/masakari-monitors/+/951336 masakari_host_monitor: kernelpanic53/rackerlabs-masakari-monitors:zhmarvi-ubuntu_jammy_v1.0 - masakari_process_monitor: quay.io/rackspace/rackerlabs-masakari-monitors:2024.1-ubuntu_jammy - masakari_instance_monitor: quay.io/rackspace/rackerlabs-masakari-monitors:2024.1-ubuntu_jammy + masakari_process_monitor: ghcr.io/rackerlabs/genestack-images/masakari:2024.1-latest + masakari_instance_monitor: ghcr.io/rackerlabs/genestack-images/masakari:2024.1-latest rabbit_init: docker.io/rabbitmq:3.13-management dep_check: ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest pull_policy: "IfNotPresent" diff --git a/base-helm-configs/neutron/neutron-helm-overrides.yaml b/base-helm-configs/neutron/neutron-helm-overrides.yaml index 7d3b714ff..c7f082bfe 100644 --- a/base-helm-configs/neutron/neutron-helm-overrides.yaml +++ b/base-helm-configs/neutron/neutron-helm-overrides.yaml @@ -20,16 +20,16 @@ images: neutron_rpc_server: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" neutron_bagpipe_bgp: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" neutron_netns_cleanup_cron: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" - test: "quay.io/rackspace/rackerlabs-xrally-openstack:2.0.0" + test: null purge_test: "quay.io/rackspace/rackerlabs-ospurge:latest" - rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" - netoffload: "quay.io/rackspace/rackerlabs-netoffload:v1.0.1" + rabbit_init: null + netoffload: null neutron_sriov_agent: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" neutron_sriov_agent_init: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" neutron_bgp_dragent: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" neutron_ironic_agent: "ghcr.io/rackerlabs/genestack-images/neutron:2024.1-latest" dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" - image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" + image_repo_sync: null labels: ovs: diff --git a/base-helm-configs/nova/nova-helm-overrides.yaml b/base-helm-configs/nova/nova-helm-overrides.yaml index c148babf3..846cf599d 100644 --- a/base-helm-configs/nova/nova-helm-overrides.yaml +++ b/base-helm-configs/nova/nova-helm-overrides.yaml @@ -5,7 +5,7 @@ images: db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" - image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" + image_repo_sync: null ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" @@ -26,8 +26,8 @@ images: nova_spiceproxy_assets: "ghcr.io/rackerlabs/genestack-images/nova:2024.1-latest" nova_storage_init: "quay.io/rackspace/rackerlabs-ceph-config-helper:latest-ubuntu_jammy" nova_wait_for_computes_init: "quay.io/rackspace/rackerlabs-hyperkube-amd64:v1.11.6" - rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" - test: "quay.io/rackspace/rackerlabs-xrally-openstack:2.0.0" + rabbit_init: null + test: null network: backend: diff --git a/base-helm-configs/octavia/octavia-helm-overrides.yaml b/base-helm-configs/octavia/octavia-helm-overrides.yaml index efd338a00..d9e658d25 100644 --- a/base-helm-configs/octavia/octavia-helm-overrides.yaml +++ b/base-helm-configs/octavia/octavia-helm-overrides.yaml @@ -5,7 +5,7 @@ images: db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" - image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" + image_repo_sync: null ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" @@ -16,8 +16,8 @@ images: octavia_housekeeping: "ghcr.io/rackerlabs/genestack-images/octavia:2024.1-latest" octavia_worker: "ghcr.io/rackerlabs/genestack-images/octavia:2024.1-latest" openvswitch_vswitchd: "docker.io/kolla/centos-source-openvswitch-vswitchd:rocky" - rabbit_init: "quay.io/rackspace/rackerlabs-rabbitmq:3.13-management" - test: "quay.io/rackspace/rackerlabs-xrally-openstack:2.0.0" + rabbit_init: null + test: null dependencies: static: diff --git a/base-helm-configs/placement/placement-helm-overrides.yaml b/base-helm-configs/placement/placement-helm-overrides.yaml index 541cf81dc..1a8aff451 100644 --- a/base-helm-configs/placement/placement-helm-overrides.yaml +++ b/base-helm-configs/placement/placement-helm-overrides.yaml @@ -4,7 +4,7 @@ images: db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" - image_repo_sync: "quay.io/rackspace/rackerlabs-docker:17.07.0" + image_repo_sync: null ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_service: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" ks_user: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" diff --git a/base-kustomize/skyline/base/deployment-apiserver.yaml b/base-kustomize/skyline/base/deployment-apiserver.yaml index 32784aef9..dedfb32cc 100644 --- a/base-kustomize/skyline/base/deployment-apiserver.yaml +++ b/base-kustomize/skyline/base/deployment-apiserver.yaml @@ -335,7 +335,7 @@ spec: key: prometheus_endpoint optional: true - name: skyline-apiserver-db-migrate - image: "quay.io/rackspace/rackerlabs-skyline-rxt:master-ubuntu_jammy-1748595671" + image: "ghcr.io/rackerlabs/genestack-images/skyline:2024.2-latest" imagePullPolicy: IfNotPresent resources: requests: @@ -358,7 +358,7 @@ spec: readOnly: true containers: - name: skyline-apiserver - image: "quay.io/rackspace/rackerlabs-skyline-rxt:master-ubuntu_jammy-1748595671" + image: "ghcr.io/rackerlabs/genestack-images/skyline:2024.2-latest" imagePullPolicy: IfNotPresent resources: limits: From 2d19b2d1a9ab64df4a18c40beb0513828c348441 Mon Sep 17 00:00:00 2001 From: Gerald Williams Date: Thu, 31 Jul 2025 09:42:07 -0500 Subject: [PATCH 45/97] fix: Removed "display_name" from network attributes. (#1105) The schema for the network payload does not include the "diplay_name" attribute, so it is not necessary to include it in the ceilometer configuration. Havng it here will cause an issue for ceilometer sending events to gnocchi since it won't be in the gnocchi schema. --- base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml b/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml index 6eb0ef00e..c74741aec 100644 --- a/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml +++ b/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml @@ -1066,7 +1066,6 @@ conf: project_id: project_id floating_ip_address: ip_address event_type: event_type - display_name: name - resource_type: stack metrics: From 4bbb49913576c9bb3ccc950b4e081cfcb119831a Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Thu, 31 Jul 2025 11:12:58 -0500 Subject: [PATCH 46/97] feat: add new ent ready images (#1106) Signed-off-by: Kevin Carter --- .original-images.json | 1 - base-helm-configs/horizon/horizon-helm-overrides.yaml | 4 ++-- base-kustomize/keystone/federation/kustomization.yaml | 4 ++-- docs/openstack-keystone-federation.md | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.original-images.json b/.original-images.json index fa4d79a38..f549d5c71 100644 --- a/.original-images.json +++ b/.original-images.json @@ -40,7 +40,6 @@ "ghcr.io/rackerlabs/genestack/nova-efi:2024.1-ubuntu_jammy-1737928811", "ghcr.io/rackerlabs/genestack/nova-efi:2025.1-ubuntu_jammy-1750943616", "ghcr.io/rackerlabs/genestack/octavia-ovn:2024.1-ubuntu_jammy-1737651745", - "ghcr.io/rackerlabs/keystone-rxt/shibd:1747958286", "ghcr.io/rackerlabs/keystone-rxt:2024.1-ubuntu_jammy-1747958291", "ghcr.io/rackerlabs/skyline-rxt:master-ubuntu_jammy-1748595671", "ghcr.io/vexxhost/netoffload:v1.0.1", diff --git a/base-helm-configs/horizon/horizon-helm-overrides.yaml b/base-helm-configs/horizon/horizon-helm-overrides.yaml index 0e133cdd5..f27c28f78 100644 --- a/base-helm-configs/horizon/horizon-helm-overrides.yaml +++ b/base-helm-configs/horizon/horizon-helm-overrides.yaml @@ -3,8 +3,8 @@ images: tags: db_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" db_drop: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" - horizon_db_sync: "quay.io/rackspace/rackerlabs-horizon:2024.1-ubuntu_jammy" - horizon: "quay.io/rackspace/rackerlabs-horizon:2024.1-ubuntu_jammy" + horizon_db_sync: "ghcr.io/rackerlabs/genestack-images/horizon:2024.1-latest" + horizon: "ghcr.io/rackerlabs/genestack-images/horizon:2024.1-latest" dep_check: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" image_repo_sync: null diff --git a/base-kustomize/keystone/federation/kustomization.yaml b/base-kustomize/keystone/federation/kustomization.yaml index 34a394d6f..630078f64 100644 --- a/base-kustomize/keystone/federation/kustomization.yaml +++ b/base-kustomize/keystone/federation/kustomization.yaml @@ -6,8 +6,8 @@ resources: images: - name: keystone-shib - newName: ghcr.io/rackerlabs/keystone-rxt/shibd - newTag: "1747958286" + newName: ghcr.io/rackerlabs/genestack-images/shibd + newTag: "latest" patches: - target: diff --git a/docs/openstack-keystone-federation.md b/docs/openstack-keystone-federation.md index dd14fdb60..b1811f429 100644 --- a/docs/openstack-keystone-federation.md +++ b/docs/openstack-keystone-federation.md @@ -101,7 +101,7 @@ Using the `docker` command and the shibd image, retrieve the SAML2 files from th ``` shell docker run -v /etc/genestack/keystone-sp:/mnt \ - ghcr.io/rackerlabs/keystone-rxt/shibd:latest \ + ghcr.io/rackerlabs/genestack-images/shibd:latest \ cp -R /etc/shibboleth /mnt/ ``` From bb752e6ade2ce518cd67df4777ef7c9d5a209456 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Fri, 1 Aug 2025 11:25:39 -0500 Subject: [PATCH 47/97] feat: rev kube-ovn to 1.13.14 (#1107) Signed-off-by: Kevin Carter --- base-helm-configs/kube-ovn/kube-ovn-helm-overrides.yaml | 2 +- bin/install-kube-ovn.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base-helm-configs/kube-ovn/kube-ovn-helm-overrides.yaml b/base-helm-configs/kube-ovn/kube-ovn-helm-overrides.yaml index 6406adb72..9dbca37e5 100644 --- a/base-helm-configs/kube-ovn/kube-ovn-helm-overrides.yaml +++ b/base-helm-configs/kube-ovn/kube-ovn-helm-overrides.yaml @@ -9,7 +9,7 @@ global: kubeovn: repository: kube-ovn vpcRepository: vpc-nat-gateway - tag: v1.13.13 + tag: v1.13.14 support_arm: true thirdparty: true diff --git a/bin/install-kube-ovn.sh b/bin/install-kube-ovn.sh index 7590ef284..3a29573fd 100755 --- a/bin/install-kube-ovn.sh +++ b/bin/install-kube-ovn.sh @@ -4,7 +4,7 @@ GLOBAL_OVERRIDES_DIR="/etc/genestack/helm-configs/global_overrides" SERVICE_CONFIG_DIR="/etc/genestack/helm-configs/kube-ovn" BASE_OVERRIDES="/opt/genestack/base-helm-configs/kube-ovn/kube-ovn-helm-overrides.yaml" -KUBE_OVN_VERSION="v1.13.13" +KUBE_OVN_VERSION="v1.13.14" MASTER_NODES=$(kubectl get nodes -l kube-ovn/role=master -o json | jq -r '[.items[].status.addresses[] | select(.type == "InternalIP") | .address] | join(",")' | sed 's/,/\\,/g') MASTER_NODE_COUNT=$(kubectl get nodes -l kube-ovn/role=master -o json | jq -r '.items[].status.addresses[] | select(.type=="InternalIP") | .address' | wc -l) From 7f814b3e8f682ec0578119e2421da1d1a7b23605 Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Fri, 1 Aug 2025 11:58:13 -0500 Subject: [PATCH 48/97] fix: add static vendordata mount point to nova-compute pods (#1110) --- base-helm-configs/nova/nova-helm-overrides.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/base-helm-configs/nova/nova-helm-overrides.yaml b/base-helm-configs/nova/nova-helm-overrides.yaml index 846cf599d..24c84c968 100644 --- a/base-helm-configs/nova/nova-helm-overrides.yaml +++ b/base-helm-configs/nova/nova-helm-overrides.yaml @@ -350,6 +350,18 @@ pod: use_fqdn: compute: false mounts: + nova_compute: + init_container: null + nova_compute: + volumeMounts: + - name: metadata-api-static-vendordata + mountPath: /etc/nova/vendor_data.json + subPath: vendor_data.json + readOnly: true + volumes: + - name: metadata-api-static-vendordata + configMap: + name: static-vendor-data nova_api_metadata: init_container: null nova_api_metadata: From 765524218e94ed502e288303c5c66d6c36ec7abc Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Mon, 4 Aug 2025 08:32:33 -0500 Subject: [PATCH 49/97] Update kube-ovn configuragtions (#1113) * fix: Disable EIP Disable EIP to quiet logs and improve general performance. > I0803 09:55:50.961102 7 vpc.go:72] enqueue update vpc neutron-ee30d873-5b4d-453b-8412-db5098362c01 > I0803 09:55:50.979070 7 vpc.go:256] handle add/update vpc neutron-ee30d873-5b4d-453b-8412-db5098362c01 > W0803 09:55:50.979244 7 vpc.go:337] enable-eip-snat need external subnet external to be exist: subnet.kubeovn.io "external" not found This error is found in v1.14.x but present in all version before. Because this error is attempting to run updates on networks, and we do not configure a specific "external" network we're disabling the EIP function. Signed-off-by: Kevin Carter * chore: use ent kube-ovn image Signed-off-by: Kevin Carter --------- Signed-off-by: Kevin Carter --- base-helm-configs/kube-ovn/kube-ovn-helm-overrides.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base-helm-configs/kube-ovn/kube-ovn-helm-overrides.yaml b/base-helm-configs/kube-ovn/kube-ovn-helm-overrides.yaml index 9dbca37e5..3296fea38 100644 --- a/base-helm-configs/kube-ovn/kube-ovn-helm-overrides.yaml +++ b/base-helm-configs/kube-ovn/kube-ovn-helm-overrides.yaml @@ -3,13 +3,13 @@ # Declare variables to be passed into your templates. global: registry: - address: docker.io/kubeovn + address: ghcr.io/rackerlabs/genestack-images imagePullSecrets: [] images: kubeovn: repository: kube-ovn vpcRepository: vpc-nat-gateway - tag: v1.13.14 + tag: v1.13.14-latest support_arm: true thirdparty: true @@ -41,7 +41,7 @@ networking: #VLAN_NAME: "ovn-vlan" #VLAN_ID: "100" EXCHANGE_LINK_NAME: false - ENABLE_EIP_SNAT: true + ENABLE_EIP_SNAT: false DEFAULT_SUBNET: "ovn-default" DEFAULT_VPC: "ovn-cluster" NODE_SUBNET: "join" #mesh network From 68613fe2500789245111ea60a07c6ee3dcde492d Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Mon, 4 Aug 2025 14:09:42 -0500 Subject: [PATCH 50/97] feat: add metadata ratelimits (#1114) This change adds limits to our metadata servers which will be essential to ensuring we maintain a stable environment. Docs: https://docs.openstack.org/neutron/2024.1/admin/config-metadata-rate-limiting.html Signed-off-by: Kevin Carter Co-authored-by: James Denton --- base-helm-configs/neutron/neutron-helm-overrides.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/base-helm-configs/neutron/neutron-helm-overrides.yaml b/base-helm-configs/neutron/neutron-helm-overrides.yaml index c7f082bfe..25a324a6d 100644 --- a/base-helm-configs/neutron/neutron-helm-overrides.yaml +++ b/base-helm-configs/neutron/neutron-helm-overrides.yaml @@ -205,6 +205,13 @@ conf: ovn_l3_scheduler: leastloaded ovn_nb_connection: "tcp:127.0.0.1:6641" ovn_sb_connection: "tcp:127.0.0.1:6642" + metadata_rate_limiting: + rate_limit_enabled: true + ip_versions: 4 + base_window_duration: 60 + base_query_rate_limit: 6 + burst_window_duration: 10 + burst_query_rate_limit: 2 neutron_api_uwsgi: uwsgi: processes: 2 From 5e77ea820e718845c6638372b618a411d3e0cac5 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Tue, 5 Aug 2025 08:39:15 -0500 Subject: [PATCH 51/97] chore: remove extra container files (#1115) This change removes several container files and workflows that have been replace with the genestack-images repo. Signed-off-by: Kevin Carter --- .github/workflows/release-horizon-rxt.yml | 118 ---------------------- .github/workflows/smoke-horizon-rxt.yml | 42 -------- Containerfiles/HorizonRXT-Containerfile | 15 --- Containerfiles/MagnumRXT-Containerfile | 14 --- 4 files changed, 189 deletions(-) delete mode 100644 .github/workflows/release-horizon-rxt.yml delete mode 100644 .github/workflows/smoke-horizon-rxt.yml delete mode 100644 Containerfiles/HorizonRXT-Containerfile delete mode 100644 Containerfiles/MagnumRXT-Containerfile diff --git a/.github/workflows/release-horizon-rxt.yml b/.github/workflows/release-horizon-rxt.yml deleted file mode 100644 index 9d7f7d8b6..000000000 --- a/.github/workflows/release-horizon-rxt.yml +++ /dev/null @@ -1,118 +0,0 @@ -# -name: Create and publish a Horizon RXT compatible image - -# Configures this workflow to run every time a change is pushed to the branch called `release`. -on: - workflow_dispatch: - inputs: - imageTag: - description: 'Set tag for the image' - required: true - default: 'master-ubuntu_jammy' - type: choice - options: - - master-ubuntu_jammy - - 2023.1-ubuntu_jammy - - 2023.2-ubuntu_jammy - - 2024.1-ubuntu_jammy - pluginTag: - description: 'Set release used for the build environment' - required: true - default: 'master' - type: choice - options: - - "master" - - "2023.1" - - "2023.2" - - "2024.1" - -# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - build-and-push-image: - outputs: - MY_DATE: ${{ steps.mydate.outputs.MY_DATE }} - MY_CONTAINER: ${{ steps.mycontainer.outputs.MY_CONTAINER }} - runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. - # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. - # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - - name: Dynamically set MY_DATE environment variable - run: echo "MY_DATE=$(date +%s)" >> $GITHUB_ENV - - name: Build and push Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: Containerfiles/HorizonRXT-Containerfile - push: true - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/horizon-rxt:${{ github.event.inputs.pluginTag }} - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/horizon-rxt:${{ github.event.inputs.pluginTag }}-${{ env.MY_DATE }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - VERSION=${{ github.event.inputs.imageTag }} - PLUGIN_VERSION=${{ github.event.inputs.pluginTag }} - - name: Dynamically set MY_CONTAINER output option - id: mycontainer - run: echo "MY_CONTAINER=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/horizon:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }}" >> $GITHUB_OUTPUT - - name: Dynamically set MY_DATE output option - id: mydate - run: echo "MY_DATE=${{ env.MY_DATE }}" >> $GITHUB_OUTPUT - - change-original-images: - runs-on: ubuntu-latest - needs: [build-and-push-image] - permissions: - contents: write - pull-requests: write - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Dynamically update the original images file - run: jq '. + ["${{ needs.build-and-push-image.outputs.MY_CONTAINER }}"] | sort' .original-images.json | tee .original-images.json.new - - name: Rewrite original images file - run: mv .original-images.json.new .original-images.json - - name: Create Pull Request - id: cpr - uses: peter-evans/create-pull-request@v7 - with: - commit-message: Update original images with new container - committer: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> - author: ${{ github.actor }} <${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com> - signoff: false - branch: ${{ needs.build-and-push-image.outputs.MY_DATE }} - sign-commits: true - delete-branch: true - title: 'chore: Update original images' - body: | - Update container image - - Updated original image file with container ${{needs.build-and-push-image.outputs.MY_CONTAINER}} - change request Auto-generated - labels: | - container images - automated pr - draft: false diff --git a/.github/workflows/smoke-horizon-rxt.yml b/.github/workflows/smoke-horizon-rxt.yml deleted file mode 100644 index 5c96b5072..000000000 --- a/.github/workflows/smoke-horizon-rxt.yml +++ /dev/null @@ -1,42 +0,0 @@ -# -name: Run build check for the Horizon RXT compatible image - -on: - pull_request: - paths: - - Containerfiles/HorizonRXT-Containerfile - -# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. -jobs: - build-and-push-image: - runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. - permissions: - contents: read - packages: read - steps: - - name: Checkout repository - uses: actions/checkout@v4 - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 - with: - context: . - file: Containerfiles/HorizonRXT-Containerfile - push: false - tags: | - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/horizon-rxt:master-ubuntu_jammy - build-args: | - VERSION=master-ubuntu_jammy - PLUGIN_VERSION=master diff --git a/Containerfiles/HorizonRXT-Containerfile b/Containerfiles/HorizonRXT-Containerfile deleted file mode 100644 index 696b2c971..000000000 --- a/Containerfiles/HorizonRXT-Containerfile +++ /dev/null @@ -1,15 +0,0 @@ -ARG VERSION=master-ubuntu_jammy -FROM openstackhelm/horizon:${VERSION} as build -ARG PLUGIN_VERSION=master -RUN apt update && apt install -y git -RUN /var/lib/openstack/bin/pip install --upgrade --force-reinstall pip -RUN export ORIG_PLUGIN_VERSION="${PLUGIN_VERSION}"; \ - if [ "${PLUGIN_VERSION}" != 'master' ]; then export PLUGIN_VERSION=stable/${PLUGIN_VERSION}; fi; \ - . /var/lib/openstack/bin/activate; \ - /var/lib/openstack/bin/pip install --constraint=https://releases.openstack.org/constraints/upper/${ORIG_PLUGIN_VERSION} \ - git+https://opendev.org/openstack/heat-dashboard@${PLUGIN_VERSION}#egg=heat_dashboard \ - git+https://opendev.org/openstack/octavia-dashboard@${PLUGIN_VERSION}#egg=octavia_dashboard -RUN find /var/lib/openstack -regex '^.*\(__pycache__\|\.py[co]\)$' -delete - -FROM openstackhelm/horizon:${VERSION} -COPY --from=build /var/lib/openstack/. /var/lib/openstack/ diff --git a/Containerfiles/MagnumRXT-Containerfile b/Containerfiles/MagnumRXT-Containerfile deleted file mode 100644 index 5b6c78b36..000000000 --- a/Containerfiles/MagnumRXT-Containerfile +++ /dev/null @@ -1,14 +0,0 @@ -ARG VERSION=master-ubuntu_jammy -FROM openstackhelm/magnum:${VERSION} as build -ARG PLUGIN_VERSION=master -RUN apt-get update && apt-get install -y git && apt clean -RUN export ORIG_PLUGIN_VERSION="${PLUGIN_VERSION}"; \ -if [ "${PLUGIN_VERSION}" != 'master' ]; then export PLUGIN_VERSION=stable/${PLUGIN_VERSION}; fi; \ -/var/lib/openstack/bin/activate; \ -/var/lib/openstack/bin/pip install git+https://github.com/openstack/oslo.db@${PLUGIN_VERSION}#egg=oslo_db \ - git+https://opendev.org/openstack/magnum-capi-helm@${PLUGIN_VERSION}#egg=magnum_capi_helm -RUN /var/lib/openstack/bin/pip install --upgrade --force-reinstall pip -RUN find /var/lib/openstack -regex '^.*\(__pycache__\|\.py[co]\)$' -delete - -FROM openstackhelm/magnum:${VERSION} -COPY --from=build /var/lib/openstack/. /var/lib/openstack/ From 754f839f7e2d6740eca1dd913253d1909e4e788c Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Tue, 5 Aug 2025 09:46:54 -0500 Subject: [PATCH 52/97] fix: unify the skyline limits (#1116) The basic skyline limits were set so low that the HPA would scale to the max for no reason. This change sets the skyline limits to a more sane value for a typical small scale environment. Signed-off-by: Kevin Carter --- base-kustomize/skyline/base/deployment-apiserver.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/base-kustomize/skyline/base/deployment-apiserver.yaml b/base-kustomize/skyline/base/deployment-apiserver.yaml index dedfb32cc..a438485ca 100644 --- a/base-kustomize/skyline/base/deployment-apiserver.yaml +++ b/base-kustomize/skyline/base/deployment-apiserver.yaml @@ -339,7 +339,7 @@ spec: imagePullPolicy: IfNotPresent resources: requests: - memory: "64Mi" + memory: "256Mi" cpu: "100m" limits: memory: "4096Mi" @@ -361,11 +361,11 @@ spec: image: "ghcr.io/rackerlabs/genestack-images/skyline:2024.2-latest" imagePullPolicy: IfNotPresent resources: - limits: - memory: "1Gi" requests: - cpu: "0.25" - memory: "64Mi" + memory: "256Mi" + cpu: "100m" + limits: + memory: "4096Mi" command: - bash - -c From 578ed91295b4c185486b9e60b4df064269454241 Mon Sep 17 00:00:00 2001 From: Nitin Gupta <63388430+niti6869@users.noreply.github.com> Date: Wed, 6 Aug 2025 22:40:29 +0530 Subject: [PATCH 53/97] fix: OSPC-1358: Neutron server QueuePool limit exceeded (#1119) Increasing max_pool_size from 5 (default value) to 30, and max_overflow from 50 to 60. --- base-helm-configs/neutron/neutron-helm-overrides.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/base-helm-configs/neutron/neutron-helm-overrides.yaml b/base-helm-configs/neutron/neutron-helm-overrides.yaml index 25a324a6d..30e5a03d9 100644 --- a/base-helm-configs/neutron/neutron-helm-overrides.yaml +++ b/base-helm-configs/neutron/neutron-helm-overrides.yaml @@ -177,6 +177,8 @@ conf: idle_timeout: 3600 mysql_sql_mode: {} use_db_reconnect: true + max_pool_size: 30 + max_overflow: 60 pool_timeout: 60 max_retries: -1 oslo_messaging_rabbit: From 48011d3e82736f14b093fd9022004498400b5581 Mon Sep 17 00:00:00 2001 From: Jitendra Date: Wed, 6 Aug 2025 23:24:58 +0530 Subject: [PATCH 54/97] OSPC-1403: added the script , values.yaml and docs for redis operator --- .../redis-operator-helm-overrides.yaml | 227 ++++++++++++++++++ bin/install-redis-operator.sh | 58 +++++ docs/redis-setup-guide.md | 57 +++++ 3 files changed, 342 insertions(+) create mode 100644 base-helm-configs/redis-operator/redis-operator-helm-overrides.yaml create mode 100644 bin/install-redis-operator.sh create mode 100644 docs/redis-setup-guide.md diff --git a/base-helm-configs/redis-operator/redis-operator-helm-overrides.yaml b/base-helm-configs/redis-operator/redis-operator-helm-overrides.yaml new file mode 100644 index 000000000..b4199de1a --- /dev/null +++ b/base-helm-configs/redis-operator/redis-operator-helm-overrides.yaml @@ -0,0 +1,227 @@ +# Redis Operator and Cluster Overrides +# Defines custom settings to override defaults in base values.yaml + +# -- Cluster DNS name +clusterName: ${CLUSTER_NAME} + +# Namespace configuration +# Controls the namespace for the Redis operator and cluster +namespace: + create: true + name: redis-systems + +# Redis Cluster Configuration Overrides +redisCluster: + # -- Name of the Redis cluster (optional, defaults to empty) + name: "redis-cluster" + # -- Number of shards in the cluster (implied by leader/follower replicas) + clusterSize: 3 + # -- Redis version to use + clusterVersion: v7 + # -- Enable persistence for the cluster + persistenceEnabled: true + # -- Image configuration for Redis pods + image: + repository: quay.io/opstree/redis + tag: v7.0.15 + pullPolicy: IfNotPresent + # -- Secrets for image pull (optional) + imagePullSecrets: [] + # - name: Secret with Registry credentials + # -- Redis authentication secret (optional) + redisSecret: + secretName: "" + secretKey: "" + # -- Resource requests and limits (optional) + resources: {} + # requests: + # cpu: 100m + # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + # -- Minimum seconds a pod must be ready before considered available + minReadySeconds: 0 + # -- Some fields of statefulset are immutable, such as volumeClaimTemplates. + # When set to true, the operator will delete the statefulset and recreate it. + #Default is false. + recreateStatefulSetOnUpdateInvalid: false + # -- Enable pod anti-affinity between leader and follower pods by adding the + # appropriate label. + # Notice that this requires the operator to have its mutating webhook enabled, + # otherwise it will only add an annotation to the RedisCluster CR. Default is + # false. + enableMasterSlaveAntiAffinity: false + # -- Leader configuration + leader: + replicas: 3 + serviceType: ClusterIP + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: disktype + # operator: In + # values: + # - ssd + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + nodeSelector: {} + # memory: medium + securityContext: {} + pdb: + enabled: false + maxUnavailable: 1 + minAvailable: 1 + livenessProbe: {} + # timeoutSeconds: 30 + # periodSeconds: 45 + # successThreshold: 1 + # failureThreshold: 4 + # initialDelaySeconds: 15 + readinessProbe: {} + # timeoutSeconds: 30 + # periodSeconds: 45 + # successThreshold: 1 + # failureThreshold: 4 + # initialDelaySeconds: 15 + # -- Follower configuration + follower: + replicas: 3 + serviceType: ClusterIP + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: disktype + # operator: In + # values: + # - ssd + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + nodeSelector: {} + # memory: medium + securityContext: {} + pdb: + enabled: false + maxUnavailable: 1 + minAvailable: 1 + livenessProbe: {} + # timeoutSeconds: 30 + # periodSeconds: 45 + # successThreshold: 1 + # failureThreshold: 4 + # initialDelaySeconds: 15 + readinessProbe: {} + # timeoutSeconds: 30 + # periodSeconds: 45 + # successThreshold: 1 + # failureThreshold: 4 + # initialDelaySeconds: 15 + +# -- Labels and Annotations +labels: {} + # foo: bar + # test: echo + +# -- External Configuration +externalConfig: + enabled: false + data: {} + # tcp-keepalive 400 + # slowlog-max-len 158 + # stream-node-max-bytes 2048 + +# -- External Service Configuration +externalService: + enabled: false + serviceType: {} + port: {} + annotations: {} + # foo: bar + +# -- Monitoring and Exporter +serviceMonitor: + enabled: false + interval: {} + scrapeTimeout: {} + namespace: {} + extraLabels: {} + # foo: bar + # team: devops +redisExporter: + enabled: false + image: {} + tag: {} + imagePullPolicy: {} + resources: {} + # requests: + # cpu: 100m + # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + env: [] + # - name: VAR_NAME + # value: "value1" + securityContext: {} + +# -- Sidecars and Init Containers +sidecars: {} +initContainer: + enabled: false + image: {} + imagePullPolicy: {} + resources: {} + # requests: + # memory: "64Mi" + # cpu: "250m" + # limits: + # memory: "128Mi" + # cpu: "500m" + env: [] + command: [] + args: [] + +# -- Priority and Security +priorityClassName: "" +podSecurityContext: {} +TLS: + ca: {} + cert: {} + key: {} + secret: + secretName: "" +acl: + secret: + secretName: "" +env: [] + # - name: VAR_NAME + # value: "value1" +serviceAccountName: "" + +# -- Storage Specification +storageSpec: + volumeClaimTemplate: + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi + # storageClassName: standard + nodeConfVolume: true + nodeConfVolumeClaimTemplate: + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi + # selector: {} diff --git a/bin/install-redis-operator.sh b/bin/install-redis-operator.sh new file mode 100644 index 000000000..33b79c72e --- /dev/null +++ b/bin/install-redis-operator.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# shellcheck disable=SC2124,SC2145,SC2294 + +export VERSION="${VERSION:-0.21.0}" + +# Default parameter value +export CLUSTER_NAME=${CLUSTER_NAME:-cluster.local} + +# Directory to check for YAML files +CONFIG_DIR="/etc/genestack/helm-configs/redis-operator" + +# 'cluster.local' is the default value in base helm values file +if [ "${CLUSTER_NAME}" != "cluster.local" ]; then + CONFIG_FILE="$CONFIG_DIR/redis-operator-helm-overrides.yaml" + + mkdir -p "$CONFIG_DIR" + touch "$CONFIG_FILE" + + # Check if the file is empty and add/modify content accordingly + if [ ! -s "$CONFIG_FILE" ]; then + echo "clusterName: $CLUSTER_NAME" > "$CONFIG_FILE" + else + # If the clusterName line exists, modify it, otherwise add it at the end + if grep -q "^clusterName:" "$CONFIG_FILE"; then + sed -i -e "s/^clusterName: .*/clusterName: ${CLUSTER_NAME}/" "$CONFIG_FILE" + else + echo "clusterName: $CLUSTER_NAME" >> "$CONFIG_FILE" + fi + fi +fi + +# Add the redis-operator helm repository +helm repo add ot-helm https://ot-container-kit.github.io/helm-charts/ +helm repo update + +# Install the CRDs that match the version defined +helm upgrade --install --namespace=redis-systems --create-namespace redis-operator ot-helm/redis-operator --version "${VERSION}" + +# Helm command setup for Redis operator and cluster +HELM_CMD="helm upgrade --install redis-operator ot-helm/redis-operator \ + --namespace=redis-systems \ + --timeout 120m \ + -f /opt/genestack/base-helm-configs/redis-operator/redis-operator-helm-overrides.yaml" + +# Check if YAML files exist in the specified directory +if compgen -G "${CONFIG_DIR}/*.yaml" > /dev/null; then + # Add all YAML files from the directory to the helm command + for yaml_file in "${CONFIG_DIR}"/*.yaml; do + HELM_CMD+=" -f ${yaml_file}" + done +fi + +HELM_CMD+=" $@" + +# Run the helm command +echo "Executing Helm command:" +echo "${HELM_CMD}" +eval "${HELM_CMD}" diff --git a/docs/redis-setup-guide.md b/docs/redis-setup-guide.md new file mode 100644 index 000000000..1fdbd421b --- /dev/null +++ b/docs/redis-setup-guide.md @@ -0,0 +1,57 @@ +Redis Cluster Setup with Helm (OT-Container-Kit Redis Operator 0.21.0) +Redis Git hub Repository: https://github.com/OT-CONTAINER-KIT/redis-operator +Overview +Deploys a 6-pod Redis cluster (3 leaders, 3 followers) on an OpenStack Kubernetes cluster using Helm, integrated into Genestack base-helm-configs, distributed across 3 nodes. +Prerequisites + +Kubernetes cluster with kubectl configured. +Helm installed. +Genestack repository cloned: git clone https://github.com/rackerlabs/genestack.git. + +Deployment Steps + +Ensure values.yaml is configured in genestack/base-helm-configs with: +leader.replicas: 3 +follower.replicas: 3 +storageClassName: general +persistenceEnabled: true + + +Create the override file: Save redis-operator-helm-overrides.yaml in /etc/genestack/helm-configs/redis-operator/ with the provided content. +Deploy the Redis operator and cluster: +Run: ./bin/install-redis-operator.sh +Optional: Customize CLUSTER_NAME: ./bin/install-redis-operator.sh CLUSTER_NAME=mycluster + + +Verify deployment: kubectl get pods -n redis-systems -o wide + +Basic Testing + +Verify Pods: kubectl get pods -n redis-systems -o wide +Expected: 6 pods across 3 nodes, e.g., redis-cluster-leader-0 on node-1, etc. + + +Cluster Health: Inside a pod (e.g., kubectl exec -it redis-cluster-leader-0 -n redis-systems -- /bin/sh), run redis-cli --cluster check 127.0.0.1:6379 +Expected: cluster_state:ok + + +Read/Write: redis-cli --cluster call 127.0.0.1:6379 SET testkey testvalue and GET testkey +Expected: OK and testvalue + + +Replication: Write to leader, check follower with redis-cli --cluster call 127.0.0.1:6379 GET repltest +Expected: replvalue + + +Persistence: Set persistkey, restart pod, verify with GET persistkey +Expected: persistvalue + + +Logs: kubectl logs redis-cluster-leader-0 -n redis-systems +Expected: No critical errors + + +Customization + +Use redis-operator-helm-overrides.yaml to adjust clusterName, namespace, or enable externalService for external access. +Example: Set externalService.enabled: true and serviceType: LoadBalancer for external connectivity. From d0c83c5df2cb6db1451ddea30796255084398a27 Mon Sep 17 00:00:00 2001 From: Jitendra Date: Thu, 7 Aug 2025 22:46:57 +0530 Subject: [PATCH 55/97] OSPC-1403: Modified the yaml and script file to work on DFW-DEV env --- .../redis-operator/redis-operator-helm-overrides.yaml | 9 ++++----- bin/install-redis-operator.sh | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/base-helm-configs/redis-operator/redis-operator-helm-overrides.yaml b/base-helm-configs/redis-operator/redis-operator-helm-overrides.yaml index b4199de1a..7c465cd11 100644 --- a/base-helm-configs/redis-operator/redis-operator-helm-overrides.yaml +++ b/base-helm-configs/redis-operator/redis-operator-helm-overrides.yaml @@ -21,10 +21,9 @@ redisCluster: # -- Enable persistence for the cluster persistenceEnabled: true # -- Image configuration for Redis pods - image: - repository: quay.io/opstree/redis - tag: v7.0.15 - pullPolicy: IfNotPresent + image: quay.io/opstree/redis + tag: v7.0.15 + imagePullPolicy: IfNotPresent # -- Secrets for image pull (optional) imagePullSecrets: [] # - name: Secret with Registry credentials @@ -212,11 +211,11 @@ serviceAccountName: "" storageSpec: volumeClaimTemplate: spec: + # storageClassName: standard accessModes: ["ReadWriteOnce"] resources: requests: storage: 1Gi - # storageClassName: standard nodeConfVolume: true nodeConfVolumeClaimTemplate: spec: diff --git a/bin/install-redis-operator.sh b/bin/install-redis-operator.sh index 33b79c72e..598f3b5b9 100644 --- a/bin/install-redis-operator.sh +++ b/bin/install-redis-operator.sh @@ -37,7 +37,7 @@ helm repo update helm upgrade --install --namespace=redis-systems --create-namespace redis-operator ot-helm/redis-operator --version "${VERSION}" # Helm command setup for Redis operator and cluster -HELM_CMD="helm upgrade --install redis-operator ot-helm/redis-operator \ +HELM_CMD="helm upgrade --install redis-cluster ot-helm/redis-cluster \ --namespace=redis-systems \ --timeout 120m \ -f /opt/genestack/base-helm-configs/redis-operator/redis-operator-helm-overrides.yaml" From 214698441a15b50163ee788ab77487c4abe4377e Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Fri, 15 Aug 2025 10:29:34 -0500 Subject: [PATCH 56/97] feat: replace VERY OLD image with a newer one (#1127) Signed-off-by: Kevin Carter --- .original-images.json | 1 - base-helm-configs/octavia/octavia-helm-overrides.yaml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.original-images.json b/.original-images.json index f549d5c71..39b005732 100644 --- a/.original-images.json +++ b/.original-images.json @@ -1,7 +1,6 @@ [ "cr.fluentbit.io/fluent/fluent-bit", "docker.io/docker:17.07.0", - "docker.io/kolla/centos-source-openvswitch-vswitchd:master", "docker.io/kolla/ubuntu-source-nova-compute-ironic:master", "docker.io/library/postgres:14.5", "docker.io/library/postgres:14.5", diff --git a/base-helm-configs/octavia/octavia-helm-overrides.yaml b/base-helm-configs/octavia/octavia-helm-overrides.yaml index d9e658d25..34d7309d4 100644 --- a/base-helm-configs/octavia/octavia-helm-overrides.yaml +++ b/base-helm-configs/octavia/octavia-helm-overrides.yaml @@ -15,7 +15,7 @@ images: octavia_health_manager_init: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" octavia_housekeeping: "ghcr.io/rackerlabs/genestack-images/octavia:2024.1-latest" octavia_worker: "ghcr.io/rackerlabs/genestack-images/octavia:2024.1-latest" - openvswitch_vswitchd: "docker.io/kolla/centos-source-openvswitch-vswitchd:rocky" + openvswitch_vswitchd: "ghcr.io/rackerlabs/genestack-images/ovs:v3.5.1-latest" rabbit_init: null test: null From 1ed089bd00d0cd64a357c0157260c9fadf69e316 Mon Sep 17 00:00:00 2001 From: Dan With Date: Fri, 15 Aug 2025 14:33:18 -0500 Subject: [PATCH 57/97] Feat: Add SSL encryption between cinder-volume Ontap driver and NetApp (#1129) This ensures that username/pass are not sent in clear text. This does NOT encrypt all traffic from virtual machines to attached NetApp volumes. Adds variables and logic to playbook for SSL cert deployment to storage node. Correctly updates ca certificates on storage node based on Linux OS version. This assumes DNS resolution for the NetApp cluster hostname and NetApp vserver name. --- ...eploy-cinder-netapp-volumes-reference.yaml | 58 +++++++++++++++++++ .../playbooks/templates/eventlet_ssl_patch.py | 13 +++++ .../templates/zzz_eventlet_ssl_patch.pth | 1 + 3 files changed, 72 insertions(+) create mode 100644 ansible/playbooks/templates/eventlet_ssl_patch.py create mode 100644 ansible/playbooks/templates/zzz_eventlet_ssl_patch.pth diff --git a/ansible/playbooks/deploy-cinder-netapp-volumes-reference.yaml b/ansible/playbooks/deploy-cinder-netapp-volumes-reference.yaml index 50cf5db3b..cf91d3434 100644 --- a/ansible/playbooks/deploy-cinder-netapp-volumes-reference.yaml +++ b/ansible/playbooks/deploy-cinder-netapp-volumes-reference.yaml @@ -9,6 +9,11 @@ cinder_storage_network_interface_secondary: ansible_br_storage_secondary cinder_backend_name: "block-ha-performance-at-rest-encrypted,block-ha-standard-at-rest-encrypted,block-ha-performance-end-to-end-encrypted,block-ha-standard-end-to-end-encrypted" storage_network_multipath: false + enable_netapp_ssl: false + netapp_cert_src_dir: "/opt/genestack/ansible/playbooks/templates/" + netapp_cert_filenames: + - "ontap-cluster-host.crt" + - "ontap-vserver-host.crt" handlers: - name: Restart cinder-volume-netapp systemd services ansible.builtin.systemd: @@ -65,6 +70,44 @@ regexp: '^InitiatorName=.*|^GenerateName=.*' line: "InitiatorName={{ initiator_name }}" + - name: Copy NetApp client certificates Debian + ansible.builtin.copy: + src: "{{ netapp_cert_src_dir }}/{{ item }}" + dest: "/usr/local/share/ca-certificates/{{ item }}" + owner: root + group: root + mode: '0644' + when: + - enable_netapp_ssl | bool + - ansible_os_family | lower == "debian" + loop: "{{ netapp_cert_filenames }}" + + - name: Copy NetApp client certificates Redhat + ansible.builtin.copy: + src: "{{ netapp_cert_src_dir }}/{{ item }}" + dest: "/etc/pki/ca-trust/source/anchors/{{ item }}" + owner: root + group: root + mode: '0644' + when: + - enable_netapp_ssl | bool + - ansible_os_family | lower == "redhat" + loop: "{{ netapp_cert_filenames }}" + + - name: Update CA certificate trust Debian + ansible.builtin.command: + cmd: /usr/sbin/update-ca-certificates + when: + - enable_netapp_ssl | bool + - ansible_os_family | lower == "debian" + + - name: Update CA certificate trust Redhat + ansible.builtin.command: + cmd: /usr/sbin/update-ca-trust extract + when: + - enable_netapp_ssl | bool + - ansible_os_family | lower == "redhat" + - name: Upgrade pip and install required packages ansible.builtin.pip: name: @@ -76,6 +119,21 @@ virtualenv: /opt/cinder virtualenv_command: python3 -m venv + - name: Install eventlet SSL patch + ansible.builtin.copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: root + group: root + mode: "{{ item.mode | default('0644') }}" + loop: + - src: "{{ playbook_dir }}/templates/zzz_eventlet_ssl_patch.pth" + dest: /opt/cinder/lib/python3.12/site-packages/zzz_eventlet_ssl_patch.pth + mode: "0644" + - src: "{{ playbook_dir }}/templates/eventlet_ssl_patch.py" + dest: /opt/cinder/lib/python3.12/site-packages/eventlet_ssl_patch.py + mode: "0644" + - name: Create the cinder system user ansible.builtin.user: name: cinder diff --git a/ansible/playbooks/templates/eventlet_ssl_patch.py b/ansible/playbooks/templates/eventlet_ssl_patch.py new file mode 100644 index 000000000..4b5f03098 --- /dev/null +++ b/ansible/playbooks/templates/eventlet_ssl_patch.py @@ -0,0 +1,13 @@ +import os, ssl + +os.environ.setdefault("EVENTLET_NO_GREENDNS", "yes") +try: + from eventlet.green import ssl as gssl + + def _safe_green_create_default_context(*a, **kw): + return ssl._create_default_https_context(*a, **kw) + + gssl.green_create_default_context = _safe_green_create_default_context +except Exception as e: + # don't crash the process if eventlet isn't here yet + pass diff --git a/ansible/playbooks/templates/zzz_eventlet_ssl_patch.pth b/ansible/playbooks/templates/zzz_eventlet_ssl_patch.pth new file mode 100644 index 000000000..2bc84d6c5 --- /dev/null +++ b/ansible/playbooks/templates/zzz_eventlet_ssl_patch.pth @@ -0,0 +1 @@ +import eventlet_ssl_patch From f5d42719686f5f489c52f972655d84f84e54aef7 Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Fri, 15 Aug 2025 15:32:13 -0500 Subject: [PATCH 58/97] fix: correct helm labels to keep resources on uninstall (#1128) * fix: correct helm labels to keep resources on uninstall * fix: yaml indent errors * fix: one more yamllint error * fix: maybe, finally, done --- .../base/barbican-rabbitmq-queue.yaml | 6 +- .../glance/base/glance-rabbitmq-queue.yaml | 7 +- .../heat/base/heat-rabbitmq-queue.yaml | 28 +++- .../ironic/{aoi => aio}/kustomization.yaml | 0 .../ironic/base/ironic-mariadb-database.yaml | 12 ++ .../ironic/base/ironic-rabbitmq-queue.yaml | 16 +++ .../base/keystone-rabbitmq-queue.yaml | 127 +++++++++--------- .../base/masakari-rabbitmq-queue.yaml | 16 +++ 8 files changed, 142 insertions(+), 70 deletions(-) rename base-kustomize/ironic/{aoi => aio}/kustomization.yaml (100%) diff --git a/base-kustomize/barbican/base/barbican-rabbitmq-queue.yaml b/base-kustomize/barbican/base/barbican-rabbitmq-queue.yaml index dcbe1893d..9dd6f2505 100644 --- a/base-kustomize/barbican/base/barbican-rabbitmq-queue.yaml +++ b/base-kustomize/barbican/base/barbican-rabbitmq-queue.yaml @@ -4,14 +4,16 @@ kind: User metadata: name: barbican namespace: openstack + labels: + app.kubernetes.io/managed-by: "Helm" annotations: helm.sh/resource-policy: keep meta.helm.sh/release-name: "barbican" meta.helm.sh/release-namespace: "openstack" spec: tags: - - management # available tags are 'management', 'policymaker', 'monitoring' and 'administrator' - - policymaker + - management # available tags are 'management', 'policymaker', 'monitoring' and 'administrator' + - policymaker rabbitmqClusterReference: name: rabbitmq # rabbitmqCluster must exist in the same namespace as this resource namespace: openstack diff --git a/base-kustomize/glance/base/glance-rabbitmq-queue.yaml b/base-kustomize/glance/base/glance-rabbitmq-queue.yaml index f3066cf28..2c48dc15d 100644 --- a/base-kustomize/glance/base/glance-rabbitmq-queue.yaml +++ b/base-kustomize/glance/base/glance-rabbitmq-queue.yaml @@ -4,15 +4,16 @@ kind: User metadata: name: glance namespace: openstack + labels: + app.kubernetes.io/managed-by: "Helm" annotations: helm.sh/resource-policy: keep - app.kubernetes.io/managed-by: "Helm" meta.helm.sh/release-name: "glance" meta.helm.sh/release-namespace: "openstack" spec: tags: - - management # available tags are 'management', 'policymaker', 'monitoring' and 'administrator' - - policymaker + - management # available tags are 'management', 'policymaker', 'monitoring' and 'administrator' + - policymaker rabbitmqClusterReference: name: rabbitmq # rabbitmqCluster must exist in the same namespace as this resource namespace: openstack diff --git a/base-kustomize/heat/base/heat-rabbitmq-queue.yaml b/base-kustomize/heat/base/heat-rabbitmq-queue.yaml index d9b6e0f87..1daaa7a87 100644 --- a/base-kustomize/heat/base/heat-rabbitmq-queue.yaml +++ b/base-kustomize/heat/base/heat-rabbitmq-queue.yaml @@ -4,10 +4,16 @@ kind: User metadata: name: heat namespace: openstack + labels: + app.kubernetes.io/managed-by: "Helm" + annotations: + helm.sh/resource-policy: keep + meta.helm.sh/release-name: "heat" + meta.helm.sh/release-namespace: "openstack" spec: tags: - - management # available tags are 'management', 'policymaker', 'monitoring' and 'administrator' - - policymaker + - management # available tags are 'management', 'policymaker', 'monitoring' and 'administrator' + - policymaker rabbitmqClusterReference: name: rabbitmq # rabbitmqCluster must exist in the same namespace as this resource namespace: openstack @@ -19,6 +25,12 @@ kind: Vhost metadata: name: heat-vhost namespace: openstack + labels: + app.kubernetes.io/managed-by: "Helm" + annotations: + helm.sh/resource-policy: keep + meta.helm.sh/release-name: "heat" + meta.helm.sh/release-namespace: "openstack" spec: name: "heat" # vhost name; required and cannot be updated defaultQueueType: quorum # default queue type for this vhost; require RabbitMQ version 3.11.12 or above @@ -38,6 +50,12 @@ kind: Queue metadata: name: heat-queue namespace: openstack + labels: + app.kubernetes.io/managed-by: "Helm" + annotations: + helm.sh/resource-policy: keep + meta.helm.sh/release-name: "heat" + meta.helm.sh/release-namespace: "openstack" spec: name: heat-qq # name of the queue vhost: "heat" # default to '/' if not provided @@ -53,6 +71,12 @@ kind: Permission metadata: name: heat-permission namespace: openstack + labels: + app.kubernetes.io/managed-by: "Helm" + annotations: + helm.sh/resource-policy: keep + meta.helm.sh/release-name: "heat" + meta.helm.sh/release-namespace: "openstack" spec: vhost: "heat" # name of a vhost userReference: diff --git a/base-kustomize/ironic/aoi/kustomization.yaml b/base-kustomize/ironic/aio/kustomization.yaml similarity index 100% rename from base-kustomize/ironic/aoi/kustomization.yaml rename to base-kustomize/ironic/aio/kustomization.yaml diff --git a/base-kustomize/ironic/base/ironic-mariadb-database.yaml b/base-kustomize/ironic/base/ironic-mariadb-database.yaml index 332004146..7bea3bae6 100644 --- a/base-kustomize/ironic/base/ironic-mariadb-database.yaml +++ b/base-kustomize/ironic/base/ironic-mariadb-database.yaml @@ -4,8 +4,12 @@ kind: Database metadata: name: ironic namespace: openstack + labels: + app.kubernetes.io/managed-by: "Helm" annotations: helm.sh/resource-policy: keep + meta.helm.sh/release-name: "ironic" + meta.helm.sh/release-namespace: "openstack" spec: # If you want the database to be created with a different name than the resource name # name: data-custom @@ -20,8 +24,12 @@ kind: User metadata: name: ironic namespace: openstack + labels: + app.kubernetes.io/managed-by: "Helm" annotations: helm.sh/resource-policy: keep + meta.helm.sh/release-name: "ironic" + meta.helm.sh/release-namespace: "openstack" spec: # If you want the user to be created with a different name than the resource name # name: user-custom @@ -40,8 +48,12 @@ kind: Grant metadata: name: ironic-grant namespace: openstack + labels: + app.kubernetes.io/managed-by: "Helm" annotations: helm.sh/resource-policy: keep + meta.helm.sh/release-name: "ironic" + meta.helm.sh/release-namespace: "openstack" spec: mariaDbRef: name: mariadb-cluster diff --git a/base-kustomize/ironic/base/ironic-rabbitmq-queue.yaml b/base-kustomize/ironic/base/ironic-rabbitmq-queue.yaml index 2846e95c6..89d710459 100644 --- a/base-kustomize/ironic/base/ironic-rabbitmq-queue.yaml +++ b/base-kustomize/ironic/base/ironic-rabbitmq-queue.yaml @@ -4,8 +4,12 @@ kind: User metadata: name: ironic namespace: openstack + labels: + app.kubernetes.io/managed-by: "Helm" annotations: helm.sh/resource-policy: keep + meta.helm.sh/release-name: "ironic" + meta.helm.sh/release-namespace: "openstack" spec: tags: - management # available tags are 'management', 'policymaker', 'monitoring' and 'administrator' @@ -21,8 +25,12 @@ kind: Vhost metadata: name: ironic-vhost namespace: openstack + labels: + app.kubernetes.io/managed-by: "Helm" annotations: helm.sh/resource-policy: keep + meta.helm.sh/release-name: "ironic" + meta.helm.sh/release-namespace: "openstack" spec: name: "ironic" # vhost name; required and cannot be updated defaultQueueType: quorum # default queue type for this vhost; require RabbitMQ version 3.11.12 or above @@ -42,8 +50,12 @@ kind: Queue metadata: name: ironic-queue namespace: openstack + labels: + app.kubernetes.io/managed-by: "Helm" annotations: helm.sh/resource-policy: keep + meta.helm.sh/release-name: "ironic" + meta.helm.sh/release-namespace: "openstack" spec: name: ironic-qq # name of the queue vhost: "ironic" # default to '/' if not provided @@ -59,8 +71,12 @@ kind: Permission metadata: name: ironic-permission namespace: openstack + labels: + app.kubernetes.io/managed-by: "Helm" annotations: helm.sh/resource-policy: keep + meta.helm.sh/release-name: "ironic" + meta.helm.sh/release-namespace: "openstack" spec: vhost: "ironic" # name of a vhost userReference: diff --git a/base-kustomize/keystone/base/keystone-rabbitmq-queue.yaml b/base-kustomize/keystone/base/keystone-rabbitmq-queue.yaml index 9d129a725..50a350e17 100644 --- a/base-kustomize/keystone/base/keystone-rabbitmq-queue.yaml +++ b/base-kustomize/keystone/base/keystone-rabbitmq-queue.yaml @@ -2,40 +2,41 @@ apiVersion: rabbitmq.com/v1beta1 kind: User metadata: - name: keystone - namespace: openstack - annotations: - helm.sh/resource-policy: keep - app.kubernetes.io/managed-by: "Helm" - meta.helm.sh/release-name: "keystone" - meta.helm.sh/release-namespace: "openstack" + name: keystone + namespace: openstack + labels: + app.kubernetes.io/managed-by: "Helm" + annotations: + helm.sh/resource-policy: keep + meta.helm.sh/release-name: "keystone" + meta.helm.sh/release-namespace: "openstack" spec: - tags: - - management - - policymaker - rabbitmqClusterReference: - name: rabbitmq # rabbitmqCluster must exist in the same namespace as this resource - namespace: openstack - importCredentialsSecret: - name: keystone-rabbitmq-password + tags: + - management + - policymaker + rabbitmqClusterReference: + name: rabbitmq # rabbitmqCluster must exist in the same namespace as this resource + namespace: openstack + importCredentialsSecret: + name: keystone-rabbitmq-password --- apiVersion: rabbitmq.com/v1beta1 kind: Vhost metadata: - name: keystone-vhost - namespace: openstack - labels: - app.kubernetes.io/managed-by: "Helm" - annotations: - helm.sh/resource-policy: keep - meta.helm.sh/release-name: "keystone" - meta.helm.sh/release-namespace: "openstack" + name: keystone-vhost + namespace: openstack + labels: + app.kubernetes.io/managed-by: "Helm" + annotations: + helm.sh/resource-policy: keep + meta.helm.sh/release-name: "keystone" + meta.helm.sh/release-namespace: "openstack" spec: - name: "keystone" # vhost name; required and cannot be updated - defaultQueueType: quorum # default queue type for this vhost; require RabbitMQ version 3.11.12 or above - rabbitmqClusterReference: - name: rabbitmq # rabbitmqCluster must exist in the same namespace as this resource - namespace: openstack + name: "keystone" # vhost name; required and cannot be updated + defaultQueueType: quorum # default queue type for this vhost; require RabbitMQ version 3.11.12 or above + rabbitmqClusterReference: + name: rabbitmq # rabbitmqCluster must exist in the same namespace as this resource + namespace: openstack # status: # conditions: # - lastTransitionTime: "" @@ -47,50 +48,50 @@ spec: apiVersion: rabbitmq.com/v1beta1 kind: Queue metadata: - name: keystone-queue - namespace: openstack - labels: - app.kubernetes.io/managed-by: "Helm" - annotations: - helm.sh/resource-policy: keep - meta.helm.sh/release-name: "keystone" - meta.helm.sh/release-namespace: "openstack" + name: keystone-queue + namespace: openstack + labels: + app.kubernetes.io/managed-by: "Helm" + annotations: + helm.sh/resource-policy: keep + meta.helm.sh/release-name: "keystone" + meta.helm.sh/release-namespace: "openstack" spec: - name: keystone-qq # name of the queue - vhost: "keystone" # default to '/' if not provided - type: quorum # without providing a queue type, rabbitmq creates a classic queue - autoDelete: false - durable: true # setting 'durable' to false means this queue won't survive a server restart - rabbitmqClusterReference: - name: rabbitmq # rabbitmqCluster must exist in the same namespace as this resource - namespace: openstack + name: keystone-qq # name of the queue + vhost: "keystone" # default to '/' if not provided + type: quorum # without providing a queue type, rabbitmq creates a classic queue + autoDelete: false + durable: true # setting 'durable' to false means this queue won't survive a server restart + rabbitmqClusterReference: + name: rabbitmq # rabbitmqCluster must exist in the same namespace as this resource + namespace: openstack --- apiVersion: rabbitmq.com/v1beta1 kind: Permission metadata: - name: keystone-permission - namespace: openstack - labels: - app.kubernetes.io/managed-by: "Helm" - annotations: - helm.sh/resource-policy: keep - meta.helm.sh/release-name: "keystone" - meta.helm.sh/release-namespace: "openstack" + name: keystone-permission + namespace: openstack + labels: + app.kubernetes.io/managed-by: "Helm" + annotations: + helm.sh/resource-policy: keep + meta.helm.sh/release-name: "keystone" + meta.helm.sh/release-namespace: "openstack" spec: - vhost: "keystone" # name of a vhost - userReference: - name: "keystone" # name of a user.rabbitmq.com in the same namespace; must specify either spec.userReference or spec.user - permissions: - write: ".*" - configure: ".*" - read: ".*" - rabbitmqClusterReference: - name: rabbitmq # rabbitmqCluster must exist in the same namespace as this resource - namespace: openstack + vhost: "keystone" # name of a vhost + userReference: + name: "keystone" # name of a user.rabbitmq.com in the same namespace; must specify either spec.userReference or spec.user + permissions: + write: ".*" + configure: ".*" + read: ".*" + rabbitmqClusterReference: + name: rabbitmq # rabbitmqCluster must exist in the same namespace as this resource + namespace: openstack # status: # conditions: # - lastTransitionTime: "" # status: "True" # true, false, or unknown # type: Ready # Reason: "SuccessfulCreateOrUpdate" # status false result in reason FailedCreateOrUpdate -# Message: "" # set when status is false \ No newline at end of file +# Message: "" # set when status is false diff --git a/base-kustomize/masakari/base/masakari-rabbitmq-queue.yaml b/base-kustomize/masakari/base/masakari-rabbitmq-queue.yaml index a770034ef..3a5bdbfd0 100644 --- a/base-kustomize/masakari/base/masakari-rabbitmq-queue.yaml +++ b/base-kustomize/masakari/base/masakari-rabbitmq-queue.yaml @@ -4,8 +4,12 @@ kind: User metadata: name: masakari namespace: openstack + labels: + app.kubernetes.io/managed-by: "Helm" annotations: helm.sh/resource-policy: keep + meta.helm.sh/release-name: "masakari" + meta.helm.sh/release-namespace: "openstack" spec: tags: - management # available tags are 'management', 'policymaker', 'monitoring' and 'administrator' @@ -21,8 +25,12 @@ kind: Vhost metadata: name: masakari-vhost namespace: openstack + labels: + app.kubernetes.io/managed-by: "Helm" annotations: helm.sh/resource-policy: keep + meta.helm.sh/release-name: "masakari" + meta.helm.sh/release-namespace: "openstack" spec: name: "masakari" # vhost name; required and cannot be updated defaultQueueType: quorum # default queue type for this vhost; require RabbitMQ version 3.11.12 or above @@ -35,8 +43,12 @@ kind: Queue metadata: name: masakari-queue namespace: openstack + labels: + app.kubernetes.io/managed-by: "Helm" annotations: helm.sh/resource-policy: keep + meta.helm.sh/release-name: "masakari" + meta.helm.sh/release-namespace: "openstack" spec: name: masakari-qq # name of the queue vhost: "masakari" # default to '/' if not provided @@ -52,8 +64,12 @@ kind: Permission metadata: name: masakari-permission namespace: openstack + labels: + app.kubernetes.io/managed-by: "Helm" annotations: helm.sh/resource-policy: keep + meta.helm.sh/release-name: "masakari" + meta.helm.sh/release-namespace: "openstack" spec: vhost: "masakari" # name of a vhost userReference: From 308ad9fbfd45dd1f6acb59408101881f1e339f8b Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Mon, 18 Aug 2025 09:49:28 -0500 Subject: [PATCH 59/97] fix: add waiting for pkg mgr to be available --- bootstrap.sh | 16 ++++++---- scripts/hyperconverged-lab.sh | 3 +- scripts/lib/functions.sh | 56 ++++++++++++++++++++++++++++++++++- 3 files changed, 68 insertions(+), 7 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index 891f6f03c..3c1d985e3 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -27,12 +27,18 @@ success "Environment variables:" env | grep -E '^(SUDO|RPC_|ANSIBLE_|GENESTACK_|K8S|CONTAINER_|OPENSTACK_|OSH_)' | sort -u success "Installing base packages (git):" -apt update -DEBIAN_FRONTEND=noninteractive \ - apt-get -o "Dpkg::Options::=--force-confdef" \ - -o "Dpkg::Options::=--force-confold" \ - -qy install git python3-pip python3-venv python3-dev jq build-essential > ~/genestack-base-package-install.log 2>&1 +if wait_for_package_manager_locks; then + echo "Package manager ready, proceeding with installations/updates." + apt update + DEBIAN_FRONTEND=noninteractive \ + apt-get -o "Dpkg::Options::=--force-confdef" \ + -o "Dpkg::Options::=--force-confold" \ + -qy install git python3-pip python3-venv python3-dev jq build-essential > ~/genestack-base-package-install.log 2>&1 +else + echo "Failed to acquire package manager lock, exiting." + exit 1 +fi if [ $? -gt 1 ]; then error "Check for ansible errors at ~/genestack-base-package-install.log" diff --git a/scripts/hyperconverged-lab.sh b/scripts/hyperconverged-lab.sh index c18c76772..eb2824b40 100755 --- a/scripts/hyperconverged-lab.sh +++ b/scripts/hyperconverged-lab.sh @@ -312,8 +312,9 @@ if [ "${HYPERCONVERGED_DEV:-false}" = "true" ]; then echo "HYPERCONVERGED_DEV is true, but we've failed to determine the base genestack directory" exit 1 fi + # NOTE: (brew) we are assuming an Ubunut (apt) based instance here ssh -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -t ${SSH_USERNAME}@${JUMP_HOST_VIP} \ - "timeout 1m bash -c 'while ! sudo apt update; do sleep 2; done' && sudo apt install -y rsync git" + "while sudo fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock >/dev/null 2>&1; do echo 'Waiting for apt locks to be released...'; sleep 5; done && sudo apt-get update && sudo apt install -y rsync git" echo "Copying the development source code to the jump host" rsync -az \ -e "ssh -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" \ diff --git a/scripts/lib/functions.sh b/scripts/lib/functions.sh index 0fe46c670..2cc0b9b10 100644 --- a/scripts/lib/functions.sh +++ b/scripts/lib/functions.sh @@ -24,8 +24,62 @@ sudo -l |grep -q NOPASSWD && SUDO_CMD="/usr/bin/sudo -n " test -f ~/.rackspace/datacenter && export RAX_DC="$(cat ~/.rackspace/datacenter |tr '[:upper:]' '[:lower:]')" test -f /etc/openstack_deploy/openstack_inventory.json && export RPC_CONFIG_IN_PLACE=true || export RPC_CONFIG_IN_PLACE=false +# Global functions + +wait_for_dnf_locks() { + local max_retries=180 # Maximum retries for dnf commands + local retry_delay=10 # Delay between retries in seconds + + for i in $(seq 1 $max_retries); do + if sudo dnf clean all && sudo dnf makecache; then + echo "dnf locks released. Proceeding." + return 0 # Indicate success + else + echo "dnf still locked. Retrying in $retry_delay seconds..." + sleep $retry_delay + fi + done + + echo "Error: Timed out after $max_retries retries waiting for dnf to become available." >&2 + return 1 # Indicate failure +} + +wait_for_apt_locks() { + local max_wait_time=180 # Maximum time to wait in seconds + local elapsed_time=0 + + while sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || \ + sudo fuser /var/lib/dpkg/lock >/dev/null 2>&1; do + + if (( elapsed_time >= max_wait_time )); then + echo "Error: Timed out waiting for apt locks to release." >&2 + return 1 # Indicate failure + fi + + echo "Waiting for apt locks to release..." + sleep 5 # Wait for 5 seconds before checking again + elapsed_time=$((elapsed_time + 5)) + done + echo "apt locks released. Proceeding." + return 0 # Indicate success +} + +wait_for_package_manager_locks() { + # Check if apt or dnf is the package manager + if command -v apt-get &>/dev/null; then + echo "Detected apt package manager." + wait_for_apt_locks + return $? + elif command -v dnf &>/dev/null; then + echo "Detected dnf package manager." + wait_for_dnf_locks + return $? + else + echo "Error: Neither apt nor dnf package manager found." >&2 + return 1 + fi +} - # Global functions function success { echo -e "\n\n\x1B[32m>> $1\x1B[39m" } From 22dd0961660bf26da3f5affd1512f9529961f01c Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Mon, 18 Aug 2025 10:33:46 -0500 Subject: [PATCH 60/97] fix: generalize pkg installation --- bootstrap.sh | 15 ++----- scripts/hyperconverged-lab.sh | 7 +-- scripts/lib/functions.sh | 84 ++++++++++++++++------------------- 3 files changed, 44 insertions(+), 62 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index 3c1d985e3..3db84047f 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -28,17 +28,10 @@ env | grep -E '^(SUDO|RPC_|ANSIBLE_|GENESTACK_|K8S|CONTAINER_|OPENSTACK_|OSH_)' success "Installing base packages (git):" -if wait_for_package_manager_locks; then - echo "Package manager ready, proceeding with installations/updates." - apt update - DEBIAN_FRONTEND=noninteractive \ - apt-get -o "Dpkg::Options::=--force-confdef" \ - -o "Dpkg::Options::=--force-confold" \ - -qy install git python3-pip python3-venv python3-dev jq build-essential > ~/genestack-base-package-install.log 2>&1 -else - echo "Failed to acquire package manager lock, exiting." - exit 1 -fi +# NOTE: (brew) This function will determine wether DNF or APT should be used to install +# packages and will install then. +# Package list found here: scripts/lib/funcitons.sh ['apt_packages', 'dnf_packages'] +wait_and_install_packages if [ $? -gt 1 ]; then error "Check for ansible errors at ~/genestack-base-package-install.log" diff --git a/scripts/hyperconverged-lab.sh b/scripts/hyperconverged-lab.sh index eb2824b40..8a4dccaa7 100755 --- a/scripts/hyperconverged-lab.sh +++ b/scripts/hyperconverged-lab.sh @@ -324,12 +324,9 @@ fi ssh -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -t ${SSH_USERNAME}@${JUMP_HOST_VIP} < /dev/null; then - echo "git could not be found, installing..." - sudo apt update && sudo apt install -y git -fi if [ ! -d "/opt/genestack" ]; then - sudo git clone --recurse-submodules -j4 https://github.com/rackerlabs/genestack /opt/genestack +# sudo git clone --recurse-submodules -j4 https://github.com/rackerlabs/genestack /opt/genestack + sudo git clone --recurse-submodules -j4 --branch add-wait-for-pkg-mgr https://github.com/rackerchris/genestack.git /opt/genestack else sudo git config --global --add safe.directory /opt/genestack pushd /opt/genestack diff --git a/scripts/lib/functions.sh b/scripts/lib/functions.sh index 2cc0b9b10..1bae62f53 100644 --- a/scripts/lib/functions.sh +++ b/scripts/lib/functions.sh @@ -25,59 +25,51 @@ test -f ~/.rackspace/datacenter && export RAX_DC="$(cat ~/.rackspace/datacenter test -f /etc/openstack_deploy/openstack_inventory.json && export RPC_CONFIG_IN_PLACE=true || export RPC_CONFIG_IN_PLACE=false # Global functions - -wait_for_dnf_locks() { - local max_retries=180 # Maximum retries for dnf commands - local retry_delay=10 # Delay between retries in seconds - - for i in $(seq 1 $max_retries); do - if sudo dnf clean all && sudo dnf makecache; then - echo "dnf locks released. Proceeding." - return 0 # Indicate success - else - echo "dnf still locked. Retrying in $retry_delay seconds..." - sleep $retry_delay - fi +# Function to wait for Apt and DNF locks, then install packages +wait_and_install_packages() { + local sleep_time=5 # Default sleep time between checks (in seconds) + local pkg_manager="" + local apt_packages=("python3-pip" "python3-venv" "python3-dev" "jq" "build-essential") + local dnf_packages=("npython3-pip" "python3-venv" "python3-dev" "jq" "build-essential") + + # Check for Apt locks + echo "Checking for Apt locks..." + while sudo fuser /var/lib/dpkg/lock /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || sudo fuser /var/cache/apt/archives/lock >/dev/null 2>&1; do + echo "Apt lock detected. Waiting for it to be released..." + sleep "$sleep_time" done - echo "Error: Timed out after $max_retries retries waiting for dnf to become available." >&2 - return 1 # Indicate failure -} - -wait_for_apt_locks() { - local max_wait_time=180 # Maximum time to wait in seconds - local elapsed_time=0 - - while sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || \ - sudo fuser /var/lib/dpkg/lock >/dev/null 2>&1; do - - if (( elapsed_time >= max_wait_time )); then - echo "Error: Timed out waiting for apt locks to release." >&2 - return 1 # Indicate failure - fi - - echo "Waiting for apt locks to release..." - sleep 5 # Wait for 5 seconds before checking again - elapsed_time=$((elapsed_time + 5)) + # Check for DNF process (indicating a DNF operation) + echo "Checking for DNF locks..." + while pgrep dnf >/dev/null; do + echo "DNF process detected. Waiting for it to finish..." + sleep "$sleep_time" done - echo "apt locks released. Proceeding." - return 0 # Indicate success -} -wait_for_package_manager_locks() { - # Check if apt or dnf is the package manager - if command -v apt-get &>/dev/null; then - echo "Detected apt package manager." - wait_for_apt_locks - return $? - elif command -v dnf &>/dev/null; then - echo "Detected dnf package manager." - wait_for_dnf_locks - return $? + echo "No package manager locks or active processes found. Proceeding with installation." + + # Detect package manager + if command -v apt >/dev/null 2>&1; then + pkg_manager="apt" + elif command -v dnf >/dev/null 2>&1; then + pkg_manager="dnf" else - echo "Error: Neither apt nor dnf package manager found." >&2 + echo "Error: Neither Apt nor DNF package manager found. Cannot install packages." return 1 fi + + # Install packages based on detected manager + if [[ "$pkg_manager" == "apt" ]]; then + echo "Detected Apt. Installing packages: ${apt_packages[@]}" + sudo apt update + sudo apt install -y "${apt_packages[@]}" # -y to auto-confirm installations + elif [[ "$pkg_manager" == "dnf" ]]; then + echo "Detected DNF. Installing packages: ${dnf_packages[@]}" + sudo dnf check-update # Checks for updates, but does not download or install packages + sudo dnf install -y "${dnf_packages[@]}" # -y to auto-confirm installations + fi + + echo "Package installation complete." } function success { From 16beaebe342627cc97813d496389c7a3df33a358 Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Mon, 18 Aug 2025 10:36:56 -0500 Subject: [PATCH 61/97] fix: output and bug fixes --- bootstrap.sh | 1 + scripts/lib/functions.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bootstrap.sh b/bootstrap.sh index 3db84047f..ede15d32c 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -31,6 +31,7 @@ success "Installing base packages (git):" # NOTE: (brew) This function will determine wether DNF or APT should be used to install # packages and will install then. # Package list found here: scripts/lib/funcitons.sh ['apt_packages', 'dnf_packages'] +echo "Starting package installation..." wait_and_install_packages if [ $? -gt 1 ]; then diff --git a/scripts/lib/functions.sh b/scripts/lib/functions.sh index 1bae62f53..a20afe9fb 100644 --- a/scripts/lib/functions.sh +++ b/scripts/lib/functions.sh @@ -30,7 +30,7 @@ wait_and_install_packages() { local sleep_time=5 # Default sleep time between checks (in seconds) local pkg_manager="" local apt_packages=("python3-pip" "python3-venv" "python3-dev" "jq" "build-essential") - local dnf_packages=("npython3-pip" "python3-venv" "python3-dev" "jq" "build-essential") + local dnf_packages=("python3-pip" "python3-venv" "python3-dev" "jq" "build-essential") # Check for Apt locks echo "Checking for Apt locks..." From 480fb6b884e5a92a4b848b1783da5b61216a86d6 Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Mon, 18 Aug 2025 10:51:49 -0500 Subject: [PATCH 62/97] feat: wait for cloud-init to finish --- scripts/lib/functions.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/lib/functions.sh b/scripts/lib/functions.sh index a20afe9fb..e6f7d8300 100644 --- a/scripts/lib/functions.sh +++ b/scripts/lib/functions.sh @@ -25,6 +25,17 @@ test -f ~/.rackspace/datacenter && export RAX_DC="$(cat ~/.rackspace/datacenter test -f /etc/openstack_deploy/openstack_inventory.json && export RPC_CONFIG_IN_PLACE=true || export RPC_CONFIG_IN_PLACE=false # Global functions +# Function to wait for cloud-init to finish. BLOCKING +wait_for_cloud_init() { + if command -v cloud-init &> /dev/null; then + cloud-init status --wait + return $? + else + echo "Error: cloud-init command not found." + return 3 + fi +} + # Function to wait for Apt and DNF locks, then install packages wait_and_install_packages() { local sleep_time=5 # Default sleep time between checks (in seconds) From 4caf2bea5b088d0978ccb795a6de3bdd1abdd76f Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Mon, 18 Aug 2025 10:58:48 -0500 Subject: [PATCH 63/97] feat: add wait-for-cloud-init --- bootstrap.sh | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index ede15d32c..0c2569c68 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -26,12 +26,24 @@ set -e success "Environment variables:" env | grep -E '^(SUDO|RPC_|ANSIBLE_|GENESTACK_|K8S|CONTAINER_|OPENSTACK_|OSH_)' | sort -u -success "Installing base packages (git):" +bash -# NOTE: (brew) This function will determine wether DNF or APT should be used to install -# packages and will install then. -# Package list found here: scripts/lib/funcitons.sh ['apt_packages', 'dnf_packages'] -echo "Starting package installation..." +echo "Waiting for cloud-init to finish..." +wait_for_cloud_init + +if [[ $? -eq 0 ]]; then + echo "Cloud-init completed successfully!" +elif [[ $? -eq 1 ]]; then + echo "Cloud-init crashed or experienced a serious issue." +elif [[ $? -eq 2 ]]; then + echo "Cloud-init completed with errors." +else + echo "Cloud-init command not found." +fi + +# NOTE: (brew) This function will determine wether DNF or APT should be used +# to install packages and will install then. +# Package: scripts/lib/funcitons.sh ['apt_packages', 'dnf_packages'] wait_and_install_packages if [ $? -gt 1 ]; then @@ -41,7 +53,7 @@ else fi # Install project dependencies -success "Installing genestack dependencies" +success "Configuring genestack directory and overrides directory structure:" test -L "$GENESTACK_CONFIG" 2>&1 || mkdir -p "${GENESTACK_CONFIG}" # Set config @@ -92,13 +104,13 @@ for service in "$base_source_dir"/*; do ln -s "$service" "$base_target_dir/$service_name" success "Created symlink for $service_name directly under $base_target_dir" else - message "Symlink for $service_name already exists directly under $base_target_dir" + message "Symlink for $service_name already exists directly under $base_target_dir." fi else if [ -d "$base_target_dir/$service_name" ]; then - message "$base_target_dir/$service_name already exists" + message "$base_target_dir/$service_name already exists." else - message "Creating $base_target_dir/$service_name" + message "Creating $base_target_dir/$service_name." mkdir -p "$base_target_dir/$service_name" fi for item in "$service"/*; do @@ -107,7 +119,7 @@ for service in "$base_source_dir"/*; do ln -s "$item" "$base_target_dir/$service_name/$item_name" success "Created symlink for $service_name/$item_name" else - message "Symlink for $service_name/$item_name already exists" + message "Symlink for $service_name/$item_name already exists." fi done fi @@ -175,7 +187,7 @@ done if [ ! -d "/etc/genestack/helm-configs/global_overrides" ]; then mkdir -p /etc/genestack/helm-configs/global_overrides - echo "Created /etc/genestack/helm-configs/global_overrides" + echo "Created /etc/genestack/helm-configs/global_overrides." else echo "/etc/genestack/helm-configs/global_overrides already exists, skipping creation." fi From 7eec68c9c327ed9e3e78af2976076ab65cc06326 Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Mon, 18 Aug 2025 11:09:39 -0500 Subject: [PATCH 64/97] fix: typo's and some errant bash command --- bootstrap.sh | 4 +--- scripts/lib/functions.sh | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index 0c2569c68..ba621705c 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -26,8 +26,6 @@ set -e success "Environment variables:" env | grep -E '^(SUDO|RPC_|ANSIBLE_|GENESTACK_|K8S|CONTAINER_|OPENSTACK_|OSH_)' | sort -u -bash - echo "Waiting for cloud-init to finish..." wait_for_cloud_init @@ -42,7 +40,7 @@ else fi # NOTE: (brew) This function will determine wether DNF or APT should be used -# to install packages and will install then. +# to install packages and will install them. # Package: scripts/lib/funcitons.sh ['apt_packages', 'dnf_packages'] wait_and_install_packages diff --git a/scripts/lib/functions.sh b/scripts/lib/functions.sh index e6f7d8300..79e9a4f3f 100644 --- a/scripts/lib/functions.sh +++ b/scripts/lib/functions.sh @@ -25,7 +25,8 @@ test -f ~/.rackspace/datacenter && export RAX_DC="$(cat ~/.rackspace/datacenter test -f /etc/openstack_deploy/openstack_inventory.json && export RPC_CONFIG_IN_PLACE=true || export RPC_CONFIG_IN_PLACE=false # Global functions -# Function to wait for cloud-init to finish. BLOCKING +# Function to wait for cloud-init to finish. +# BLOCKING if cloud-init is found and will retur exit code. wait_for_cloud_init() { if command -v cloud-init &> /dev/null; then cloud-init status --wait From 90fa9b051f31cbe832094dc988d9d726fc9ea8a4 Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Mon, 18 Aug 2025 12:28:40 -0500 Subject: [PATCH 65/97] fix: remove set -e --- bootstrap.sh | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index ba621705c..6b2a516d0 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -21,11 +21,12 @@ cd "${BASEDIR}" || error "Could not change to ${BASEDIR}" source scripts/lib/functions.sh -set -e +#set -e success "Environment variables:" env | grep -E '^(SUDO|RPC_|ANSIBLE_|GENESTACK_|K8S|CONTAINER_|OPENSTACK_|OSH_)' | sort -u +# Wait until cloud-init is finished before proceeding echo "Waiting for cloud-init to finish..." wait_for_cloud_init @@ -44,11 +45,11 @@ fi # Package: scripts/lib/funcitons.sh ['apt_packages', 'dnf_packages'] wait_and_install_packages -if [ $? -gt 1 ]; then - error "Check for ansible errors at ~/genestack-base-package-install.log" -else - success "Local base OS packages installed" -fi +#if [ $? -gt 1 ]; then +# error "Check for ansible errors at ~/genestack-base-package-install.log" +#else +# success "Local base OS packages installed." +#fi # Install project dependencies success "Configuring genestack directory and overrides directory structure:" @@ -70,14 +71,14 @@ test -d "$GENESTACK_CONFIG/gateway-api" || cp -a "${BASEDIR}/etc/gateway-api" "$ # Create venv and prepare Ansible python3 -m venv "${HOME}/.venvs/genestack" "${HOME}/.venvs/genestack/bin/pip" install pip --upgrade -source "${HOME}/.venvs/genestack/bin/activate" && success "Switched to venv ~/.venvs/genestack" -pip install -r "${BASEDIR}/requirements.txt" && success "Installed ansible package" +source "${HOME}/.venvs/genestack/bin/activate" && success "Switched to venv ~/.venvs/genestack." +pip install -r "${BASEDIR}/requirements.txt" && success "Installed ansible package." ansible-playbook "${BASEDIR}/scripts/get-ansible-collection-requirements.yml" \ -e collections_file="${ANSIBLE_COLLECTION_FILE}" \ -e user_collections_file="${USER_COLLECTION_FILE}" source "${BASEDIR}/scripts/genestack.rc" -success "Environment sourced per ${BASEDIR}/scripts/genestack.rc" +success "Environment sourced per ${BASEDIR}/scripts/genestack.rc." message "OpenStack Release: ${OPENSTACK_RELEASE}" message "Target OS Distro: ${CONTAINER_DISTRO_NAME}:${CONTAINER_DISTRO_VERSION}" @@ -100,7 +101,7 @@ for service in "$base_source_dir"/*; do # If no subdirectories, symlink the service directly under the target dir if [ ! -L "$base_target_dir/$service_name" ]; then ln -s "$service" "$base_target_dir/$service_name" - success "Created symlink for $service_name directly under $base_target_dir" + success "Created symlink for $service_name directly under $base_target_dir." else message "Symlink for $service_name already exists directly under $base_target_dir." fi @@ -115,7 +116,7 @@ for service in "$base_source_dir"/*; do item_name=$(basename "$item") if [ ! -L "$base_target_dir/$service_name/$item_name" ]; then ln -s "$item" "$base_target_dir/$service_name/$item_name" - success "Created symlink for $service_name/$item_name" + success "Created symlink for $service_name/$item_name." else message "Symlink for $service_name/$item_name already exists." fi @@ -147,12 +148,12 @@ for service in "$overlay_target_dir"/*; do if [ ! -d "$overlay_path" ]; then mkdir -p "$overlay_path" - success "Creating overlay path $overlay_path" + success "Creating overlay path $overlay_path." fi if [ ! -f "$overlay_path/kustomization.yaml" ]; then echo "$kustomization_content" > "$overlay_path/kustomization.yaml" - success "Created overlay and kustomization.yaml for $(basename "$service")" + success "Created overlay and kustomization.yaml for $(basename "$service")." else message "kustomization.yaml already exists for $(basename "$service"), skipping..." fi @@ -165,7 +166,7 @@ done if [ ! -d "/etc/genestack/helm-configs" ]; then mkdir -p /etc/genestack/helm-configs - success "Created /etc/genestack/helm-configs" + success "Created /etc/genestack/helm-configs." else message "/etc/genestack/helm-configs already exists, skipping creation." fi @@ -176,7 +177,7 @@ for src_dir in /opt/genestack/base-helm-configs/*; do dest_dir="/etc/genestack/helm-configs/$dir_name" if [ ! -d "$dest_dir" ]; then mkdir -p "$dest_dir" - success "Created $dest_dir" + success "Created $dest_dir." else message "$dest_dir already exists, skipping creation." fi @@ -193,7 +194,7 @@ fi # Copy manifests if it does not already exist if [ ! -d "/etc/genestack/manifests" ]; then cp -r /opt/genestack/manifests /etc/genestack/ - success "Copied manifests to /etc/genestack/" + success "Copied manifests to /etc/genestack/." else message "manifests already exists in /etc/genestack, skipping copy." fi From a64e853b25005ab92fd885db06bc96d5f822cdbe Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Mon, 18 Aug 2025 12:30:31 -0500 Subject: [PATCH 66/97] fix: move set e to after cloud-init and pkg functions --- bootstrap.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index 6b2a516d0..87f5a968b 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -21,8 +21,6 @@ cd "${BASEDIR}" || error "Could not change to ${BASEDIR}" source scripts/lib/functions.sh -#set -e - success "Environment variables:" env | grep -E '^(SUDO|RPC_|ANSIBLE_|GENESTACK_|K8S|CONTAINER_|OPENSTACK_|OSH_)' | sort -u @@ -45,11 +43,14 @@ fi # Package: scripts/lib/funcitons.sh ['apt_packages', 'dnf_packages'] wait_and_install_packages -#if [ $? -gt 1 ]; then -# error "Check for ansible errors at ~/genestack-base-package-install.log" -#else -# success "Local base OS packages installed." -#fi +if [ $? -gt 1 ]; then + error "Check for ansible errors at ~/genestack-base-package-install.log" +else + success "Local base OS packages installed." +fi + +# Set script to exit on any non-zero error code +set -e # Install project dependencies success "Configuring genestack directory and overrides directory structure:" From ca1be116d958090be7a050e1d26180f23b30dee7 Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Mon, 18 Aug 2025 12:38:03 -0500 Subject: [PATCH 67/97] fix: explictly disable exit on non-zero --- bootstrap.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bootstrap.sh b/bootstrap.sh index 87f5a968b..243ece7b7 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -24,6 +24,9 @@ source scripts/lib/functions.sh success "Environment variables:" env | grep -E '^(SUDO|RPC_|ANSIBLE_|GENESTACK_|K8S|CONTAINER_|OPENSTACK_|OSH_)' | sort -u +# Explictily do not exit script on non-zero returns +set +e + # Wait until cloud-init is finished before proceeding echo "Waiting for cloud-init to finish..." wait_for_cloud_init From 4fdea3b3ec97a64908fc38e2214170142acf0ec0 Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Mon, 18 Aug 2025 13:54:18 -0500 Subject: [PATCH 68/97] fix: move back to cloning from genestack --- scripts/hyperconverged-lab.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/hyperconverged-lab.sh b/scripts/hyperconverged-lab.sh index 8a4dccaa7..890adceed 100755 --- a/scripts/hyperconverged-lab.sh +++ b/scripts/hyperconverged-lab.sh @@ -325,7 +325,6 @@ fi ssh -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -t ${SSH_USERNAME}@${JUMP_HOST_VIP} < Date: Mon, 18 Aug 2025 14:01:29 -0500 Subject: [PATCH 69/97] fix: correctly, for real this time, clone from genestack repo --- scripts/hyperconverged-lab.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/hyperconverged-lab.sh b/scripts/hyperconverged-lab.sh index 890adceed..3433a8e69 100755 --- a/scripts/hyperconverged-lab.sh +++ b/scripts/hyperconverged-lab.sh @@ -325,7 +325,7 @@ fi ssh -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -t ${SSH_USERNAME}@${JUMP_HOST_VIP} < Date: Tue, 19 Aug 2025 10:29:28 -0500 Subject: [PATCH 70/97] fix: Resolve Octavia server QueuePool limit exceeded errors --- base-helm-configs/octavia/octavia-helm-overrides.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/base-helm-configs/octavia/octavia-helm-overrides.yaml b/base-helm-configs/octavia/octavia-helm-overrides.yaml index 34d7309d4..3b0adf587 100644 --- a/base-helm-configs/octavia/octavia-helm-overrides.yaml +++ b/base-helm-configs/octavia/octavia-helm-overrides.yaml @@ -75,6 +75,8 @@ conf: use_db_reconnect: true pool_timeout: 60 max_retries: -1 + max_overflow: 60 + max_pool_size: 30 driver_agent: enabled_provider_agents: ovn glance: From 5366e4f53bb856417fbbc8be54e68e1ea00065ca Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Tue, 19 Aug 2025 17:00:56 -0500 Subject: [PATCH 71/97] fix: remove neutron-rpc-server --- .../neutron/neutron-helm-overrides.yaml | 1 + .../neutron/base/hpa-neutron-rpc-server.yaml | 26 ------------------- .../neutron/base/kustomization.yaml | 1 - 3 files changed, 1 insertion(+), 27 deletions(-) delete mode 100644 base-kustomize/neutron/base/hpa-neutron-rpc-server.yaml diff --git a/base-helm-configs/neutron/neutron-helm-overrides.yaml b/base-helm-configs/neutron/neutron-helm-overrides.yaml index 30e5a03d9..38489d2a0 100644 --- a/base-helm-configs/neutron/neutron-helm-overrides.yaml +++ b/base-helm-configs/neutron/neutron-helm-overrides.yaml @@ -330,3 +330,4 @@ manifests: secret_ingress_tls: false secret_rabbitmq: false service_ingress_server: false + deployment_rpc_server: false diff --git a/base-kustomize/neutron/base/hpa-neutron-rpc-server.yaml b/base-kustomize/neutron/base/hpa-neutron-rpc-server.yaml deleted file mode 100644 index 0ac8c8d1e..000000000 --- a/base-kustomize/neutron/base/hpa-neutron-rpc-server.yaml +++ /dev/null @@ -1,26 +0,0 @@ ---- -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: neutron-rpc-server - namespace: openstack -spec: - maxReplicas: 9 - minReplicas: 2 - metrics: - - resource: - name: cpu - target: - averageUtilization: 80 - type: Utilization - type: Resource - - resource: - name: memory - target: - averageUtilization: 80 - type: Utilization - type: Resource - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: neutron-rpc-server diff --git a/base-kustomize/neutron/base/kustomization.yaml b/base-kustomize/neutron/base/kustomization.yaml index 677186a8f..98617a65a 100644 --- a/base-kustomize/neutron/base/kustomization.yaml +++ b/base-kustomize/neutron/base/kustomization.yaml @@ -5,7 +5,6 @@ resources: - neutron-rabbitmq-queue.yaml - all.yaml - hpa-neutron-server.yaml - - hpa-neutron-rpc-server.yaml - policies.yaml patches: From ce37d1fa33f0f7fb90d55bf62b84f5c4d8e2fa05 Mon Sep 17 00:00:00 2001 From: Chris Blumentritt Date: Thu, 21 Aug 2025 12:17:41 -0500 Subject: [PATCH 72/97] fix: Specify tag for fluentbit image (#1137) Signed-off-by: Chris Blumentritt --- .original-images.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.original-images.json b/.original-images.json index 39b005732..74abf9c4a 100644 --- a/.original-images.json +++ b/.original-images.json @@ -1,5 +1,5 @@ [ - "cr.fluentbit.io/fluent/fluent-bit", + "cr.fluentbit.io/fluent/fluent-bit:4.0.7", "docker.io/docker:17.07.0", "docker.io/kolla/ubuntu-source-nova-compute-ironic:master", "docker.io/library/postgres:14.5", From 2d3ecb0e56106676532a682716881209ab476182 Mon Sep 17 00:00:00 2001 From: James Denton Date: Thu, 21 Aug 2025 12:18:04 -0500 Subject: [PATCH 73/97] Fixed base helm overrides to allow successful Ironic deploy (#1123) * Renamed file and fixed base helm overrides to allow successful Ironic deployment * Updated image location for conductor and pxe --- .../ironic/ironic-helm-overrides.yaml | 40 ++++++++++++++----- .../ironic/base/hpa-ironic-api.yaml | 2 +- ...nductor.yaml => hpa-ironic-conductor.yaml} | 0 3 files changed, 32 insertions(+), 10 deletions(-) rename base-kustomize/ironic/base/{hpa-iconic-conductor.yaml => hpa-ironic-conductor.yaml} (100%) diff --git a/base-helm-configs/ironic/ironic-helm-overrides.yaml b/base-helm-configs/ironic/ironic-helm-overrides.yaml index 0b6e181b5..3012526a1 100644 --- a/base-helm-configs/ironic/ironic-helm-overrides.yaml +++ b/base-helm-configs/ironic/ironic-helm-overrides.yaml @@ -18,8 +18,8 @@ images: ks_endpoints: "ghcr.io/rackerlabs/genestack-images/heat:2024.1-latest" rabbit_init: null ironic_api: "ghcr.io/rackerlabs/genestack-images/ironic-api:2024.1-latest" - ironic_conductor: "ghcr.io/rackerlabs/genestack-images/ironic-api:2024.1-latest" - ironic_pxe: "ghcr.io/rackerlabs/genestack-images/ironic-api:2024.1-latest" + ironic_conductor: "ghcr.io/rackerlabs/genestack-images/ironic-conductor:2024.1-latest" + ironic_pxe: "ghcr.io/rackerlabs/genestack-images/ironic-pxe:2024.1-latest" ironic_pxe_init: "ghcr.io/rackerlabs/genestack-images/ironic-api:2024.1-latest" ironic_pxe_http: "docker.io/nginx:1.13.3" # Retained from openstack-helm default ironic_inspector: "ghcr.io/rackerlabs/genestack-images/ironic-inspector:2024.1-latest" @@ -112,6 +112,32 @@ network: neutron_subnet_alloc_end: 172.24.6.200 neutron_subnet_dns_nameserver: 8.8.8.8 # Aligned with Neutron's OVN DNS +bootstrap: + image: + enabled: true + openstack: + enabled: true + ks_user: ironic + # NOTE: if source_base is null the source will be used as is + source_base: http://tarballs.openstack.org/ironic-python-agent/tinyipa/files + structured: + ironic-agent.initramfs: + source: tinyipa-stable-2024.1.gz + disk_format: ari + container_format: ari + ironic-agent.kernel: + source: tinyipa-stable-2024.1.vmlinuz + disk_format: aki + container_format: aki + network: + enabled: false + openstack: + enabled: false + object_store: + enabled: false + openstack: + enabled: false + dependencies: static: api: @@ -119,8 +145,6 @@ dependencies: - ironic-db-sync - ironic-ks-user - ironic-ks-endpoints - - ironic-manage-cleaning-network - - ironic-rabbit-init services: - endpoint: internal service: oslo_db @@ -133,8 +157,6 @@ dependencies: - ironic-db-sync - ironic-ks-user - ironic-ks-endpoints - - ironic-manage-cleaning-network - - ironic-rabbit-init services: - endpoint: internal service: oslo_db @@ -233,12 +255,12 @@ manifests: ingress_api: false job_bootstrap: false job_db_drop: false - job_db_init: true + job_db_init: false job_db_sync: true job_ks_endpoints: true job_ks_service: true job_ks_user: true - job_manage_cleaning_network: true - job_rabbit_init: true + job_manage_cleaning_network: false + job_rabbit_init: false service_ingress_api: false statefulset_conductor: true diff --git a/base-kustomize/ironic/base/hpa-ironic-api.yaml b/base-kustomize/ironic/base/hpa-ironic-api.yaml index dfe154f48..f3b498042 100644 --- a/base-kustomize/ironic/base/hpa-ironic-api.yaml +++ b/base-kustomize/ironic/base/hpa-ironic-api.yaml @@ -4,7 +4,7 @@ kind: HorizontalPodAutoscaler metadata: name: ironic-api namespace: openstack - spec: +spec: maxReplicas: 9 minReplicas: 2 metrics: diff --git a/base-kustomize/ironic/base/hpa-iconic-conductor.yaml b/base-kustomize/ironic/base/hpa-ironic-conductor.yaml similarity index 100% rename from base-kustomize/ironic/base/hpa-iconic-conductor.yaml rename to base-kustomize/ironic/base/hpa-ironic-conductor.yaml From f3c1cfff6c6883483870226f282d5de775d64f37 Mon Sep 17 00:00:00 2001 From: Chris Blumentritt Date: Thu, 21 Aug 2025 17:42:44 -0500 Subject: [PATCH 74/97] fix: Set install of fluentbit chart version to 0.52.0 (#1140) Signed-off-by: Chris Blumentritt --- bin/install-fluentbit.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/install-fluentbit.sh b/bin/install-fluentbit.sh index bbb2c759d..8bf442464 100755 --- a/bin/install-fluentbit.sh +++ b/bin/install-fluentbit.sh @@ -3,8 +3,12 @@ GLOBAL_OVERRIDES_DIR="/etc/genestack/helm-configs/global_overrides" SERVICE_CONFIG_DIR="/etc/genestack/helm-configs/fluentbit" +FLUENTBIT_CHART_VERSION="0.52.0" -HELM_CMD="helm upgrade --install --namespace fluentbit --create-namespace fluentbit fluent/fluent-bit" +HELM_CMD="helm upgrade --install \ + --version $FLUENTBIT_CHART_VERSION \ + --namespace fluentbit \ + --create-namespace fluentbit fluent/fluent-bit" HELM_CMD+=" -f /opt/genestack/base-helm-configs/fluentbit/fluentbit-helm-overrides.yaml" From e78882453f67f418b8b6d32f035e0e2831d41b1e Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Thu, 21 Aug 2025 20:46:30 -0500 Subject: [PATCH 75/97] fix: ensure no-tenant network leakage (#1139) The following configuration will ensure that we're not permitting LLDP traffic over tenant networks. In the before now we were not controlling the installation of LLDP, however, we were requiring it for QC tools. This change ensures that we're controlling the installation of LLDP and the config, which ensures that we're not leaking host information to tenants of the cloud platform. Signed-off-by: Kevin Carter --- ansible/roles/host_setup/handlers/main.yml | 6 ++++++ ansible/roles/host_setup/tasks/main.yml | 11 +++++++++++ ansible/roles/host_setup/vars/debian.yml | 3 ++- ansible/roles/host_setup/vars/ubuntu.yml | 1 + 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/ansible/roles/host_setup/handlers/main.yml b/ansible/roles/host_setup/handlers/main.yml index 0c814b8c7..bcb81b0b1 100644 --- a/ansible/roles/host_setup/handlers/main.yml +++ b/ansible/roles/host_setup/handlers/main.yml @@ -65,3 +65,9 @@ ansible.builtin.apt: update_cache: yes cache_valid_time: 600 + +- name: Restart lldpd + ansible.builtin.systemd: + name: "lldpd.service" + state: "restarted" + enabled: true diff --git a/ansible/roles/host_setup/tasks/main.yml b/ansible/roles/host_setup/tasks/main.yml index 91fcafeef..1b54c5ee5 100644 --- a/ansible/roles/host_setup/tasks/main.yml +++ b/ansible/roles/host_setup/tasks/main.yml @@ -137,6 +137,17 @@ retries: 5 delay: 2 +# NOTE(cloudnull): This configuration will ensure that LLDP is working on all interfaces +# except our overlay and tenant networks. +- name: Create base LLDPD configuration + ansible.builtin.copy: + content: | + DAEMON_ARGS="-c -I *,!tap*,!ovn*,!genev*,!mirror*,!o-hm*" + dest: /etc/default/lldpd + mode: "0644" + notify: + - Restart lldpd + - name: Ensure timesyncd is running ansible.builtin.service: name: systemd-timesyncd diff --git a/ansible/roles/host_setup/vars/debian.yml b/ansible/roles/host_setup/vars/debian.yml index bb7a70fba..e1adb329a 100644 --- a/ansible/roles/host_setup/vars/debian.yml +++ b/ansible/roles/host_setup/vars/debian.yml @@ -41,8 +41,8 @@ _host_distro_packages: - apt-utils - bridge-utils - cgroup-tools - - curl - cryptsetup + - curl - dmeventd - dstat - ebtables @@ -50,6 +50,7 @@ _host_distro_packages: - iptables - irqbalance - libkmod2 + - lldpd - lsscsi - lvm2 - nfs-client diff --git a/ansible/roles/host_setup/vars/ubuntu.yml b/ansible/roles/host_setup/vars/ubuntu.yml index 9f19fb40a..429cc103c 100644 --- a/ansible/roles/host_setup/vars/ubuntu.yml +++ b/ansible/roles/host_setup/vars/ubuntu.yml @@ -49,6 +49,7 @@ _host_distro_packages: - iptables - irqbalance - libkmod2 + - lldpd - lsscsi - lvm2 - nfs-client From 66759f28f33d307701a11ba01a1d8d087f6e7177 Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Fri, 22 Aug 2025 10:46:57 -0500 Subject: [PATCH 76/97] fix: specify ovn gateway doc update Where to place your ovn gateways needs to be an informed decision. With our current example, this would enable all tenant/compute nodes to act as ovn gateways. This is not ideal in most deployments so change the example to be more explicit and add a NOTE. --- docs/infrastructure-ovn-setup.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/infrastructure-ovn-setup.md b/docs/infrastructure-ovn-setup.md index 7229669e7..b66b27357 100644 --- a/docs/infrastructure-ovn-setup.md +++ b/docs/infrastructure-ovn-setup.md @@ -99,11 +99,12 @@ kubectl annotate \ ### Set `ovn.openstack.org/gateway` Define where the gateways nodes will reside. There are many ways to run this, some like every compute node to be a gateway, some like dedicated gateway hardware. Either way you will need at least one gateway node within your environment. +NOTE: In the following example, we will apply the 'ovn.openstack.org/gateway' to dedicated network nodes. You will want to change the node filter to the specific nodes you wish to use as ovn gateway nodes. ``` shell kubectl annotate \ nodes \ - -l openstack-network-node=enabled \ + -l $(kubectl get nodes | awk '/network/ {print $1}') \ ovn.openstack.org/gateway='enabled' ``` From bfc629e9ed440d0d43d5ed760bca9f17aabf203c Mon Sep 17 00:00:00 2001 From: Jake Briggs Date: Fri, 22 Aug 2025 16:28:15 -0500 Subject: [PATCH 77/97] FIX: Misc Fixes (#1142) * FIX: Cleaning up old cruff * FIX: Modified to be a bit simpler to use * Documentation on how to use the Yamel editor * Modified bootstrap to install Yamel editor * Created longhorn multi attach storage class * Added documentation for longhorn general-multi-attach * emoved Sealed Secrets. Until they work * Made setup-openstack smart. I hope. * FIX: added section to create config file for setup and make logic case insensitive * FIX: Black did not like it --- bin/setup-openstack.sh | 77 ++++++++++++++----- bootstrap.sh | 3 + cve/filter.py | 36 --------- cve/requirements.txt | 0 docs/storage-longhorn.md | 18 +++++ ...orn-general-multi-attach-storageclass.yaml | 17 ++++ mkdocs.yml | 1 - requirements.txt | 1 + scripts/hyperconverged-lab.sh | 21 +++++ yaml-editor/README.txt | 7 ++ yaml-editor/ye | 20 +++-- 11 files changed, 136 insertions(+), 65 deletions(-) delete mode 100644 cve/filter.py delete mode 100644 cve/requirements.txt create mode 100644 manifests/longhorn/longhorn-general-multi-attach-storageclass.yaml create mode 100644 yaml-editor/README.txt diff --git a/bin/setup-openstack.sh b/bin/setup-openstack.sh index 60fe22c00..c8d14dc2a 100755 --- a/bin/setup-openstack.sh +++ b/bin/setup-openstack.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -e -export GENESTACK_INSTALL_TELEMETRY=${GENESTACK_INSTALL_TELEMETRY:-false} - # Track the PIDs of the services deploying in parallel pids=() @@ -11,7 +9,7 @@ function runTrackErator() { pids+=($!) } -function waitErator () { +function waitErator() { for pid in ${pids[*]}; do if ! timeout --preserve-status --verbose 30m tail --pid=${pid} -f /dev/null; then echo "==== PROCESS TIMEOUT =====================================" @@ -23,28 +21,65 @@ function waitErator () { done } +# Function to prompt user for component installation +prompt_component() { + local component=$1 + local prompt=$2 + read -p "Install ${prompt}? (y/n): " answer + if [[ "$answer" =~ ^[Yy]$ ]]; then + echo " ${component}: true" >> /etc/genestack/openstack-components.yaml + else + echo " ${component}: false" >> /etc/genestack/openstack-components.yaml + fi +} + +# Function to check if a component is set to true in the YAML file +is_component_enabled() { + local component=$1 + grep -qi "^[[:space:]]*${component}:[[:space:]]*true" "$CONFIG_FILE" +} + +# Check for YAML file and create if it doesn't exist +CONFIG_FILE="/etc/genestack/openstack-components.yaml" +if [ ! -f "$CONFIG_FILE" ]; then + echo "Configuration file $CONFIG_FILE not found. Creating it..." + cat > "$CONFIG_FILE" << EOF +components: + keystone: true +EOF + prompt_component "glance" "Glance (Image Service)" + prompt_component "heat" "Heat (Orchestration)" + prompt_component "barbican" "Barbican (Key Manager)" + prompt_component "cinder" "Cinder (Block Storage)" + prompt_component "placement" "Placement" + prompt_component "nova" "Nova (Compute)" + prompt_component "neutron" "Neutron (Networking)" + prompt_component "magnum" "Magnum (Container Orchestration)" + prompt_component "octavia" "Octavia (Load Balancer)" + prompt_component "masakari" "Masakari (Instance High Availability)" + prompt_component "ceilometer" "Ceilometer (Telemetry)" + prompt_component "gnocchi" "Gnocchi (Time Series Database)" + prompt_component "skyline" "Skyline (Dashboard)" +fi + # Block on Keystone /opt/genestack/bin/install-keystone.sh -# Run the rest of the services in parallel -runTrackErator /opt/genestack/bin/install-glance.sh -runTrackErator /opt/genestack/bin/install-heat.sh -runTrackErator /opt/genestack/bin/install-barbican.sh -runTrackErator /opt/genestack/bin/install-cinder.sh -runTrackErator /opt/genestack/bin/install-placement.sh -runTrackErator /opt/genestack/bin/install-nova.sh -runTrackErator /opt/genestack/bin/install-neutron.sh -runTrackErator /opt/genestack/bin/install-magnum.sh -runTrackErator /opt/genestack/bin/install-octavia.sh -runTrackErator /opt/genestack/bin/install-masakari.sh - -# Install telemetry services -if [ "${GENESTACK_INSTALL_TELEMETRY}" = true ]; then - runTrackErator /opt/genestack/bin/install-ceilometer.sh - runTrackErator /opt/genestack/bin/install-gnocchi.sh -fi +# Run selected services in parallel +is_component_enabled "glance" && runTrackErator /opt/genestack/bin/install-glance.sh +is_component_enabled "heat" && runTrackErator /opt/genestack/bin/install-heat.sh +is_component_enabled "barbican" && runTrackErator /opt/genestack/bin/install-barbican.sh +is_component_enabled "cinder" && runTrackErator /opt/genestack/bin/install-cinder.sh +is_component_enabled "placement" && runTrackErator /opt/genestack/bin/install-placement.sh +is_component_enabled "nova" && runTrackErator /opt/genestack/bin/install-nova.sh +is_component_enabled "neutron" && runTrackErator /opt/genestack/bin/install-neutron.sh +is_component_enabled "magnum" && runTrackErator /opt/genestack/bin/install-magnum.sh +is_component_enabled "octavia" && runTrackErator /opt/genestack/bin/install-octavia.sh +is_component_enabled "masakari" && runTrackErator /opt/genestack/bin/install-masakari.sh +is_component_enabled "ceilometer" && runTrackErator /opt/genestack/bin/install-ceilometer.sh +is_component_enabled "gnocchi" && runTrackErator /opt/genestack/bin/install-gnocchi.sh waitErator # Install skyline after all services are up -/opt/genestack/bin/install-skyline.sh +is_component_enabled "skyline" && /opt/genestack/bin/install-skyline.sh diff --git a/bootstrap.sh b/bootstrap.sh index 243ece7b7..42d48e034 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -203,4 +203,7 @@ else message "manifests already exists in /etc/genestack, skipping copy." fi +# Copy yaml editor to /usr/local/bin +cp /opt/genestack/yaml-editor/ye /usr/local/bin/ye + echo diff --git a/cve/filter.py b/cve/filter.py deleted file mode 100644 index a9f4f3ffe..000000000 --- a/cve/filter.py +++ /dev/null @@ -1,36 +0,0 @@ -import json - -try: - with open("installed.json") as f: - # Only store package names for comparison, ignore versions - installed = {pkg["name"].lower() for pkg in json.load(f)} -except (json.JSONDecodeError, FileNotFoundError): - installed = set() - -print("Installed packages:") -print("\n".join(sorted(installed)) if installed else "No installed packages found") -print("\n" + "=" * 50 + "\n") - -with open("cve/requirements.txt") as f: - requirements = [ - line.strip() for line in f if line.strip() and not line.startswith("#") - ] - -print("Requirements from requirements.txt:") -print("\n".join(requirements) if requirements else "No requirements found") -print("\n" + "=" * 50 + "\n") - -filtered = [] -for req in requirements: - # Only get package name for comparison - pkg_name = req.split("==")[0].strip().lower() - if pkg_name in installed: - # Add the full original requirement (including version) - filtered.append(req) - -print("Filtered requirements (matching installed packages):") -print("\n".join(filtered) if filtered else "No matching packages found") -print("\n" + "=" * 50 + "\n") - -with open("filtered-requirements.txt", "w") as f: - f.write("\n".join(filtered)) diff --git a/cve/requirements.txt b/cve/requirements.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/docs/storage-longhorn.md b/docs/storage-longhorn.md index face94141..2527f67d9 100644 --- a/docs/storage-longhorn.md +++ b/docs/storage-longhorn.md @@ -179,6 +179,24 @@ kubectl apply -f /etc/genestack/manifests/longhorn/longhorn-general-storageclass With the `general` StorageClass in place, you can now create PVCs that reference it to dynamically provision Longhorn volumes with the desired settings. +### General Multi Attach StorageClass + +For the purposes of Genestack, it is recommended that you create the `general-multi-attach` StorageClass to avoid deployment confusion. + +!!! example "longhorn-general-multi-attach-storageclass.yaml" + + ``` yaml + --8<-- "manifests/longhorn/longhorn-general-multi-attach-storageclass.yaml" + ``` + +Apply the general-multi-attach storage class manifest to create the StorageClass. + +``` shell +kubectl apply -f /etc/genestack/manifests/longhorn/longhorn-general-multi-attach-storageclass.yaml +``` + +With the `general-multi-attach` StorageClass in place, you can now create PVCs that reference it to dynamically provision Longhorn volumes with the desired settings. + ### (Optional) Create an Encrypted StorageClass If you want to enable data encryption, you can create an encrypted StorageClass. This feature encrypts the data at rest within the Longhorn volumes. Opting for the diff --git a/manifests/longhorn/longhorn-general-multi-attach-storageclass.yaml b/manifests/longhorn/longhorn-general-multi-attach-storageclass.yaml new file mode 100644 index 000000000..c2dab8863 --- /dev/null +++ b/manifests/longhorn/longhorn-general-multi-attach-storageclass.yaml @@ -0,0 +1,17 @@ +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: general-multi-attach + annotations: + storageclass.kubernetes.io/is-default-class: "false" +provisioner: driver.longhorn.io +allowVolumeExpansion: true +reclaimPolicy: Delete +volumeBindingMode: Immediate +parameters: + numberOfReplicas: "2" # This example uses a single replica, but you can adjust this value as needed + dataLocality: "best-effort" + staleReplicaTimeout: "2880" + fromBackup: "" + fsType: "ext4" diff --git a/mkdocs.yml b/mkdocs.yml index 17c1f4f93..1343564a3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -193,7 +193,6 @@ nav: - OVN: infrastructure-ovn-setup.md - FluentBit: infrastructure-fluentbit.md - Loki: infrastructure-loki.md - - Sealed Secrets: infrastructure-sealed-secrets.md - OpenStack: - openstack-overview.md - OpenStack Services: diff --git a/requirements.txt b/requirements.txt index 7a1538248..53111561e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ ruamel.yaml.clib==0.2.8 kubernetes>=24.2.0 openstacksdk>=1.0.0 python-openstackclient==7.4.0 +dictdiffer==0.9.0 diff --git a/scripts/hyperconverged-lab.sh b/scripts/hyperconverged-lab.sh index 3433a8e69..af0a83323 100755 --- a/scripts/hyperconverged-lab.sh +++ b/scripts/hyperconverged-lab.sh @@ -921,6 +921,27 @@ pushd /opt/kube-plugins popd EOC +echo "Creating config for setup-openstack.sh" + +if [ ! -f "/etc/genestack/openstack-components.yaml" ]; then +cat > /etc/genestack/openstack-components.yaml < +Example: ye nova diff --git a/yaml-editor/ye b/yaml-editor/ye index 25da8b22f..e84aab563 100755 --- a/yaml-editor/ye +++ b/yaml-editor/ye @@ -69,17 +69,23 @@ def compute_patch(base, edited): def main(): - if len(sys.argv) != 3: - print("Usage: {} ".format(sys.argv[0])) + if len(sys.argv) != 2: + print("Usage: {} ".format(sys.argv[0])) print( - "ye - (YamlEditor) will launch an editor to edit the values in base_yaml_file. If the override_yaml_file exists, " - "it will read its values in and will overlay the valies on top of base." - "After editing, the edited values will be saved to override_yaml_file and base will remain the same." + "ye - (YamlEditor) will launch an editor to edit the values for the specified service's base YAML file. " + "The base file is located at /opt/genestack/base-helm-configs//-helm-overrides.yaml. " + "If the override file exists at /etc/genestack/helm-configs//-helm-overrides.yaml, " + "its values will be overlaid on the base. After editing, the changes will be saved to the override file." + "Created by Jake Briggs Rackspace" ) sys.exit(1) - base_filename = sys.argv[1] - override_filename = sys.argv[2] + service_name = sys.argv[1] + base_filename = f"/opt/genestack/base-helm-configs/{service_name}/{service_name}-helm-overrides.yaml" + override_filename = ( + f"/etc/genestack/helm-configs/{service_name}/{service_name}-helm-overrides.yaml" + ) + base_data = load_yaml_file(base_filename) if not base_data: print(f"Error: Base file '{base_filename}' not found or is empty.") From 04997d26d4243d494d965bbad6ac6a3b51ac368a Mon Sep 17 00:00:00 2001 From: phillip-toohill Date: Mon, 25 Aug 2025 13:00:56 -0500 Subject: [PATCH 78/97] fix: Update Octavia preconf for various fixes like gateway disabling --- ansible/playbooks/octavia-preconf-main.yaml | 2 ++ .../octavia_amphora_keypair_image_flavor.yml | 2 +- .../tasks/octavia_lb_net_setup.yml | 18 ++++++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/ansible/playbooks/octavia-preconf-main.yaml b/ansible/playbooks/octavia-preconf-main.yaml index 618a1c7ba..5edde4b1e 100644 --- a/ansible/playbooks/octavia-preconf-main.yaml +++ b/ansible/playbooks/octavia-preconf-main.yaml @@ -14,6 +14,8 @@ octavia_os_identity_api_version: 3 octavia_os_auth_version: 3 octavia_nova_endpoint_type: "{{ octavia_os_endpoint_type }}" + interface: "{{ interface }}" + endpoint_type: "{{ endpoint_type }}" environment: OS_ENDPOINT_TYPE: "{{ octavia_os_endpoint_type }}" OS_INTERFACE: "{{ octavia_os_interface}}" diff --git a/ansible/roles/octavia_preconf/tasks/octavia_amphora_keypair_image_flavor.yml b/ansible/roles/octavia_preconf/tasks/octavia_amphora_keypair_image_flavor.yml index 2dfcdaa14..f6bba54bc 100644 --- a/ansible/roles/octavia_preconf/tasks/octavia_amphora_keypair_image_flavor.yml +++ b/ansible/roles/octavia_preconf/tasks/octavia_amphora_keypair_image_flavor.yml @@ -49,7 +49,7 @@ filename: /tmp/test-only-amphora-x64-haproxy-ubuntu-{{ amphora_image_version }}.qcow2 container_format: bare disk_format: qcow2 - visibility: public + visibility: private protected: true tags: - amphora diff --git a/ansible/roles/octavia_preconf/tasks/octavia_lb_net_setup.yml b/ansible/roles/octavia_preconf/tasks/octavia_lb_net_setup.yml index 679cdf46e..e1313b44f 100644 --- a/ansible/roles/octavia_preconf/tasks/octavia_lb_net_setup.yml +++ b/ansible/roles/octavia_preconf/tasks/octavia_lb_net_setup.yml @@ -30,3 +30,21 @@ until: create_lb_mgmt_subnet is success retries: 5 delay: 5 + +# Bit of a hack to ensure gateway is disabled until ansible-playbook releases the fix.. +# disabling the gateway works on 'updates' as-is but not on creates yet.. +- name: Update subnet for lb-mgmt-net + openstack.cloud.subnet: + name: lb-mgmt-subnet + state: present + enable_dhcp: true + cidr: "{{ lb_mgmt_subnet_cidr }}" + allocation_pool_start: "{{ lb_mgmt_subnet_pool_start }}" + allocation_pool_end: "{{ lb_mgmt_subnet_pool_end }}" + disable_gateway_ip: true + network_name: lb-mgmt-net + interface: public + register: create_lb_mgmt_subnet + until: create_lb_mgmt_subnet is success + retries: 5 + delay: 5 From ea4752bdd98be79216d8e331e56e50a94659581d Mon Sep 17 00:00:00 2001 From: phillip-toohill Date: Mon, 25 Aug 2025 13:05:09 -0500 Subject: [PATCH 79/97] Fixing yamllint error --- .../tasks/octavia_amphora_keypair_image_flavor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/octavia_preconf/tasks/octavia_amphora_keypair_image_flavor.yml b/ansible/roles/octavia_preconf/tasks/octavia_amphora_keypair_image_flavor.yml index f6bba54bc..7ade3dc41 100644 --- a/ansible/roles/octavia_preconf/tasks/octavia_amphora_keypair_image_flavor.yml +++ b/ansible/roles/octavia_preconf/tasks/octavia_amphora_keypair_image_flavor.yml @@ -52,7 +52,7 @@ visibility: private protected: true tags: - - amphora + - amphora interface: public register: push_amphora_image until: push_amphora_image is success From c46186cf5f1494b7a523adadff4e40567d08e4b2 Mon Sep 17 00:00:00 2001 From: "phillip.toohill" Date: Mon, 25 Aug 2025 15:04:24 -0500 Subject: [PATCH 80/97] chore: Updating hyperconverged-lab.sh for octavia preconf and k9s (#1143) * chore: Updating hyperconverged-lab.sh for octavia preconf and k9s * fix: Updating hyperconverged lab script missing conditional closer * fix: Updating hyperconverged lab script for correct config location * fix: Updating hyperconverged lab with proper openstack command usage --- scripts/hyperconverged-lab.sh | 40 +++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/scripts/hyperconverged-lab.sh b/scripts/hyperconverged-lab.sh index af0a83323..e00301da0 100755 --- a/scripts/hyperconverged-lab.sh +++ b/scripts/hyperconverged-lab.sh @@ -922,7 +922,8 @@ popd EOC echo "Creating config for setup-openstack.sh" - +ssh -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -t ${SSH_USERNAME}@${JUMP_HOST_VIP} < /etc/genestack/openstack-components.yaml < Date: Tue, 26 Aug 2025 10:59:46 -0500 Subject: [PATCH 81/97] fix: Removing rabbit init deps from ceilometer overrides --- base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml b/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml index c74741aec..8808dbd6f 100644 --- a/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml +++ b/base-helm-configs/ceilometer/ceilometer-helm-overrides.yaml @@ -1697,7 +1697,6 @@ dependencies: central: jobs: - ceilometer-db-sync - - ceilometer-rabbit-init - ceilometer-ks-user services: - endpoint: internal @@ -1707,7 +1706,6 @@ dependencies: ipmi: jobs: - ceilometer-db-sync - - ceilometer-rabbit-init - ceilometer-ks-user services: - endpoint: internal @@ -1717,7 +1715,6 @@ dependencies: compute: jobs: - ceilometer-db-sync - - ceilometer-rabbit-init - ceilometer-ks-user services: - endpoint: internal @@ -1730,7 +1727,6 @@ dependencies: notification: jobs: - ceilometer-db-sync - - ceilometer-rabbit-init - ceilometer-ks-user services: - endpoint: internal @@ -1818,6 +1814,7 @@ manifests: deployment_collector: false ingress_api: false # using gnocchi so no db init + job_rabbit_init: false job_db_init: false job_db_init_mongodb: false job_ks_endpoints: false From 78591310fa7c731b9b310c6650daaf65bd5456f6 Mon Sep 17 00:00:00 2001 From: phillip-toohill Date: Mon, 25 Aug 2025 17:53:38 -0500 Subject: [PATCH 82/97] fix: Updating hyperconverged lab script and preconf for setting override file --- ansible/playbooks/octavia-preconf-main.yaml | 1 + scripts/hyperconverged-lab.sh | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ansible/playbooks/octavia-preconf-main.yaml b/ansible/playbooks/octavia-preconf-main.yaml index 5edde4b1e..aae0778e2 100644 --- a/ansible/playbooks/octavia-preconf-main.yaml +++ b/ansible/playbooks/octavia-preconf-main.yaml @@ -14,6 +14,7 @@ octavia_os_identity_api_version: 3 octavia_os_auth_version: 3 octavia_nova_endpoint_type: "{{ octavia_os_endpoint_type }}" + octavia_helm_values_file: "{{ octavia_helm_file}}" interface: "{{ interface }}" endpoint_type: "{{ endpoint_type }}" environment: diff --git a/scripts/hyperconverged-lab.sh b/scripts/hyperconverged-lab.sh index e00301da0..1b3a30df0 100755 --- a/scripts/hyperconverged-lab.sh +++ b/scripts/hyperconverged-lab.sh @@ -1012,18 +1012,25 @@ echo "Installing Octavia preconf" ssh -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -t ${SSH_USERNAME}@${JUMP_HOST_VIP} << 'EOC' set -e +if [ ! -f ~/.config/openstack ]; then + sudo cp -r /root/.config/openstack ~/.config/ +fi + source ~/.venvs/genestack/bin/activate +OCTAVIA_HELM_FILE=/tmp/octavia_helm_overrides.yaml + ANSIBLE_SSH_PIPELINING=0 ansible-playbook /opt/genestack/ansible/playbooks/octavia-preconf-main.yaml \ -e octavia_os_password=$(/usr/local/bin/kubectl get secrets keystone-admin -n openstack -o jsonpath='{.data.password}' | base64 -d) \ -e octavia_os_region_name=$(sudo ~/.venvs/genestack/bin/openstack --os-cloud=default endpoint list --service keystone --interface internal -c Region -f value) \ -e octavia_os_auth_url=$(sudo ~/.venvs/genestack/bin/openstack --os-cloud=default endpoint list --service keystone --interface internal -c URL -f value) \ -e octavia_os_endpoint_type=internal \ + -e octavia_helm_file=$OCTAVIA_HELM_FILE \ -e interface=internal \ -e endpoint_type=internal echo "Installing Octavia" -sudo /opt/genestack/bin/install-octavia.sh +sudo /opt/genestack/bin/install-octavia.sh -f $OCTAVIA_HELM_FILE EOC { cat | tee /tmp/output.txt; } < Date: Mon, 25 Aug 2025 21:18:15 -0500 Subject: [PATCH 83/97] fixing white space issue --- ansible/playbooks/octavia-preconf-main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/octavia-preconf-main.yaml b/ansible/playbooks/octavia-preconf-main.yaml index aae0778e2..e0bc5076d 100644 --- a/ansible/playbooks/octavia-preconf-main.yaml +++ b/ansible/playbooks/octavia-preconf-main.yaml @@ -14,7 +14,7 @@ octavia_os_identity_api_version: 3 octavia_os_auth_version: 3 octavia_nova_endpoint_type: "{{ octavia_os_endpoint_type }}" - octavia_helm_values_file: "{{ octavia_helm_file}}" + octavia_helm_values_file: "{{ octavia_helm_file }}" interface: "{{ interface }}" endpoint_type: "{{ endpoint_type }}" environment: From 567831e9fe883dfb29aac68816f37a943c08d223 Mon Sep 17 00:00:00 2001 From: Dan With Date: Wed, 27 Aug 2025 05:11:58 -0500 Subject: [PATCH 84/97] Fix: Ansible playbook host_setup role raid_cli_tools.yml task (#1151) This fixes recursion error in the vars file corresponding to the raid_cli_tools.yml task and associated problems encountered with HP ssacli keyrings and apt installation. TESTED in IAD3 on nodes in maintenance. Previous tests did not pick up this problem because another version of the package had already been installed and the task was nearly wholly skipped. --- .../roles/host_setup/tasks/raid_cli_tools.yml | 72 +++++++++++++++---- .../roles/host_setup/vars/raid-cli-tools.yml | 33 --------- .../roles/host_setup/vars/raid_cli_tools.yml | 47 ++++++++++++ 3 files changed, 106 insertions(+), 46 deletions(-) delete mode 100644 ansible/roles/host_setup/vars/raid-cli-tools.yml create mode 100644 ansible/roles/host_setup/vars/raid_cli_tools.yml diff --git a/ansible/roles/host_setup/tasks/raid_cli_tools.yml b/ansible/roles/host_setup/tasks/raid_cli_tools.yml index fdc4c082e..d8e265154 100644 --- a/ansible/roles/host_setup/tasks/raid_cli_tools.yml +++ b/ansible/roles/host_setup/tasks/raid_cli_tools.yml @@ -3,6 +3,10 @@ ansible.builtin.package_facts: manager: auto +- name: Load additional variables + include_vars: + file: "{{ role_path }}/vars/raid_cli_tools.yml" + - name: Install PERCCLI command line tool for DELL servers when: - ansible_system_vendor | lower == "dell inc." @@ -64,27 +68,69 @@ - (ansible_system_vendor | lower == "hp" or ansible_system_vendor | lower == "hpe") - "'ssacli' not in ansible_facts.packages" block: + - name: Ensure keyring dir and staging dir exist + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: '0755' + loop: + - "{{ hp_tools.apt.gpg_keyring | dirname }}" + - "/usr/share/keyrings/hpe-keys.d" + become: true - name: Download HP tools apt keys ansible.builtin.uri: url: "{{ item.url }}" dest: "{{ item.download_file }}" - with_items: "{{ hp_tools.repo_keys }}" - when: - - hp_tools.repo_keys is defined + status_code: [200, 304] # accept Not Modified + mode: "0644" + loop: "{{ hp_tools.repo_keys }}" + when: hp_tools.repo_keys is defined register: download_keys_url - until: download_keys_url is success + changed_when: download_keys_url.status == 200 # 304 -> not changed retries: 2 delay: 4 - - name: Add HP tools apt keys to gpg + until: download_keys_url is success + - name: Remove existing consolidated HPE keyring (if corrupted) + ansible.builtin.file: + path: "{{ hp_tools.apt.gpg_keyring }}" + state: absent + become: true + - name: Ensure staging dir for ASCII keys exists + ansible.builtin.file: + path: "/tmp/hpe-keys.asc.d" + state: directory + mode: "0755" + - name: Stage downloaded ASCII keys + ansible.builtin.copy: + src: "{{ item.download_file }}" + dest: "/tmp/hpe-keys.asc.d/{{ item.url | basename }}" + remote_src: true + mode: "0644" + loop: "{{ hp_tools.repo_keys }}" + when: hp_tools.repo_keys is defined + - name: Concatenate ASCII keys + ansible.builtin.assemble: + src: "/tmp/hpe-keys.asc.d" + dest: "/tmp/hpe-keys.asc" + regexp: ".*\\.pub$" + mode: "0644" + - name: Build consolidated HPE keyring from ASCII bundle ansible.builtin.command: - cmd: "cat {{ item.download_file }} | gpg --dearmor | sudo tee -a {{ hp_tools.apt.gpg_keyring }} > /dev/null" - with_items: "{{ hp_tools.repo_keys }}" - when: - - hp_tools.repo_keys is defined - register: add_keys_url - until: add_keys_url is success - retries: 2 - delay: 2 + cmd: >- + gpg --batch --yes --dearmor + --output '{{ hp_tools.apt.gpg_keyring }}' + '/tmp/hpe-keys.asc' + register: dearmor_out + changed_when: dearmor_out.rc == 0 + become: true + - name: Remove ASCII staging files + ansible.builtin.file: + path: "/tmp/hpe-keys.asc.d" + state: absent + - name: Remove ASCII bundle + ansible.builtin.file: + path: "/tmp/hpe-keys.asc" + state: absent - name: Add HP tools MCP apt repositories ansible.builtin.apt_repository: repo: "{{ hp_tools.apt.deb_repo }}" diff --git a/ansible/roles/host_setup/vars/raid-cli-tools.yml b/ansible/roles/host_setup/vars/raid-cli-tools.yml deleted file mode 100644 index ac8d87a39..000000000 --- a/ansible/roles/host_setup/vars/raid-cli-tools.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -hp_tools: - sdr_url: "https://downloads.linux.hpe.com/SDR" - repo_keys: - - {url: "{{hp_tools.sdr_url}}/hpPublicKey2048_key1.pub", download_file: "/tmp/hpPublicKey2048_key1.pub"} - - {url: "{{hp_tools.sdr_url}}/hpePublicKey2048_key1.pub", download_file: "/tmp/hpePublicKey2048_key1.pub"} - - {url: "{{hp_tools.sdr_url}}/hpePublicKey2048_key2.pub", download_file: "/tmp/hpePublicKey2048_key2.pub"} - apt: - mcp_version: "current" - gpg_keyring: "/usr/share/keyrings/hpePublicKey.gpg" - repo_str: "[signed-by={{hp_tools.apt.gpg_keyring}}] https://downloads.linux.hpe.com/SDR/repo/mcp" - deb_repo: "deb {{hp_tools.apt.repo_str}} {{ansible_lsb.codename}}/{{hp_tools.apt.mcp_version}} non-free" - deb_src_repo: "deb-src {{hp_tools.apt.repo_str}} {{ansible_lsb.codename}}/{{hp_tools.apt.mcp_version}} non-free" - state: "present" - repo_list_file: "mcp" - rpm: - url: "https://downloads.linux.hpe.com/repo/mcp" - dist: "{{ansible_distribution}}" - dist_ver: "{{ansible_distribution_version}}" - arch: "{{ansible_architecture}}" - filename: "mcp" - package: ssacli - -dell_tools: - file_tar_gz: "PERCCLI_7.2616.0_Linux.tar.gz" - url: "https://dl.dell.com/FOLDER11213122M/1/{{dell_tools.file_tar_gz}}" - tmp_dir: "/tmp/perccli" - download_path: "{{dell_tools.tmp_dir}}/{{dell_tools.file_tar_gz}}" - deb_file: "{{dell_tools.tmp_dir}}/perccli_007.2616.0000.0000_all.deb" - rpm_file: "{{dell_tools.tmp_dir}}/perccli-007.2616.0000.0000-1.noarch.rpm" - perccli_path: "/opt/MegaRAID/perccli/perccli64" - symlink_path: "/usr/sbin/perccli" - symlink_64_path: "/usr/sbin/perccli64" diff --git a/ansible/roles/host_setup/vars/raid_cli_tools.yml b/ansible/roles/host_setup/vars/raid_cli_tools.yml new file mode 100644 index 000000000..6a9f81bcc --- /dev/null +++ b/ansible/roles/host_setup/vars/raid_cli_tools.yml @@ -0,0 +1,47 @@ +--- +# --------- HPE / HP --------- +hpe_sdr_url: "https://downloads.linux.hpe.com/SDR" +hpe_gpg_keyring: "/usr/share/keyrings/hpePublicKey.gpg" +hpe_repo_base: "https://downloads.linux.hpe.com/SDR/repo/mcp" +hpe_mcp_version: "current" + +hpe_repo_str: "[signed-by={{ hpe_gpg_keyring }}] {{ hpe_repo_base }}" + +hpe_repo_keys: + - {url: "{{ hpe_sdr_url }}/hpPublicKey2048_key1.pub", download_file: "/tmp/hpPublicKey2048_key1.pub"} + - {url: "{{ hpe_sdr_url }}/hpePublicKey2048_key1.pub", download_file: "/tmp/hpePublicKey2048_key1.pub"} + - {url: "{{ hpe_sdr_url }}/hpePublicKey2048_key2.pub", download_file: "/tmp/hpePublicKey2048_key2.pub"} + +hp_tools: + sdr_url: "{{ hpe_sdr_url }}" + repo_keys: "{{ hpe_repo_keys }}" + apt: + mcp_version: "{{ hpe_mcp_version }}" + gpg_keyring: "{{ hpe_gpg_keyring }}" + repo_str: "{{ hpe_repo_str }}" + deb_repo: "deb {{ hpe_repo_str }} {{ ansible_lsb.codename }}/{{ hpe_mcp_version }} non-free" + deb_src_repo: "deb-src {{ hpe_repo_str }} {{ ansible_lsb.codename }}/{{ hpe_mcp_version }} non-free" + state: "present" + repo_list_file: "mcp" + rpm: + url: "https://downloads.linux.hpe.com/repo/mcp" + dist: "{{ ansible_distribution }}" + dist_ver: "{{ ansible_distribution_version }}" + arch: "{{ ansible_architecture }}" + filename: "mcp" + package: "ssacli" + +# --------- Dell --------- +dell_perc_file_tar_gz: "PERCCLI_7.2616.0_Linux.tar.gz" +dell_tmp_dir: "/tmp/perccli" + +dell_tools: + file_tar_gz: "{{ dell_perc_file_tar_gz }}" + url: "https://dl.dell.com/FOLDER11213122M/1/{{ dell_perc_file_tar_gz }}" + tmp_dir: "{{ dell_tmp_dir }}" + download_path: "{{ dell_tmp_dir }}/{{ dell_perc_file_tar_gz }}" + deb_file: "{{ dell_tmp_dir }}/perccli_007.2616.0000.0000_all.deb" + rpm_file: "{{ dell_tmp_dir }}/perccli-007.2616.0000.0000-1.noarch.rpm" + perccli_path: "/opt/MegaRAID/perccli/perccli64" + symlink_path: "/usr/sbin/perccli" + symlink_64_path: "/usr/sbin/perccli64" From 1dad25added2dea26c1a374108dbf82ea785e616 Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Wed, 27 Aug 2025 10:07:00 -0500 Subject: [PATCH 85/97] fix: standardize EOC in script (#1152) fix: ensure newline at end of script --- scripts/hyperconverged-lab.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/hyperconverged-lab.sh b/scripts/hyperconverged-lab.sh index 1b3a30df0..b3b598353 100755 --- a/scripts/hyperconverged-lab.sh +++ b/scripts/hyperconverged-lab.sh @@ -994,7 +994,7 @@ HERE EOC echo "Installing k9s" -ssh -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -t ${SSH_USERNAME}@${JUMP_HOST_VIP} << 'EOC' +ssh -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -t ${SSH_USERNAME}@${JUMP_HOST_VIP} < Date: Wed, 27 Aug 2025 11:52:33 -0500 Subject: [PATCH 86/97] feat: Adding octavia worker port creation in octavia preconf (#1150) * feat: Adding octavia worker port creation in octavia preconf updated versions of octavia helm chart require ports created for the worker nodes as well as the health managers. I've simply duplicated the port creation as this may be something we yank out if/when octavia helm decides to make it optional. * fix port yaml naming * Add missing secgrp name --- .../roles/octavia_preconf/defaults/main.yml | 1 + .../files/create_worker_ports.sh | 39 +++++++++++++++++++ ansible/roles/octavia_preconf/tasks/main.yml | 5 +++ .../tasks/octavia_sec_group.yml | 24 ++++++++++++ .../tasks/octavia_worker_ports.yml | 22 +++++++++++ 5 files changed, 91 insertions(+) create mode 100755 ansible/roles/octavia_preconf/files/create_worker_ports.sh create mode 100644 ansible/roles/octavia_preconf/tasks/octavia_worker_ports.yml diff --git a/ansible/roles/octavia_preconf/defaults/main.yml b/ansible/roles/octavia_preconf/defaults/main.yml index ff2e3b6bb..670b020a8 100644 --- a/ansible/roles/octavia_preconf/defaults/main.yml +++ b/ansible/roles/octavia_preconf/defaults/main.yml @@ -26,6 +26,7 @@ lb_mgmt_subnet_gateway: '172.16.29.1' amphora_icmp_enabled: true amphora_ssh_enabled: true lb_health_mgr_secgrp_name: "lb-health-mgr-secgroup" +lb_worker_secgrp_name: "lb-worker-secgroup" lb_mgmt_secgrp_name: "lb-mgmt-secgroup" # these are the defaults for the flavor, image and ssh keypair diff --git a/ansible/roles/octavia_preconf/files/create_worker_ports.sh b/ansible/roles/octavia_preconf/files/create_worker_ports.sh new file mode 100755 index 000000000..d4cb7149c --- /dev/null +++ b/ansible/roles/octavia_preconf/files/create_worker_ports.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# We need to create the ports with shell scripts +# the ansible module currently doesn't provide +# --host argument + +set -xe + +# Obtain the network_id and secgroup_id from and +# cloud name from ansible task +NET_ID=$1 +SECGRP_ID=$2 +CLOUD_NAME=$3 + +export OS_CLOUD=$CLOUD_NAME + +# Obtain the list of kubernetes nodes with +# "openstack-control-plane=enabled" label +CONTROLLER_IP_PORT_LIST=() +CTRLS=$(kubectl get nodes -l openstack-control-plane=enabled -o name | awk -F"/" '{print $2}') +for node in $CTRLS +do + node_short=$(echo "$node" | awk -F"." '{print $1}') + PORTNAME=octavia-worker-port-$node_short + if ! PORT_DATA=$(openstack port show "$PORTNAME" -c fixed_ips -f json); then + PORT_DATA=$(openstack port create "$PORTNAME" --security-group "$SECGRP_ID" \ + --device-owner Octavia:worker \ + --host="$node" \ + --network "$NET_ID" \ + -c fixed_ips \ + -f json) + fi + for IP in $(echo "$PORT_DATA" | awk 'BEGIN { FS = "\"" } /ip_address/ { print $(NF - 1) }'); do + CONTROLLER_IP_PORT_LIST+=("$IP:5555") + done +done + +readarray -t sorted < <(for item in "${CONTROLLER_IP_PORT_LIST[@]}"; do echo "${item}"; done | sort) +echo $(IFS=,; echo "${sorted[*]}") > /tmp/octavia_worker_controller_ip_port_list diff --git a/ansible/roles/octavia_preconf/tasks/main.yml b/ansible/roles/octavia_preconf/tasks/main.yml index 05ba0195f..afe6911d3 100644 --- a/ansible/roles/octavia_preconf/tasks/main.yml +++ b/ansible/roles/octavia_preconf/tasks/main.yml @@ -21,6 +21,11 @@ tags: - always +- name: import tasks to create worker ports + import_tasks: octavia_worker_ports.yml + tags: + - always + - name: import tasks to create amphora image, flavor and ssh keypair import_tasks: octavia_amphora_keypair_image_flavor.yml tags: diff --git a/ansible/roles/octavia_preconf/tasks/octavia_sec_group.yml b/ansible/roles/octavia_preconf/tasks/octavia_sec_group.yml index d7180176f..3ed5c0070 100644 --- a/ansible/roles/octavia_preconf/tasks/octavia_sec_group.yml +++ b/ansible/roles/octavia_preconf/tasks/octavia_sec_group.yml @@ -81,3 +81,27 @@ until: lb_health_mgr_secgroup_r1 is success retries: 5 delay: 5 + +- name: Create security group for worker ports + openstack.cloud.security_group: + name: "{{ lb_worker_secgrp_name }}" + state: present + description: "security group for worker ports" + interface: public + register: create_lb_worker_secgroup + until: create_lb_worker_secgroup is success + retries: 5 + delay: 5 + +- name: Create Security group rule to allow traffic on port 5555 for worker + openstack.cloud.security_group_rule: + security_group: "{{ create_lb_worker_secgroup.security_group.id }}" + state: present + protocol: udp + port_range_min: 5555 + port_range_max: 5555 + interface: public + register: lb_worker_secgroup_r1 + until: lb_worker_secgroup_r1 is success + retries: 5 + delay: 5 diff --git a/ansible/roles/octavia_preconf/tasks/octavia_worker_ports.yml b/ansible/roles/octavia_preconf/tasks/octavia_worker_ports.yml new file mode 100644 index 000000000..96d92524f --- /dev/null +++ b/ansible/roles/octavia_preconf/tasks/octavia_worker_ports.yml @@ -0,0 +1,22 @@ +--- +# These are the tasks for creating health_mgr +# ports for octavia; the ports are created with +# a shell script as the ansible modules currently +# don't support all the required params for creating +# ports +- name: Obtain the UUID of the lb-mgmt-net + openstack.cloud.networks_info: + name: lb-mgmt-net + interface: public + register: lb_mgmt_info + +- name: Obtain the UUID of the worker secgroup + openstack.cloud.security_group_info: + name: "{{ lb_worker_secgrp_name }}" + interface: public + register: lb_worker_secgrp_info + +- name: run the shell script to create worker ports if required + script: + cmd: create_worker_ports.sh {{ lb_mgmt_info.networks[0].id }} {{ lb_worker_secgrp_info.security_groups[0].id }} {{ lookup('env', 'OS_CLOUD') | default('openstack_helm') }} + creates: /tmp/octavia_worker_controller_ip_port_list From 09d96fd247f7f9bc0030d4ee00ee83a705bc4d86 Mon Sep 17 00:00:00 2001 From: Chris Breu Date: Wed, 27 Aug 2025 14:29:59 -0500 Subject: [PATCH 87/97] Revert "fix: standardize EOC in script (#1152)" This reverts commit 1dad25added2dea26c1a374108dbf82ea785e616. --- scripts/hyperconverged-lab.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/hyperconverged-lab.sh b/scripts/hyperconverged-lab.sh index b3b598353..1b3a30df0 100755 --- a/scripts/hyperconverged-lab.sh +++ b/scripts/hyperconverged-lab.sh @@ -994,7 +994,7 @@ HERE EOC echo "Installing k9s" -ssh -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -t ${SSH_USERNAME}@${JUMP_HOST_VIP} < Date: Wed, 27 Aug 2025 19:14:45 +0000 Subject: [PATCH 88/97] adjust masakari-monitors genestack image repo link --- base-helm-configs/masakari/masakari-helm-overrides.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base-helm-configs/masakari/masakari-helm-overrides.yaml b/base-helm-configs/masakari/masakari-helm-overrides.yaml index 78315026f..61fc9948d 100644 --- a/base-helm-configs/masakari/masakari-helm-overrides.yaml +++ b/base-helm-configs/masakari/masakari-helm-overrides.yaml @@ -23,8 +23,8 @@ images: masakari_engine: ghcr.io/rackerlabs/genestack-images/masakari:2024.1-latest # TEMP HOST-MONITOR IMAGE TO FIX: https://review.opendev.org/c/openstack/masakari-monitors/+/951336 masakari_host_monitor: kernelpanic53/rackerlabs-masakari-monitors:zhmarvi-ubuntu_jammy_v1.0 - masakari_process_monitor: ghcr.io/rackerlabs/genestack-images/masakari:2024.1-latest - masakari_instance_monitor: ghcr.io/rackerlabs/genestack-images/masakari:2024.1-latest + masakari_process_monitor: ghcr.io/rackerlabs/genestack-images/masakari-monitors:2024.1-latest + masakari_instance_monitor: ghcr.io/rackerlabs/genestack-images/masakari-monitors:2024.1-latest rabbit_init: docker.io/rabbitmq:3.13-management dep_check: ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest pull_policy: "IfNotPresent" From e37f1497380c622563173b300d8da668c32c50e0 Mon Sep 17 00:00:00 2001 From: dwith Date: Wed, 27 Aug 2025 15:24:20 -0500 Subject: [PATCH 89/97] Fix: Ansible host-setup and deploy-cinder-netapp-volumes-reference Added remote_src to host-setup so that gunzip/unarchive works properly. deploy-cinder-netapp-volumes-reference now addresses different versions of python rather than hardcoding for site-packages path. Tested in SJC after deploy to prove remediation of the problems. --- ...eploy-cinder-netapp-volumes-reference.yaml | 22 ++++++++++++++++--- .../roles/host_setup/tasks/raid_cli_tools.yml | 3 +++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/ansible/playbooks/deploy-cinder-netapp-volumes-reference.yaml b/ansible/playbooks/deploy-cinder-netapp-volumes-reference.yaml index cf91d3434..4c1414ac0 100644 --- a/ansible/playbooks/deploy-cinder-netapp-volumes-reference.yaml +++ b/ansible/playbooks/deploy-cinder-netapp-volumes-reference.yaml @@ -8,6 +8,7 @@ cinder_storage_network_interface: ansible_br_storage cinder_storage_network_interface_secondary: ansible_br_storage_secondary cinder_backend_name: "block-ha-performance-at-rest-encrypted,block-ha-standard-at-rest-encrypted,block-ha-performance-end-to-end-encrypted,block-ha-standard-end-to-end-encrypted" + virtualenv_path: "/opt/cinder" storage_network_multipath: false enable_netapp_ssl: false netapp_cert_src_dir: "/opt/genestack/ansible/playbooks/templates/" @@ -116,9 +117,24 @@ - "git+https://github.com/openstack/cinder@stable/{{ cinder_release }}" - "git+https://github.com/rackerlabs/cinder-rxt.git" state: present - virtualenv: /opt/cinder + virtualenv: "{{ virtualenv_path }}" virtualenv_command: python3 -m venv + - name: "Get Python site-packages path from virtualenv" + command: "{{ virtualenv_path }}/bin/python -c 'import site; print(site.getsitepackages()[0])'" + register: venv_site + changed_when: false + + - name: "Normalize site-packages path" + set_fact: + venv_site_packages: "{{ venv_site.stdout | trim }}" + + - name: "Ensure site-packages exists" + file: + path: "{{ venv_site_packages }}" + state: directory + when: venv_site_packages != "" + - name: Install eventlet SSL patch ansible.builtin.copy: src: "{{ item.src }}" @@ -128,10 +144,10 @@ mode: "{{ item.mode | default('0644') }}" loop: - src: "{{ playbook_dir }}/templates/zzz_eventlet_ssl_patch.pth" - dest: /opt/cinder/lib/python3.12/site-packages/zzz_eventlet_ssl_patch.pth + dest: "{{ venv_site_packages }}/zzz_eventlet_ssl_patch.pth" mode: "0644" - src: "{{ playbook_dir }}/templates/eventlet_ssl_patch.py" - dest: /opt/cinder/lib/python3.12/site-packages/eventlet_ssl_patch.py + dest: "{{ venv_site_packages }}/eventlet_ssl_patch.py" mode: "0644" - name: Create the cinder system user diff --git a/ansible/roles/host_setup/tasks/raid_cli_tools.yml b/ansible/roles/host_setup/tasks/raid_cli_tools.yml index d8e265154..34dd68921 100644 --- a/ansible/roles/host_setup/tasks/raid_cli_tools.yml +++ b/ansible/roles/host_setup/tasks/raid_cli_tools.yml @@ -23,10 +23,13 @@ http_agent: Chrome/1337 validate_certs: false dest: "{{ dell_tools.download_path }}" + status_code: [200, 304] + mode: '0755' - name: Extract PERCCLI tar.gz ansible.builtin.unarchive: src: "{{ dell_tools.download_path }}" dest: "{{ dell_tools.tmp_dir }}" + remote_src: true - name: Install perccli APT when: ansible_os_family | lower == "debian" ansible.builtin.apt: From 11a59c29be90f35745ec5678276c4a190a2e4723 Mon Sep 17 00:00:00 2001 From: James Denton Date: Thu, 28 Aug 2025 10:51:38 -0500 Subject: [PATCH 90/97] Set TMP directory to use pod /tmp; Enabled neutron network interface (#1144) --- base-helm-configs/ironic/ironic-helm-overrides.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base-helm-configs/ironic/ironic-helm-overrides.yaml b/base-helm-configs/ironic/ironic-helm-overrides.yaml index 3012526a1..97a599842 100644 --- a/base-helm-configs/ironic/ironic-helm-overrides.yaml +++ b/base-helm-configs/ironic/ironic-helm-overrides.yaml @@ -32,7 +32,7 @@ conf: ironic: DEFAULT: log_config_append: /etc/ironic/logging.conf - tempdir: /var/lib/openstack-helm/tmp # Matches openstack-helm default + tempdir: /tmp default_deploy_interface: "direct" default_inspect_interface: "inspector" default_network_interface: "neutron" @@ -41,6 +41,7 @@ conf: enabled_deploy_interfaces: "direct,ramdisk" enabled_inspect_interfaces: "inspector,no-inspect" enabled_management_interfaces: "ipmitool,redfish" + enabled_network_interfaces: "flat,neutron" enabled_power_interfaces: "ipmitool,redfish" enabled_raid_interfaces: "no-raid" database: From 599ac8d6e4f55e11d93bc8032ca7044e9ab64e75 Mon Sep 17 00:00:00 2001 From: Jake Briggs Date: Tue, 2 Sep 2025 12:53:39 -0500 Subject: [PATCH 91/97] Fix: Added docs for upgrades (#1160) --- docs/2024.1-to-2025.1.md | 101 +++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 2 files changed, 102 insertions(+) create mode 100644 docs/2024.1-to-2025.1.md diff --git a/docs/2024.1-to-2025.1.md b/docs/2024.1-to-2025.1.md new file mode 100644 index 000000000..47eff7c33 --- /dev/null +++ b/docs/2024.1-to-2025.1.md @@ -0,0 +1,101 @@ +# Upgrading Genestack from OpenStack 2024.1 (Caracal) to 2025.1 (Epoxy) + +This guide outlines the process for upgrading a Genestack deployment from OpenStack 2024.1 (Caracal) to 2025.1 (Epoxy). Genestack leverages OpenStack-Helm charts for deployment and management, making upgrades primarily a matter of updating the underlying charts and configurations via Git and Helm. + +OpenStack 2025.1 (Epoxy) is a Skip Level Upgrade Release Process (SLURP) release, which supports direct upgrades from the previous SLURP release (Caracal), skipping the intermediate 2024.2 (Dalmatian) release if desired. This simplifies the upgrade path for stable environments. + +## Prerequisites + +- A full backup of your current deployment, including databases, configurations. +- Familiarity with the Genestack installation process and operator documentation. +- Minimum downtime window planned, as some services will require restarts. +- Verify that no active jobs or migrations are running that could conflict with the upgrade. + +## Upgrade Steps + +The upgrade process is similar to a fresh installation but focuses on updating charts and applying revisions. Perform these steps on the management node. + +1. **Navigate to the Genestack Directory:** + + ```bash + cd /opt/genestack + ``` + +2. **Update the Git Repository:** + - Fetch the latest changes: + + ```bash + git fetch origin + git tag + git checkout + ``` + - If there is no release tag for Epoxy then you will need to update the images manually in the overrides + + ``` + *** Example *** + keystone-helm-overrides.yaml + + images: + tags: + bootstrap: ghcr.io/aedan/genestack-images/heat:2025.1-1750442748 + db_drop: ghcr.io/aedan/genestack-images/heat:2025.1-1750442748 + db_init: ghcr.io/aedan/genestack-images/heat:2025.1-1750442748 + keystone_api: ghcr.io/aedan/genestack-images/keystone:2025.1-1750442703 + keystone_credential_cleanup: ghcr.io/aedan/genestack-images/heat:2025.1-1750442748 + keystone_credential_rotate: ghcr.io/aedan/genestack-images/keystone:2025.1-1750442703 + keystone_credential_setup: ghcr.io/aedan/genestack-images/keystone:2025.1-1750442703 + keystone_db_sync: ghcr.io/aedan/genestack-images/keystone:2025.1-1750442703 + keystone_domain_manage: ghcr.io/aedan/genestack-images/keystone:2025.1-1750442703 + keystone_fernet_rotate: ghcr.io/aedan/genestack-images/keystone:2025.1-1750442703 + keystone_fernet_setup: ghcr.io/aedan/genestack-images/keystone:2025.1-1750442703 + ks_user: ghcr.io/aedan/genestack-images/heat:2025.1-1750442748 +``` + +4. **Handle Pre-Upgrade Cleanups:** + - Clean up Nova jobs to avoid conflicts: + + ```bash + kubectl --namespace openstack delete jobs $(kubectl --namespace openstack get jobs --no-headers -o custom-columns=":metadata.name" | grep nova) + ``` + +5. **Apply Updated Helm Charts:** + - Re-run the Helm deployments for OpenStack components, following your original installation guide. + + ```bash + /opt/genestack/setup-openstack.sh + ``` + - Or apply in sequence: Keystone, Glance, Nova, Neutron, etc., monitoring each for completion. + + Example for a basic OpenStack chart upgrade: + + ```bash + /opt/genestack/bin/install-keystone.sh + /opt/genestack/bin/install-glance.sh + /opt/genestack/bin/install-nova.sh + ``` + +7. **Post-Upgrade Verification:** + - Check pod status: + + ```bash + kubectl get pods --all-namespaces + ``` + - Run OpenStack health checks: + + ```bash + openstack endpoint list + openstack compute service list + openstack network agent list + openstack volume service list + ``` + - Test key functionalities (e.g., instance provisioning, networking). + - Monitor logs for errors: + + ```bash + kubectl logs -f + ``` +## Additional Resources + +- [Genestack General Upgrade Guide](https://docs.rackspacecloud.com/genestack-upgrade/) +- [OpenStack-Helm Upgrades](https://docs.openstack.org/openstack-helm/latest/devref/upgrades.html) +- [OpenStack Epoxy Announcement](https://www.prnewswire.com/news-releases/openinfra-foundation-openstack-epoxy-arrives-strengthening-position-as-vmware-alternative-support-for-ai-as-global-demand-surges-302418295.html) diff --git a/mkdocs.yml b/mkdocs.yml index 1343564a3..f5169c3b0 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -280,6 +280,7 @@ nav: - Monitoring Overview: monitoring-info.md - Alerting Overview: alerting-info.md - Logging Overview: genestack-logging.md + - Upgrades: 2024.1-to-2025.1.md - OpenStack: - CLI Access: - Generating Clouds YAML: openstack-clouds.md From b35ec7c79af7d3d444584ceb64911e47f2625583 Mon Sep 17 00:00:00 2001 From: Jake Briggs Date: Tue, 2 Sep 2025 14:50:53 -0500 Subject: [PATCH 92/97] Fix: Fix formatting of upgrades doc (#1161) * Fix: Added docs for upgrades * Update 2024.1-to-2025.1.md --- docs/2024.1-to-2025.1.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/2024.1-to-2025.1.md b/docs/2024.1-to-2025.1.md index 47eff7c33..45bb880b3 100644 --- a/docs/2024.1-to-2025.1.md +++ b/docs/2024.1-to-2025.1.md @@ -30,24 +30,38 @@ The upgrade process is similar to a fresh installation but focuses on updating c git checkout ``` - If there is no release tag for Epoxy then you will need to update the images manually in the overrides - - ``` + + ```bash *** Example *** + keystone-helm-overrides.yaml images: + tags: + bootstrap: ghcr.io/aedan/genestack-images/heat:2025.1-1750442748 + db_drop: ghcr.io/aedan/genestack-images/heat:2025.1-1750442748 + db_init: ghcr.io/aedan/genestack-images/heat:2025.1-1750442748 + keystone_api: ghcr.io/aedan/genestack-images/keystone:2025.1-1750442703 + keystone_credential_cleanup: ghcr.io/aedan/genestack-images/heat:2025.1-1750442748 + keystone_credential_rotate: ghcr.io/aedan/genestack-images/keystone:2025.1-1750442703 + keystone_credential_setup: ghcr.io/aedan/genestack-images/keystone:2025.1-1750442703 + keystone_db_sync: ghcr.io/aedan/genestack-images/keystone:2025.1-1750442703 + keystone_domain_manage: ghcr.io/aedan/genestack-images/keystone:2025.1-1750442703 + keystone_fernet_rotate: ghcr.io/aedan/genestack-images/keystone:2025.1-1750442703 + keystone_fernet_setup: ghcr.io/aedan/genestack-images/keystone:2025.1-1750442703 + ks_user: ghcr.io/aedan/genestack-images/heat:2025.1-1750442748 ``` From e1e26fc01b94f7ea3aa34a09696e589b39f59463 Mon Sep 17 00:00:00 2001 From: Jake Briggs Date: Tue, 2 Sep 2025 19:07:48 -0500 Subject: [PATCH 93/97] Fix: Updated ye (#1162) * Fix: Updated ye Yaml editor has been improved to pull the values.yaml from the helm chart as well as apply the base yaml from genestack and pull in the overrides file. Then save changes to overrides file * Update ye - pre-commit(3.10) fix * Update ye - pre_commit round 2 * Update ye - black fix --- yaml-editor/ye | 149 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 126 insertions(+), 23 deletions(-) diff --git a/yaml-editor/ye b/yaml-editor/ye index e84aab563..68ee7ed5b 100755 --- a/yaml-editor/ye +++ b/yaml-editor/ye @@ -5,23 +5,64 @@ import subprocess import tempfile import yaml import copy +import logging +from datetime import datetime +import argparse + +# Set up logging +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", + handlers=[ + logging.FileHandler( + f"/var/log/ye_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log" + ), + logging.StreamHandler(), + ], +) + +DEFAULT_REPOS = [ + "openstack-helm", + "openstack-helm-infra", + "grafana", + "mariadb-operator", + "metallb", + "prometheus-community", + "longhorn", + "fluent", + "kubeovn", +] def load_yaml_file(filename): + """Load a YAML file and return its contents as a dictionary.""" try: with open(filename, "r") as f: data = yaml.safe_load(f) + logging.info(f"Loaded YAML file: {filename}") return data if data is not None else {} except FileNotFoundError: + logging.warning(f"File not found: {filename}") + return {} + except yaml.YAMLError as e: + logging.error(f"Error parsing YAML file {filename}: {e}") return {} def save_yaml_file(data, filename): - with open(filename, "w") as f: - yaml.safe_dump(data, f, default_flow_style=False, sort_keys=False) + """Save a dictionary to a YAML file.""" + try: + os.makedirs(os.path.dirname(filename), exist_ok=True) + with open(filename, "w") as f: + yaml.safe_dump(data, f, default_flow_style=False, sort_keys=False) + logging.info(f"Saved YAML file: {filename}") + except Exception as e: + logging.error(f"Error saving YAML file {filename}: {e}") + sys.exit(1) def merge_dicts(base, overrides): + """Merge two dictionaries, with overrides taking precedence.""" result = copy.deepcopy(base) for key, override_value in overrides.items(): if ( @@ -36,22 +77,23 @@ def merge_dicts(base, overrides): def launch_editor(initial_content): + """Launch an editor to modify YAML content and return the edited content.""" editor = os.environ.get("EDITOR", "vim") with tempfile.NamedTemporaryFile(suffix=".yaml", mode="w+", delete=False) as tf: temp_filename = tf.name tf.write(initial_content) tf.flush() - + logging.info(f"Launching editor: {editor} {temp_filename}") subprocess.call([editor, temp_filename]) - with open(temp_filename, "r") as tf: edited_content = tf.read() - os.unlink(temp_filename) + logging.info("Editor closed, temporary file deleted") return edited_content def compute_patch(base, edited): + """Compute the difference between base and edited data, returning the patch.""" patch = {} for key, edited_value in edited.items(): if key not in base: @@ -60,7 +102,7 @@ def compute_patch(base, edited): base_value = base[key] if isinstance(base_value, dict) and isinstance(edited_value, dict): sub_patch = compute_patch(base_value, edited_value) - if sub_patch: # Only include non-empty patches. + if sub_patch: patch[key] = sub_patch else: if edited_value != base_value: @@ -68,51 +110,112 @@ def compute_patch(base, edited): return patch +def load_helm_defaults(service_name, repos): + """Load default values from the Helm chart for the given service, trying multiple repos.""" + for repo in repos: + try: + chart_ref = f"{repo}/{service_name}" + result = subprocess.run( + ["helm", "show", "values", chart_ref], + capture_output=True, + text=True, + check=True, + ) + defaults_str = result.stdout + data = yaml.safe_load(defaults_str) + logging.info( + f"Loaded Helm defaults for service: {service_name} from repo: {repo}" + ) + return data if data is not None else {} + except subprocess.CalledProcessError: + logging.warning(f"Chart '{chart_ref}' not found in repo '{repo}'") + except yaml.YAMLError as e: + logging.warning(f"Error parsing Helm defaults for {chart_ref}: {e}") + except Exception as e: + logging.warning(f"Error loading Helm chart from {repo}: {e}") + logging.warning( + f"No Helm defaults found for {service_name} in any repo. Proceeding without." + ) + return {} + + def main(): - if len(sys.argv) != 2: - print("Usage: {} ".format(sys.argv[0])) - print( - "ye - (YamlEditor) will launch an editor to edit the values for the specified service's base YAML file. " - "The base file is located at /opt/genestack/base-helm-configs//-helm-overrides.yaml. " - "If the override file exists at /etc/genestack/helm-configs//-helm-overrides.yaml, " - "its values will be overlaid on the base. After editing, the changes will be saved to the override file." - "Created by Jake Briggs Rackspace" - ) - sys.exit(1) + """Main function to orchestrate YAML editing and patching.""" + parser = argparse.ArgumentParser(description="YamlEditor for Helm configurations") + parser.add_argument( + "service_name", help="Name of the service (e.g., keystone, grafana)" + ) + parser.add_argument( + "--repos", + default=",".join(DEFAULT_REPOS), + help=f"Comma-separated list of Helm repos (default: {','.join(DEFAULT_REPOS)})", + ) + args = parser.parse_args() - service_name = sys.argv[1] + repos = [r.strip() for r in args.repos.split(",")] + service_name = args.service_name base_filename = f"/opt/genestack/base-helm-configs/{service_name}/{service_name}-helm-overrides.yaml" override_filename = ( f"/etc/genestack/helm-configs/{service_name}/{service_name}-helm-overrides.yaml" ) + # Load all configuration sources + helm_defaults = load_helm_defaults(service_name, repos) base_data = load_yaml_file(base_filename) if not base_data: - print(f"Error: Base file '{base_filename}' not found or is empty.") - sys.exit(1) - + logging.warning( + f"Base file '{base_filename}' not found or is empty. Proceeding without base." + ) + base_data = {} previous_overrides = load_yaml_file(override_filename) if not isinstance(previous_overrides, dict): previous_overrides = {} - effective_data = merge_dicts(base_data, previous_overrides) + logging.warning( + f"Override file '{override_filename}' is not a valid dictionary, using empty dict" + ) + + # Merge configurations: Helm defaults -> Base -> Overrides + base_merged = merge_dicts(helm_defaults, base_data) + effective_data = merge_dicts(base_merged, previous_overrides) + + # Launch editor with merged configuration initial_yaml_str = yaml.safe_dump( effective_data, default_flow_style=False, sort_keys=False ) + logging.info("Launching editor with merged configuration") print("Launching editor. Modify values as needed, then save and exit.") edited_yaml_str = launch_editor(initial_yaml_str) + + # Parse edited content try: edited_data = yaml.safe_load(edited_yaml_str) if edited_data is None: edited_data = {} + logging.info("Successfully parsed edited YAML") except yaml.YAMLError as e: - print("Error parsing YAML from editor:", e) + logging.error(f"Error parsing YAML from editor: {e}") sys.exit(1) - patch = compute_patch(base_data, edited_data) + + # Compute patch against merged base (Helm defaults + base config) + patch = compute_patch(base_merged, edited_data) print("Computed patch (overrides):") print(yaml.safe_dump(patch, default_flow_style=False, sort_keys=False)) + + # Save patch to override file save_yaml_file(patch, override_filename) print(f"Overrides saved to '{override_filename}'.") if __name__ == "__main__": + if len(sys.argv) < 2: + print( + f"Usage: {sys.argv[0]} \n" + "ye - (YamlEditor) launches an editor to edit Helm configuration values for the specified service. \n" + "It merges Helm chart defaults from specified or default repos, base overrides from /opt/genestack/base-helm-configs//-helm-overrides.yaml, \n" + "and user overrides from /etc/genestack/helm-configs//-helm-overrides.yaml. \n" + "If Helm defaults or base file are unavailable, it proceeds with available data. \n" + "Only changes not in defaults or base are saved to the user override file.\n" + "Created by Jake Briggs Rackspace" + ) + sys.exit(1) main() From 4a27e25583e4a6e45dc501c79c822794f814c60b Mon Sep 17 00:00:00 2001 From: Bjoern Teipel Date: Tue, 2 Sep 2025 21:58:41 -0500 Subject: [PATCH 94/97] Adding keycloak SAML settings (#1163) - Adds keystone-sp directory with keycloak oriented default settings - Adds keycloak minimal configuration to enable SAML and keystone integration --- docs/assets/images/keycloak-client-config.png | Bin 0 -> 154086 bytes docs/assets/images/keycloak-group-mapping.png | Bin 0 -> 69738 bytes docs/openstack-keystone-federation.md | 146 ++++++++- etc/keystone-sp/shibboleth/attrChecker.html | 57 ++++ etc/keystone-sp/shibboleth/attribute-map.xml | 173 ++++++++++ .../shibboleth/attribute-policy.xml | 77 +++++ .../shibboleth/bindingTemplate.html | 58 ++++ etc/keystone-sp/shibboleth/console.logger | 33 ++ .../shibboleth/discoveryTemplate.html | 48 +++ .../shibboleth/example-metadata.xml | 172 ++++++++++ .../shibboleth/example-shibboleth2.xml | 299 ++++++++++++++++++ etc/keystone-sp/shibboleth/globalLogout.html | 29 ++ etc/keystone-sp/shibboleth/localLogout.html | 27 ++ etc/keystone-sp/shibboleth/metadataError.html | 35 ++ etc/keystone-sp/shibboleth/native.logger | 30 ++ etc/keystone-sp/shibboleth/partialLogout.html | 24 ++ etc/keystone-sp/shibboleth/postTemplate.html | 37 +++ etc/keystone-sp/shibboleth/protocols.xml | 57 ++++ .../shibboleth/security-policy.xml | 44 +++ etc/keystone-sp/shibboleth/sessionError.html | 45 +++ etc/keystone-sp/shibboleth/shibboleth2.xml | 73 +++++ etc/keystone-sp/shibboleth/shibd.logger | 73 +++++ etc/keystone-sp/shibboleth/sslError.html | 33 ++ .../shibboleth/sso_callback_template.html | 22 ++ 24 files changed, 1581 insertions(+), 11 deletions(-) create mode 100644 docs/assets/images/keycloak-client-config.png create mode 100644 docs/assets/images/keycloak-group-mapping.png create mode 100644 etc/keystone-sp/shibboleth/attrChecker.html create mode 100644 etc/keystone-sp/shibboleth/attribute-map.xml create mode 100644 etc/keystone-sp/shibboleth/attribute-policy.xml create mode 100644 etc/keystone-sp/shibboleth/bindingTemplate.html create mode 100644 etc/keystone-sp/shibboleth/console.logger create mode 100644 etc/keystone-sp/shibboleth/discoveryTemplate.html create mode 100644 etc/keystone-sp/shibboleth/example-metadata.xml create mode 100644 etc/keystone-sp/shibboleth/example-shibboleth2.xml create mode 100644 etc/keystone-sp/shibboleth/globalLogout.html create mode 100644 etc/keystone-sp/shibboleth/localLogout.html create mode 100644 etc/keystone-sp/shibboleth/metadataError.html create mode 100644 etc/keystone-sp/shibboleth/native.logger create mode 100644 etc/keystone-sp/shibboleth/partialLogout.html create mode 100644 etc/keystone-sp/shibboleth/postTemplate.html create mode 100644 etc/keystone-sp/shibboleth/protocols.xml create mode 100644 etc/keystone-sp/shibboleth/security-policy.xml create mode 100644 etc/keystone-sp/shibboleth/sessionError.html create mode 100644 etc/keystone-sp/shibboleth/shibboleth2.xml create mode 100644 etc/keystone-sp/shibboleth/shibd.logger create mode 100644 etc/keystone-sp/shibboleth/sslError.html create mode 100644 etc/keystone-sp/shibboleth/sso_callback_template.html diff --git a/docs/assets/images/keycloak-client-config.png b/docs/assets/images/keycloak-client-config.png new file mode 100644 index 0000000000000000000000000000000000000000..efab4b7cbdf47c5107a956929ece0a1fd72d2da4 GIT binary patch literal 154086 zcmeEuc|4SD+rK3$B}pn|xs&Wkw(PVJLiX&*zKyao%p^r*D_c>>WM^dGsbmRb-^Uo) z_iZrD81p;b_w(HE^LcO2|Gz(8pAR!;&g;C6b32yr@jb4vhgz!ibgXm~6cqGo_wVXb zP|&neP#l{-K?^<+S?1oOprCx>c<0VTwL5pX9=f~PIl9-5stG}z?+#JVp(X*k6QdZBA>X74I1A|zqZ!PPO`Ff`OFK z6E1vypYq~O%x~d|(w3dlm(NUWWq({c_dMO|gZj-?OLJ3vO^PT5@7RfNdBr!Qf4PUN zuNTx6K4S>&Mt^<3?P*+f2}^$*e`XOEB#dpv5>lkzw@b15~^mrXKZT%3f{c&e-*gGRnFPT_Q6({v+0uV z{U6b%9a-s~#0mEna^3eKG>D8YYNQ>DVt+sUGK5Mf=^S_H9n}=yleCvI&t8doob<$S zjEHBK9skXDR`HJ*neiWNPazbm$x%|zH1|$D&2S&kha;C9?$1PixSt%Ft3-b5+JK}$ zs-^8cMz0y5At_XW(MkmyCo3YY*JoJXV8En@+dfjW)6}F80Pjyw9HV5VpaJhF!LI@( z+u!e1DETO;|NNYaf+EzB;@H31Xo1(mpBV6ac+Fq0)bBzlj)VWsfnVP=s(-hpX-%X4 z_xoe>;5!Osy*p}Z;8oAs-PRW3;o#=UKX_vkd~ovVePa&_iVHl4zm#gaSJy%R!;boo zJRfO3kg;}iy>4aW_QdwOuj|vpekkO8Wx!ikTTd%4Uso51hm5cMl|Ng^fcJ+Vi(KLQ zvx%p({FO(V54rBRx!ZC{UcYhu#uWuRE-o%PcN;qy-McFPx*Yr`f5pMm^Qnx8h>wrY zbsw?oZtnIXqSDgRA~$Y|+`K6aS_pgiK|HN|g&`i?e|7Tje(u_OSi3tu^>lQDa2@t* z^~BA~Q~t`8!yEnm^Ou~qzK;KMCy2+th6M&Fa`=si==B>SfA3@9nzb`fLuyw!V<_d1=sqjAw z_OFZo^UHr-C?|3__WziRzb5+6Pr*zp(8-DXy=n?{J9FbKU?DF%-qqFzuV9-U{!n>> zKYV|^9=^wY5plbBkAgypLhY`yzAxo6=J@{kCK7o=`R;Ph>u}*GCxAv${`c|R{iaHMjc$@qwUgd~;|0xfbQm4X^(jShf z`z_V6R$2FwlHUI^L<;cz$nhgbcG$a&?sE!1RfxAC>wgpRIfXRwsJZ*|GbL5Jvs`Pf zs7`h6|0d#DE_=?S=Jn6d=I>KdO|w^x7&{(4`JfTi^fL8-9UD8p((_twDOthD|0bf+ z^N)d#t4{@xwukI`zo^-b*d|KU`EMt!k{nyk1mH0~nf6^NE0s?9Oh>LC{iz3dioHH< zMOS3Qfj@2@5uH?lg~<}|14fn~_U2NtLX{IQ?0&zxD9PsXHE3_73ub5c$!jpgD2~Nc zw9@m_THORUz-SYB=92R9r_N=o(evzNwxGx5$Tq2UGmX)6tA zNDYsS>T|j8Z`14}FA zgYhylb1Okfib;(LK;0N=gyYbv7uo4;nHu4DB{aG>ic_B_$8LYI!x2|*VwI`}^~<8Jh_)@tA@`-$>Jv3w>D$Be{cno9!Yf!NS@nhg&ooV^QaC@F{DMuCShv>$f+Ph-x5{-1;iq zED`$y^3a|9Tg2m>0guJ8r^PxR$6Zrhc;>{9qKnZ^mK3hu$bi7ACSIIUQE@c7*59kz zpvAG;$9=r2;f%T8!D`J~(yv%v)v9(ckFoJOs82?fgGEJw0&%&-0sFyk_Cw8BCTq6= zuU_T@(UL9&s3&)yoFaFXXN<~GXU0S&Er)Uw?baYKp}0GfXbCCDc%wrY&u8o1C?8K( zTFur$L?je)l~{WA-OhmOOaxZVB;c?DP#DH*=tJWfSs_t#N;pj;15gOKR_^FH%q5)_ z@~l*YC^%DY=Swv@%x0Q`85A6}6iHk2w2^fBgnMVe-Ref{Qv-9yeK)%AFh~qp!UN@M zv`)`oiyqZdH^A*qMYpUigi%!=_``sP)4{yCUd?CHFk}HmRgMipR0^^PU$*-Rb~IVI z(=l+uLe^|)pO)A~%YY)kji!s{Mw2bfMrJ)iW&OH#Z-mQV@6_ZgG>Prw)r>F|BX(@K zF^{e$loR?DTEx{-@39eYyrE^_wy%Z!$vUreHD1dfH=j;?n{T7j>>8atkYjVJeRd`A zEg|qHqw4zx(u$i3n(&J^RcHw>vy^WQBcib?#1yTlev@(kC1H<2WBU&pB6X!Mrlj>y z#R#poc^FMzL3xs~2TCHcyg;z}iOu@N_f^~dY?3Z%zuofSTp7!1tdq~T=ahC}LM5Op zWr-hL%SR^t{Ej;A-M^_at|b(dkLKF=F_FE&71yqd)-4?FPn%%pFxX4drX`s$)Nnlz z%)>(DNRDKdzp-gnioeq@TA}t#;!5?LbxRb=1r2YcWzvuwHr7jaA_8^1X)gk;@6(dy z87&TK>;SY{Z)8!m)dfE&$>&UpvD@82 zyJZa)h;J;HbRkE~NyU^4y`Y70F(Oec8P#sSv?jFZ*AKIyZlp)$&A0$Xx>23sirYnz z6^f*qHHDmk5fC9wUTnDkpnZ`{%jWX1Seg9wSMd-QMs)o}Tkq_o*FNgMRM5USdS1hJ zC5lZ7`Gs^V5B}S8rIGM)I8M{nRh06GgPw*v_MpVjSb3rJ`NkKu*rc-7esAXGQtoHY zXVR*kkLKnfO^T&xG+o7nE0<>Twl|8~uWPuXH9y|4x{aZ96gH*o0T!oOBG&hGaK->( zk!sreygLSlbWNW9VxwnqmGJqZW5_mbYnMEJEQFY`l5Du?DdO~e%(cdZbL9l-ou)xt zGmEAk1t#j2o3CjV>?P%93kOtaTef+ia4TDBui;?3utNbydxX_ivOgg%tOV)vj-&Z` z{_0X^x>6Ck99p6%C z`(zAQCGvX49WzXxnILIwiH=`;3BM~v=i$Wqo29jvfy(1gdF|7$a=v(1k2 zS||_dYYN3)C@1XwC~wu00L`{}^k|4Z}GrRzh;2rew7W@XarR7KTJ z^4FK#t$npy#%i*^XBN+}%ZR&3`WJMi?a&*E1n4Kn@7OlLDp9=(XeVv%iD>zq;dh#{ z`X+95+q%Sp1xX1Fr_>Lxk8l>crKvPe#RzV{;vDP6Iq@rP+8K3UUk;{nfbUOnG6lKO zY!vH$8NNJbi%~zd>Lg8f11&eq6KUepG`_0)}NdG?Gn(Dup^YYbbsnm)ly}z?(4N9BiQ@4)%r{~!`!Vybs(MK7`6~XRO zimDgJRu7kfanDwB-S~ja*^?S3qh}-I3gX0!kyfb^lTt!fU-fZ(w=8n7Ncs%5Dqs-XsoGZq$)du1 zx!-avdr#l7Z-DKZAsR_ugJCb6?kV>$Y0HU9%xH*V%p1bbbCF4F@U?35oouE*MtuU| z@278=WLgU@Pp_AXL7+rgbBT%|SseA6C9fRNvBL2uyZJB3@@fQ{IyPS`3 zJ%V;FI+#2U0FGT4MZyd7sg~$MHGO6Dja!E8OQvph5)dq>=<;mcLT2(Lz9ePLcPi4h zokQB)Cf(oUsgYE1zP|WvN#U91L`&F*_red@{&?c7)2XVPWZO?L zL$rUYg37Iw_MI!qVi{!OI`k%)+8;W-N5dqR52743H{fqlfI*-I7k$GG#AYt?$jf*(~k5NA4V(nQB6C3F9QB0^%|EaCWEKg#3ext5rr+6&Wn_HJlg=y&?nSzk@eJ&r0}aEO2V#J zfp+GD59fqiqFkg74deo*ro#Af5KYJJQ^GdO_g%#1K6)haBs{v(bCHcpiudEw+qJJ% z=*_7mv6U>ho~A)R_?Ak&65mllWWvb@cE5NM#WjS>^tVP`8}8Z?(??bzel3?hBBUo@ zbB}3QYhQT}Vx(Iz^OawOr9!vW>Lp}l5=t53VgUphjWsFv1FmC4d4`~*62wK|9R4Ow zbZgALT^I#yE&D(Pzh6f!t~2@lP-cZFw@SqQ2YFjpT(GpQk?i+4P*1c^+#@EVpcaEK z>!}Z2dj!c84F?_rI80gRk2oll$Iek**G<7sy}D@eNb-lC6e>T1FLWFDjTRfJ=3&J< znE4)Hi}IK2N(tK!aBj6D^Ye41Tf)^NX@abj=Gv*b1N69?q1>&Fcyq;Gj(9N>M+rky z*RtHeF`HP;w#8}r7iNKDkAvyk5@K!B8CR;M?R^n?WqmF_!gn%0sl+%3{wNifR6_pJ z@tc0D+E%VPsHu;U`)yq*Ok$kQh0cDkPY~w**a9yQm#e<0^nQ@f?0O;N5h1pWYMLXz zVK1aREACX}mv1MeLSK|w5ol?YFp*?P{oZUfl|AgN@KO~O3@I+CguL8o-0o&m{b0Y> z;rV{DTRp#PX^)EeqNy0wbhWV!H>sGrOCuFVaC`s+tw__Ni^&S4&6`>9(OMN9n`@+~ zTFl58Jvn#EQn2h{UEs68Nyej`in?P-7q~9QYI^8g&__*W#;}I5($?+ow^U2p?!t^L z1?P$cffJwV*{fECM{gTPN7A;&$I76zDSeYWudu3JPWgIbNs3!(b#Mi^O(Kq4?qv$Q zfxYt{MoBBH34Lmmo2!?USIu}elk`V{gPb1NgKdsaakkt8fx8)#+?6Aaq-VBLpL+~? zB2IewYhwpcR#{okiV#vL5Hg|6JW$NqdH7$W&V{bck!*?YQJJ2ivddrZhzkpoHzvLZ zqrM;K{P;dAwZp&}knBkqwRrlSrhStxwB7t!6EmBI-7V`9l^8VBaoeTx zwR&R68@F9AZthmqNLGWvELG0HQYX#}mH`_-15lThEegcco6;GZf5Wl^)iOp_k?>FA zFU*!MN6il@eD;(tOLz*+$as{KQ2;tZ9@_VKS2$VVg~{mWGe0mpz~^j5d?5Dlro% zvs}3LDambj{dbsmS;S#X@ukbXEM2J@%PSAOz14(_EB^F}T)Apad42>HkZrAI=%~a!)>v}aGI-g9aE+4bL#AP&5lhDgfxd}{rS?OE~ zG+`V@TrsKEr=y+*D)RN5b7qa{eUs<4DRRSB&ubI%t~ia%f}_{4{^OJ_u~^|vwdHda zFZ*~S%fPZy;N`}lpgH2GXUkf9i{j7vs4E)Sl9QEUQZ8cV*>i8hNxTmu zJPjhR>q~boZ5d$l83*ZTWwr4R9(6UdQ>yR-V0lWYrWd=vS~a#2^6TAzO5`Ei#A~@P z{EUnE(Jurs56n9}ieH#qR^>nO745ZH2e1DM#A7b5i9k342l=k6*Sai$85O+M5!Y`D_LdqZMKS3(&;y|max4UP4WQ=1+>}) z%&>Og)*JfENi}QrXrc1qCkHlt`N^KuujXlkt0|u(1wmq}desF#z(|A-Qschdi}>kZ zv6%cG1ieDi60)YVLk`ywwjfy$$s+6RB8SR+f5SV@4Pa(rge!=z!~3Xa+DoW&8bVgn zF4TjwU^&FAjh##yaJ6b#>|U-e{tQgW3idL{pt%pGwIvEExnpzNv!wad^~!H@yA#1- z2HoWYC^;y8m0jB{p~m#v?#2|##JOO`qB(kWaP;De(c2z{-HjH`EXYM@@sG=}-QODl zt6}o|Pf8Ue9b7oT{|+oWSGBgh!AT>&BUhE?*hkRo*UXA>3Sz$Wp~Srw`{z}%KQ}-v zqert=e(}R~7d?OnQeBk!v3%U$86X8iR~Hw=>VAi?v9jHY-l436t&|OQ7#Nu|W8@@t zM(V#B6~4DkFuwM&<48AI>)d=Nj=I`$4geHP(S*`YdU>PO32=gtnMmiCe(y~XDKi31 z_a7xmzZetNKscAE^bYpI+0J`wHbs|3T#FKfYKs$U5^N1~W9Lq|XRIEEk2U>S0JOoF zHj?FCLu#DOb6yl9L)KS{dx!Zh)>(=yRW)=C7Ne4-fet+AlwF(Ntn%S*l^6{GsBpor zEA96eAbdw{q!SL}b`=*g;J1KzaHcz-NmFq~WnoxuE=c^w#@ipj&$IW3he4Hk_~ z9T3+n4W(+;=GcA4db)oV0)v*TakQVMj^W_)nQeTp|=us)p$!}#V72iFMvcsLs^W~SPCGW=@ z$2FQ%B_3is)^78Tb|0TL^GSJP>jNVCV#TEa}(P_yEERDc6LAkC_3R1i4WP?{xT1NV4scRbTp-*~fVMexES+ zcEN_qfOu~B=T{eQwy7yLo)+6R@QXUatUQ127)BwDK18e5X@EawKL~A z;h%RcjtmsLX#i$!{>|F(*x79JZ<%*Q55Z%{5Z4OKpc$~YVb#?)mAI#mRMaP?d6b^Z ziKvY#Qv^K-0%$uMl-jriSSeM+H`h+zK7?v?F@Y{U)qB*jc~)7wHtW0)SW>MgtRhE2 z#b@VpL9P%}ss0GvY(CXpmlD*4nI8Bd)(rqy#u)Fzr7f594d9B zNDg;F{*T+KBbb6p;uLUqCF~08c5VW%3xdK`hO4)`gFTeYxj%6na_&g<wMAEB#NvxBB?sN4(MmO%ZXPV##m|7cSlJ`9RhAY`Y>XQBUT=aj0xfn>0` zLeZ(t|8e#tAZ^8w^@UL!60c=QwgKGT0V#}4I6I@XtuwIr&v`1C?#;}u&G~x@5O(}Dx>+=us)S%71x6q&^?@i z((_k7DgSRUD`?^RU$QRLFLi@{=52z~Ipk#gl8aXVA?w3`HQjXmbr=$+p^t_8ug^DAf4Wc|ZiL?#TlYyuDagH>xF41=fsAuB7>Urnvi$yLD34Kh3IwbgdU@z_6_ zj;j1MEZ34##aF;?xqB#UxPxZuKh`zP?_bk8uAtn0xUdF?O>>c#nE&BPry&lvD)EQUsm(b6p#y*z%(Pv06cY?ZIIBUmAyV zB};QSS6bW&ul~`pK;*9#7{pf0|3P)4v`9&Zfy1>{?UBSk1aMILYXaz9T)9M$lJJVj7Z9}mj5h5)*e3kgu49tz@X_FfY?BLc0L`*f zAn$tsS`U`2InDC#W6+JF@@u+PU4_mppqeJ=+*-;nN#X!uUIlmi9UwF7xz?qLq65NG zGT>&kbw~`E&4OHwL$*kVe)6}K3M>R<*QT>K0fq;i-D?_0U$Y@?rVUP=l3`d z){o>=I874 z>jfZ=E1-=SiMfu7mk<{7k`Yld0u>yAAY_~E)ew=8!g$6WW_zc+w`CvC#2ZQGA**vo zobn1f0I!Urp@r$+bDTDEIKU)K*H_X4D;Ie>R=Wd`EINQTb-iHWYb6Hqb+1SowF(Pg zVmknh#0da}jwJx%NIJycVQ@f#nFXk26zo`vN2j!1-S*%c?9`za>&~U}W0yM-7D2G3vf}h4&!Tw3FypsVS%I7@X4l_xQL(f@+^!_S?=H^sbgrIzUNbFOhCtR)p31?QgW80369gG6Puy1ZNbW zT}AJW#IyZJ2(ikrZ!e!cV9eG;#JqXE#+eW1Kn0?2SssC__JR zu6|8k(yfq#5iveHFWX9Xuwr(bas1xoz5e-g`tgTszg2sWq$byOr&r90Qtd)@aUY88 zyC4e}Pbq&pIqkLHDyDP*2nlzw5`b5!h}s^qbOXWlOm#BhRuiloS-Yu@1MG>U;4?C< z`we6~S$C5x1JlxTQ(JE1ZV4KKf$pyk#=FM8@*?rXbe$=H$v*a!_@ENEG`!sAnDxo791DZu->Lq4mC_vl z+=YYRBSLC~m5^_-HOs{s;sjZQ$cBwMUH0SlXxJ9QHz~!2rj^rgnz3ZmDX+vP^2L7C#m9h>RLn}pjZG$)54u zlMG4i$jY{TB9^T6pvzwrscrFM02tkzCdOq*Vzxh@W&I?(u)z z3Asd5u;06&EJQVZF#gTs*e28b)>^{;Rh*+*Ljgu!*)v^VsuqQq7x)OcsM4(W4*F6L zN}t453*Z3(=k?Dq%EbgT`VZGvId>hY7Cvj%iA&et+5bw(o$FVFO2C^td&vg^=%70vYL=pi(O z*8t6+RT?(cplty_sxinUf^m{*hs8!U3d>%mP(lix5R^UVnmJ-GhTSLP97gd8kACF) z?Yy!pe|6zxO>f3fv>c&Qn*5?+n70FGo**i0HGWH-I}_jQ9_QnK z_6>#Ks70grvFhuGTh9DqN8u{~thKn*td#2n14IxZJrk$5={#2cCGS3TYhI@e1xTYJ z^>4QT&l;i<&0+4vA@KJ^nlm0wWW0j-O*cL2>9)7>kH0;?Z)J$$sqfNP)Da zR|3eZO^$ctLTo@@^I4e_T~KmQa51XyW{sDpmU*x|6FU6%aW5#v$6rpgi65g{YC*Vur6Aod~EbyKgT zbJTe>5Ghw!N7(LB6ii6>_0JGY?H0Z_B5elIVY zIQ)TNvM-DCr+)AE<&ln}*9-l?2eJzKOBMBNap!IuHO)774TP@-A12q+W*Gz9c=Dz` zK&p-T6Z3n+7OtfTD3bTO5Q0fR>ZjR%`UCb)r0(zepDFc3lIQi`#W+~OTE`q2TGn}P z=Vz8~o4(-oswE7%2?s3wNSFxU5VDuXbLLBBFSA2~F>qvZXGihG-zhaDL*99>l8ySE zt1c-mIv63n)Kl_KXeX?5X=S}6fyLRg3R1sNmFmQ{B$gg%Thw&n4Xs`!Yyfx5b+nYv z_T7tEC*GymP2~8pBYY4$d0B*T`X56f%SDsSp#0o$%FHCzDo#vh{ ztBj`ua9*?oz>ad|A+|(V3U7?jMopEw0Rr^AJhZXr9qGGiF*lutBp$K=@NK>k#m7vz z9L*u#WV>stH4l~}mnZT1EeFruy_Z^P%1nkNIdJD&Yb*R*Lo2ras9-7_@PVV}tLlDh zr)wEm29SC}GoIb?<~J|u|FO|vDrth)`t?S{vuvSw4_i(gnE2z4M$PF_u>+r8PHn-! z&4nymuaaxSN`4E&Q>x9lyn?#WRZGHpV|1~dl?;{nh28Z=h82Y!t3MapXHb_E))Uw4+wPiezdmv~n+MwXwRpx+!Kkno!7j~?a&{Xw(hHPzziyFG4>w0k zaytK_wNGDrGUS6#A-In{Yl(Mx%#$Lu5=QTyxXicuj-I1wC6^x6usvv+xE%JYbVU7- z*g8en(cSKs`F%GzTp(pm8&1&M_1+)NGz?oBAzJ_JBk{^`RC<+3DQ{uJQ?VeA@Qi(+ zOnH7+UcN{<`W>MN@WXf)bC3^~QD2$1w|`_p!?dVEHyU-IkTA z?IM1Mbn86&8$3yQK4kR{7t3M64t0X&?}RCivK0pD>6mx)o}nc_3ta_1`qq1N$yeCG zzs4`i-%?-9b5@OZFIjjtxhEiB#G9nC(>xNkHxM$|Hy&4JL%kbs*A<}Mz*f4l1GZ+i za0A>Qxm_k5sUg{#II5D8S_Ft~w=5c*8Zf%D*1VF%siSd>8}2JyVfmQ|tl3V;w`by~_-_XHyI80=e?T>4V1+|-P zJO!b3oZ)#1zFUbIU3Ho>=K~G51A+N0$g2WEA%nDR(&$D5XnvgGYG?=V+D;qG}klnE{k~44b!O9J^fzH^w`da56wy4?r^y{b^ zm|kA{``DRZI8@0qQT-!IeSr4$^EFA?ZVE}}SI>I_f1Pyif}>o9N(STf!3hSo$5Pde z&`07!sD+`yRxsVF0tZ@Bg5Flrccw_+abFcdkwL_iDPCaRacQ~eMy(=?m&w&f?k z!4FB|T6vAJH1I*H4Kp*gFOWc8mtA{^Z{>_6$Si%tp@3$6 z?c640s~Q<~>*=HYB;FI#{4ql#2JmBhR-x<{u$P1i3fn~*YWIK-2bl#P#;$85q$CKp z>ov%PIomdj==uD2V$RcH}UBw8AyWXvG z&lRX}DodaF2-ac4UE*AYWdm%(MRM2bV7EoFxv(!~zQzzA^Ip!+eXHEPEpuBW}jZdg!nrVd*zXD&7!B>m=_)gX46Cl+{8Q=NO zp=0eGs*(LeJAFmtB7U%DPrdKBE=5Hr3#p5EMX7_EB`H$Xsf!hFAWx8M$KyqK zrMZY!|NhKIpGPty)`Y9vK~GrA!No%* z=sT(H`RV?IaXS1-xG_PY_aFz`ZG6x1rsqHuxn^jfU9*NgiZ7AAiPcihI z3E5%t*;mh-($-6H!(df;mb1t#My%@QJs%AA1>bqP^wjIR74e2muKBwXL12p$P*;4RHPOmFk|f@95v^-DiF<6X3vdJ4b3F0ze;bn zBTXL9WIi^;Ddni_jY8nJsSOo(P34u42f3-v~&Z(vA>2Q?eNTxzxxBzYRix(A?eOW$L88(BTNT&1y}ULLg#s@ndK?$*({qfN z`r<0}@iP%r?R4FFoS>Mj0gBECMRc=36=`lMFPvL$UC53aHCor>6=uFVpr#NQEts82 zkxUgvcS~UVRrm{x_f@+9sv?sc`nX+69zdly5GxYB%ZF0e1oQ@G( z&u2O-ut)?HTvyL0gUmGUHp^OC4xhZzmA<>s40tYkN!j$tRa<4(D3{I}OpMb6EjtW# zFqIrcsz@|g^h9BLbE6-(6*ZAyeUl=9Etr``9 z9)jh6E8yfqEs14%EvY}R6(r^)2*Z71y^ZIOMZz~5u?!OCIYOQctVU-3r|Po$16h^Q zVowqx5zn*BV^4jniG@KdqyzhE!VKUi-7iHDaznDo0?z}|LU<(=uC}3M-3v6Cy zra_u`zG$MY7?t<1LkL=4k>Ysp{rPsK<<$?-W<#htg& zTB#C&q5QUCQEY^$r@Rzf_3WS6vvU1F`4(cL5-ln7HmmFH>`*NMrWa_mJb>uIS3wI} zO1r*^fR&rK`5a}w+0Vs_mma`}=%VJ!v=rtu6A6&Mxsj;KC}M7)+LFNvE`RF%tWl4e z1$*>V-{Dzei(}2+^g?T!%otXq%kTmf!9&TO>8yR^bf_1PVR~p-qpW2NSUt(@mD$8c?*+d7#F3qz_@&WqO<}rmE~O1dywHe zx459Q#C$=T$nOys#Wj`{z)*%D@d`Pq=7^4MZn6C#k)88sAOsR+nue45H12qUeeoj+&Q zGpFAm>zbwzAn0GPrGaz}9D2z!X5YJ0w@%bw$yi}h|1k9+k2><-&jLuufQ1IqC3=Ak z9f}FF2;E**t1R|6#tP>SFO!UxX5enE1G5|=O9)ahKVda7jGU~upt>I@E6DyjAfW38 zNq?8+p}^0t*$fAwDl7a2Mvq*$6nl6Tc{Cmk)$I_o;%-^#gkOfs$LsPI%-4Kdne*=; zXg$R(;%ntM*hx6_kOw;`$LU11aQ_>Gj;TbXgEEiS5WbibYjl! zsuEHu2rEM4uzB5W+@l3fH4T>$UCRV~Us-;}*7~jRytagAv-7xfUOyj!oB;5UX^~7zH^+z}_)#qatAM`dDylITh zpDK~IJvv7jQ#WMz9uIeS|NV&YTB1_lCenr(PY2t1v95w#j9zj$e3)_|SH)2xWw7J)zi8dBO{4d1KsnMpP{ zf4^7lkD<966syNC%zQLB@9X87T?f>z^{2T3H4t#IeMF+vtiSP6ky%c#?DNb9}WIM%*&EPoQ~ z@hxh#D>vcR-Z@%Nl5GZ!BS~eO*tab$?2xpBnViK}*o*C0%HCA$sdy@Ht~h4BYifo+ z_jKvnkUFI~Nz2E}OOdi+HM&0Y%HeXO65?1vsd&6Js zAT^=)bWP&X~@^~EXv$n=X;N<2)=R8mwO@D=KNGTp|vl+<;|l{pnqK9KSN z6#PT8A_bjAys$(T(ydlL@d#<@V}6)>{{AzZIAQ+yll`737}TV`#yIVTVLa7PQWwJ$ z$GA#GUBi{yJv+j$TiC|T`i+{0J7z&{J};Ch*&62QuFX01$?m${>ykLvQ^!fWyzf^c z*T#DKT7NNo;aJ#G@Xm@;6DMQ70-=dP$;ipH>ia`l zqkO&UI76tmkwRs8BSjJC<>fZ}2Z8Iq)ur@<`Jtsx0Gs*koi%l*m9h=g&>mRwp(?R?^z>CFZAki?QnD{l|rh?|Q}` z-dHU}*{94SsTv^1z6`1%a!EJA11Wqh>IxMEyB$1UC6?Ch_skvy9N|h@ zX?K;SR=b{s(Jr1;A~+iElUm1pu0kw{raV4%S+OXjV!TR0=+bWsbU>?)Itw(o_Z^b< z;X?yr`Cy;+2qUenGyR$B8Joc7Ef|7wvvx;sNEL4RG;@@8K3XAO zGGB$7;gBf6Dk9~RiKD{T{qwz2HYF_}voN6rqU>*`S`r z6p^cAU*1Hi7f#~`;H&t;!92Zv!IJ=ifj6NB2rTPMBLU=%{ohRF@Ds@`vH`Liqq7>( z6&{9*=cR(-#Vxb%ePR)+E71_F{%DfQn;m!tue+-w_VeGME9%b(e;F3CA_ej*j9RdX zvV=6{;i`2nZcw_3E556^uVCPhRAe>J0d2Be*E5ZXvlXl$VsMJEOu;yFXvZ(KR^ zI91Wk1!Yc}ATembZxMEVEoi=TLiZKn;>5`Pg)YRTOz7p5K*xGThAzfz)x&fMuut%Q zv%KT2%=unGLV8Z7e09X{aS!-DF`6#ru=XK`8+VT0!1s3+|H)vJ0~gr1IW+bmxuAaz zcD8-ncS2`7p1HC-{m`@mfyeXK0!@v}$by*e!(M!YLJID_i7`@&!- zOijKlbTNav`x9}T*@e6mj&4eJHf?JT*Zbk!8L? zjg3d9ud9AGqKgrI&F)}vhpY=6N z`~T)SI2|JAI_<@q`F(dHN4`N;n=&tnf`YF|beW?@#HyAXJgup3G4SS)nFkA0@2i7k za`4zNqscR-!`jSLbh4+4PxjC;VrXqq;#A980jM0`z^Q9@0#rZ@VDQ=D^re^B)}vF< z=LLJrmmB?7I92RGS~`jsO;KfR^Y-54Q1u&9G$`NREkh}DVCoc79yT&HaOQGAszIvr z*YH~%9S!h+hwLJ5-}E_SGI*$gxgE&3M`Mw;DV6ST#_pIGs8X68A6hRI)qTzS@T>XXmYJAp;We$!( z7s`wLkO;Lz3xCH+Xj7p*bxAX=PP&@6esYmt2C1SwWl0tZ@SA2@xsD&FL|j}rHt({t zJrMV{$t>cojHG{mZ~Gkc@~LEv(P4dEyAT$;6WH8cfAAcf=JBEL!;F}>IWB#d1GHVK z3*exBMpmUI;_GP+x#8iw!XLjjoY{VgdVjIn8jUIz7wf!by>F4hyNGA8|7f+WHd~n# z71@cM{U$LwD{xR=!TU!9{gr0iuC~iMYB->^nKm<>)CupBAG+y6xP-q6+e&;}1|Ny- z+^EkXIUha~W3J1;7!;T^w|oNyHjRh1dbv))2hFrAh`F6>8O>2iT~0rS#TlDv#SnO} z)8L6f#)vs4Uabe0OW>QasW#<|^;X)&E%m>l2hOB=9dj}+ud77w@<4(nWP^ClF z8|mNodfn+>V(AiXYjs!&DCUqnYvwvwj0cJO#Y%cMePJk8b6u4<4kwc!?oF!jxGa@V zgg}Gpcfh}WT-JMHC4Y~!Aj!onVw5rk-LFxTfSv{{epcIq% z)wIDxFW~dJ1=|S1M11<`~{lcnJ=0@MhdXRd~IsxoI_4h2x%) zS*7ImhyN+p|Bu{l>3Knu_w0cN7hVpaZj?ak5$#vJ9FeDT(Qh@v6o{1OG)nuEvpooQ zgYyj;wJ#5#zfm=l;mj%G+)63lRB5*_rdRC496tR>XF3ZpU%8*MzOq~9er{3a3lhoU zh10?G7dXtM1v5lbkHe)sJO@@S-ACeiXntwSl0-6+Ays8tn{jRMpZvb7R14f^q|=>< z`budQLA7T%t`9TO9&bE2&#$zKrO^If?H>;3zJvLH?7d}Nl-t)gE=U=aiX6!?5Rg=) z5inptN07=QAyN6+<8=ifE`nwNv?&rQ={r@kX^X7c; zu-Vt%YsXr9t?&9S=lH?V6!g8Hmor%OicSc1MDVL&*72+fS*YghG1FH~bub}vPs`JV zjCUF%Ca1(u3%!*AaWVXw+mVc(GnYD^0F5YSm4+lN?ZJ)7vT|>t#B*lt2T=(uoiV9mq z+Jff7)?&KJZ8II3+o!4L6BAc$P7RfIi!7!d^>53PTIF0| zsx4Gyos5_3Q7MX-R|CJYz~ah09_qN>(FOL)ir`81z6RLoVqLog=AG*IQNhMyk~3;)=3^ygX2?fCrZZRu3 zRAN?&91cm8X)aZF1WnMaG>)^IiP~RR&ba19%Z2+3re)e&H{7dO(vUHG z$n5@wvcFeNo1RdS>dyS#R#KRbGtKMN$8pXBAxA!0+{N@NaxQ>}?Bd<6FvhAZTPine zkJ90s&@@`lO(s9eaT5i>tpTUb2<9FvJb%i9z#^ZnIiIMm6a;$@tQjo4cW8k0vAI)- zgD}gx>Dd0YqrRQ;KKYN_wACo@U#W*fV|y>BUruu8>}Z~(Ww7I1cI{;s7(d`z zJEHAwPJOZ~p*P3TA%QV>*ETXGC$Y*$Z@bDC*j%JHml&iwZV_G>!uBEJ@)3S;=$sU*{uF+anXk^Q*4T@FRPIS8Z?e;tyr$U?97SrCy ztToLleg6nKcn5M3k})_E)$J(e1ZgvRDP%x1m;Q$@NS4R*Y=kmtHaOyQcPrGWt&4 zmEBE{ZN7kH!|3g*9GMMkLTeRB5xQU^wTQW9%2WE;zs;pI9iH*|4c|M}pF_AkY2fSAG~&}A=)3R$~;(N2r$+nX-sd zm&ZY zhY@OTPbA(@J0h~3@R?i|+Et9#-&{2hk5DyFd^@?e_1^KblBJt=!uJWK@x`|Bu+ul^ zmTG-6iiF_w&bPGpUIxj3R9|asodVc0m@7a{vUN?#5ztTqjAwwMTR(XY3q?k2>+$ZM zNWRcENo(}hz>Z?p5ik6|N=zgZ=y%Q>aT)&Hsm3jHlaG)zpYCmhO~S%`_=fh*aPIE# z+KVX=9(p#iN!ICnyl&n2T!&|DgT_rdSC@<+y>0EB<8cO5dx0z5e*7s2G5&|55IVR8_ZFHX*-(cn=eoMeNw3^YGW}M(|OT%S)u zfVkr&Mmnn?5LV<{V6M4+vy`7RU}g*S`9;a^TFLEkWwl96x31Tq5Lj@n0ccV1kEEAYA{~RMIB-o)7{0LA)j3-Vj?PwjWyR*nB&xk!Dwo0!E<9^``B_2Ao6W0Rsd2@t%1Jt{(qemG zy$W849q%1963sTZHF!nT&>eX-Z21N{rK0WHa`6f8y>mwPZrZVWhzYqO&WDv(%2z9S z@^m$ZqD^l1ai?ctA7Pk6&+bx#+-T(u7?T~KZq&5W zxhq`=*u>VW+>bU`2eA#O9Vu}U@e0!^xx90vIMz@byX8IwaIn2ifesx^W1BCUENh!m z80Sj*`}ZDqHl>$MWX4EFr-YcWs$Tb#WH%uE{v|-)xM3Q2S6r?7gn`pQ--ni;)11;H zYBJ?!&r0D7Hs1?ko870!;?|@u?%CW<%nOOxUJNdZc|KNeCx0{U90a3XJn3)xGJ(=8ilf~bxb48*HBmp3po2#=?5H%K*z`By7pAWV2h_>7Pc~VdlMun)PyKgtK!9C z@w+r6s(Vk%3!GORibc6*R{7x18Yd0z$hb3BV)CYXV?TF1vFHI;XJ_(rh22$Yv|sW^ zh23Kpn$mW_$;S%!X1Z9RI!}a6#Gd(Z28M-PX<+Ru0ixkF`aSBQv8`Aqi+17sCYcrU zvCpfuKIcxO+SjT9e(tnNE6GGUt#gU}%CxIj-d(V=VwHZzEg`F{n8H-mie7~G>~+&U zmA)%N4C_OF#^^ZXOdUqa*%NY5P8l7sJ#0#;Noi zp{l#Kn;(ki!wq92RhfDj$%NKF6~*+rEN3q_3@Fn+xPWg*0=da%@Xwq{yT2U}d%Ol5 z{+y?Ke%N%nV!%A+F3j&2ODuR&+h~AlPBB&odE*M~Lj#&^qPpvYQId_cR`Mk3!?*M! z7t+UUOJypqy0pGhCxo|m>3IsBLuW;b zW$kI-loh+D&Wb9GRH(H$=h-LM(0QG3nd(y7eSkQ%&H0QO8>xsF&y?H+W{sU(uiWjO zjXbfE)qF91s<)%dHpHcl%O6gQZAKZnl6$*a!`K{B6sL^cl?c6g9s+4bibS3oay2>HpXL;qe;fV7rBBo-fO_Xs{zU?)um18v-2JUk(aMq6l!SsS8|lFp5<;V$^GRzVuS?evd3 zysrAL97lCd@i=pBM>d#fbXJHBq?h!Fne;tq%uao*HZ5{!z91JOP8G=rWgc=_i{3zoCp3+nj(wHJ z#rweR+|%Jt>%0mSb;?z~_C67j06)28#urv#7FXRZ4tTD@r(?_OYatGeaC3!dp-$O? z*xOsR<=M>mO#rF;z^tlcd9H^iqL%2659#Ke7EGP>;})nh z6x8CK9SB0-1Tm6A`2ELNCdHN9tWAL7s0;@Y{4t5{xQy(L=mgYij!P~ekNS$;Pyp|9 zY8$}b2oIY3E}z?Ytzq2~fwrOlJn;ms0B_I%uAd8Inw4An$>xdI8rXB+Ne5jH2QC?2 zC!GdYzMqd`oaY4-%)zsIucSbyd@F48v9iElw6ET*)&i_oEyPQ_u&W~Dx|I3&g<%bm zUjou{wa3tpJ#AxV=^opa0M2Br zGoUc$N#hWY)b29^$Zqdd#WiGyxUyXMH=>p%K)}S9Z*J30OZ(F&ECH#rGWGz zE{nNtv!}wCKVaUQVC0Gt%>h^i)d96_yY}qbXu4!pAAy2_Y8OI zA_jS`5zvpNXzArWK*r8)eb)6b1w62qq0EqJ=Y-DHwV5|xKmSV7iPIv`6s%D^26ES# z=m*(+sb z#7{5dGKv`6JSb1*ty$jP>Qc$x1{(%K>p7M(mUT_md+86Bq1*$GUf*e6K7Nasf$7zP zxescO;m6?(D#Js*n{%t8pQ^SAG&guchcpY+WJ2)AG5N`ue7?%90c;~vnD(iRV7Vmy z{Tve1@f^NC&Wabm(u%BJ@MxiriEOUlSqf7$WD?@br8mw4?0fI>QQBX8D>jFu6f4`D z?Z;;a*7^-CDtF`)D92p{t=6%P*JQXR6h;N{i;qK@W+b=Rh_Pl7M{@z2?42}T03CRY zH3%qq!-q}F77R>SGrMv!w^fohsf+FNowow_@4+u#S263&a=7kDxv9X|KnP}_I;fQ_ z1%$bmt#Cxz^~^g-H?T<>`!`%My&#p$uh?GwA9e@V|fUEAAU z3MZVznr=3tYTp^xlx+=`POk6|Ugr}FOgklJGQJ*DN`9?@V=K1f=mM?CO_(8+Nk#L} zXkvKatd`ls8b3zHj4mk6B{=mG`(5mN+e>^cxihG&SVw@2N=>#t)@PK2T1~c|1o<&< zOA=d`Ad+V`JHLwSupjlY*+i--7oVIL*DY&$23%mZYJEO5y!C)gf$#q^gJaM1Z5aJT z4SgKvH>PN3zxui41@<`BVw$3C8e4yCH@+0V0VTY3q!D4~NVgh+ZIWQ&S=_2K zwDRFW&hz>!t4|!B{D`~Rx^!VDC764)rd!c!Xz=Uxc_CK(M{jP48F#j*zgLXxcf=o; zBWf;ooRHA7D4vMlG6RIiwSdq0wB3YCOR?~Euixohc~QTT4lUH;Z#y3Iq1|&Uo_mw# zV{uDGrj_)QeeXlzrGyZ*^duoWiaH4>cTiHbLgfVl;*So~vDe7EOCyod+VL=)&pbi{ zFi0oZUJ9`e_FBsCMI%>xYN3Ri*A^I%(lZ&XEH5oPCUKQebA?A$wrPHb7ci$iZ8l}y z-0fKVP!FX+{T%tSWD;Tj=OJ0}D;-a3&L#a$Xca)vH}SNH8pb#*vM$x9Wko26ZhU>V z+EbY@Ytv0RYSY3SrU9f(hyrXMc{AqNmN@fI zzyTY}d6&=aD2}zsw2cJ2WEt6NN;=KR)+l_#As58%?)e1#uOa2CMxC}LNe>pk1 z+^)dC)X1ZNqD>7K2~TkOYv8`poh0N=n4#(H5+XBp7(^{VdY zn-0Lz|2q7AlIxcantJl_^FYj%8&DQ&Kud4?ah#@k-Urj_Ga%Q$0rC?4p|PGh6)FD* z*!p{N;_bi0ny2&J6HLT?7HWE5kH_RsgzQgK=3{4gYi_!g1+6M|QQ;A2%0pufeUW(k zcZ)wVk+exxa&&6^1Ld*i&5#NMGf+j zID_&}D9?4;1>%OcY(9m%V;lihNu;3+l*Rf-ihuuU{Bjk^Vn2&93C3aWfbmbguSDQfI4 z(t`=o82h{k3&;2Mcv1^Mr(=iS875Zy=XX3jDY+aVQO zqI_Ug4itxdVh6v3LA_p7C)LFrs@I7F0enGds{)HFIcr~5YHyQpWtH~JsOwz4vJy=`4+Cx0&#Xgv=r1&EfZQmOFqb^%4;He zAaQ6;U>4K|O{a1;_PIcefbC4|SJM>=(S-bc2ywrx%E3(MOK8Bb0q947SSgBZPyM|e z__LB<)F=Z5-k)GVR&m-;uZz47acyPWoL^4>$1H;ZN^ww$3ar4@fstuk+aoXdpsAJ# ze>@MEmN7If+g|%eNL+*NSFdcAYJ<>rjo3G9l@*QXaK?I^`Czwv#+cZG$4oDH#uNM5 z_Y$R_z}Z-~ItN=oXJ|DKvJN2%niuddY>hs+2P^d8v{*I_1cU(Uzj$c7DxT5X1Vk-| z8V`W;0dd~U_lT)O^6}pv>ufzYEsTf_wV+y2Hfi5 zlvs`aN&S(fHMK3qo+^hbfJwURZjOFK=)~?6PsG)!ew!?tEWxQ5tRRTG%{Fe{7;K$ruGMG=%Nx@8sU< z%GBd3wa0`4#>sBtXyBF$ero zd@w0qoxoG(@qI-!u9ABH=~3(QdR~(b|6-e|yS{XSZDrG-Oi@56hgL}+zgag7VyP`{ zl&v9{Y>eho9*VjQJTnUF#FO7`lKXqginXn!-+o#(WG0{x5X4hvWMD>g(cxB3Te$qd zq=vH7O`FT9&|*Np`_n^X-8}%|lGe;K8nIx8Ay}_n&wV^4A1<&Up;2t2;^}K+pJ~wJ{M>4igm}%@jQ{**<87;I}g~=HRj9L zbP=?8#WIwujh`jF4cf+8b6oe_@7JEWa{u0&v#b{gPn^jsG#@BX+3hdWC@|}ZFd6DE zwvj=ru1Qm+EguIZ-0nzLb$G$`uN#3U0m*I zhs1pG6a(T0^x-b>VX?aQ7Fm*R{;7{C^(!bHHam1j;IoZeek4P{KiL?ylJRVN#?X5D z$9uQ2Qd_4xws!=pO^0wn-r85)5?$3HW@!1JdTm*@#NO9%BuX z&`Z(n$Ls8N;w8NG%U9aI54rB?KtwfyfZZhVNmGPU#WTk>2W(?wIL`#g2+38O@fHIN zotizRh^%R+os>JTPgw)7pNurxBV=#}Fi{5s%+cMmT)OO3Y^uRTq%=|squdAxYFK!^~cFo_MlihsYiHCHS-re4aQh5fptit}56+%WFR^~_C1t@{6UulZFG;kDl zC)5CUGDde1Y)<*!JmWD=G*pQ0GAP*=R@zMKj~%Mi%abhjy|xED$^*#fhPDKopla5g z+g4axfDo$#-ZG5J=I0;4$M2olmv0(|;4cORBF1;thQj80a{DH9^AxI;vvkW7L7|E2 zG|WVB@R^z3{ImhDd&1htmCi*#L@FI4;&{uvdAX!whq%G>E$zSQZ}_Q`+^wH|OFkFdYthYb~&N1kDKNVhL2y-mXdkfLv& zfjk#Ht#Z6#i#pSCw`&4)2r{F-f{eMr@;06;TBgk|`oU1*`R z0ZP#^88%1Zp?ONVO;_n0pVl|bk0i;21jhI=_noiIRf2o9RMz| zNmC8zvoNROzO_B%N-iC{d(>cQvNGz+M8GP!bmx_Tb35}c$9PCCy>t6W?nh868^w9} z3MYvYhfb^sUa{|SRKO7M^d(+mW-g7=v=|4QIY>DpPBjI=)rLAT{k>S3(IMFbG$LOi`w(ZY!2+}hXtcJeN=_HzPVk^{Z@;}xN6QR^^Mzsf?afA&~x$Oit zW<;o2PYz+a>qI`kevO)5AKH3KR4&HF#J0-*EO>w{K}zzPYYiQ)WxT)w?5uz!yy8I; znJ~_LbU=!ovI96J2veFp^c12p5UWOXa>BOI~uL7_ga zn&|3>Hkwn8*U)_h&iBa}M2z@b<8C60(W_&;{kQXljp|4?Iq6qbdi2b^LjPr=0&YV984a zI&V;AzKkEnrCs!C>ruGhU^S%W&z!u;;1_>FWP8} zZSAxBsF+ZaXWAz;psMG3+cYjKs9m9j!ysL8j5?HynMo!pa`LLC=Y$tyXm{TV%}ILO zvL}ua#MyxN{qresP$|2_YJdPf&69d3HYDc#ItOWlIyy0x`JOBhX?oQ0$ z4=B{c0jD90^m$3MQerPb9@}*(jgJf$R&3pt zVLHvo^V#%mIj|nshPMloxM-i;I`rHG|J*l7UxmAA-!{nT!H|ZnJo!m94)FB>yE|K@ zYf_!5DiSYNM!$JCfP+5{;6gTqGqQO{t#{-(R+O@IZ>&kgmVjgNNU8aFC~uWs`KG#m z26`naQ8SQ9EmOPX-ONlhLd>5$#dVHB%t)M(RW&}|!(@%nUhVWJ_7crK{xmMXz`Z`e0)>IT4$aE>C-YjzA`XY|^1TpT}^ zFF0psipmJXd*NBKh04&8nKyOx10!r7>P1A``N$hI1P9cX*cD9O-a+@g4$`lFj_sG* zaas!oPNmxawWhKhEGw3c7Fv0U+5wKL5Qr~wpoP{ST|ZEYXaFQMEHug3-u#9E1({4E z!61-mP{LdH0v!xh-dzODH9fT*nL1ieK#6DuY~y%CSc`cfsAzl5q=UsUXlDbceJNl- z4%oG)C@4_Pq9sWe?DDoi#hrNLLKo*n#*9` zxs@elcANu&!X9%qB`dUvdsXM>`H&0KpQ>x&KTB!fK_v%HyxaU)w|vY7hO^thCPBOX z&e26Fy_tMdRPRIkbEapOXHsGrX0jUYZ-MG7A-twt-8&shY)N-RtkD66snyrXBKY@& z!+&mHLYRVAL1Gqf?^ZAcZ*#2(uWqRNN}pXx;^eosf~Ey(Zy{1T5r=zDYqUlok9ua) z1IaK9fhH(d5IY+upPcUU=&=CnkN`vuqNI`FnFKk3V5m@Mr)-CU5}D) zGlPWkIm5~vZ&y;-q6^FySNLzkF(`<#10V#BdB z_4=xmt6jc{O5(R02A7&Q6d@K7mU3Ga8cSX4@NRAIVLa7lkQ zvRxlLSnf{Sre1H_d9HG8UN}j!v)*{F+62f|8Io%(Q64Ta?c~9VMY>|6f1U(IPA{>! zY-+e@m&o#&^|Jui2)Jx_v;BnIY0k%J420_a<^mAY6J^yY3BI4IeiJ-LFG@>GGn)m` z|1Tb6U~@Fk*Ez!6+yWrQ>+?0a9e1s-9?b65z^UsHz9kCyBfM%~xA=fy_Me@6@DnTm zy6#BoPS0>oXj6*A_Vwdex=Zvp*SR}8rR1#O`!rkFcl?68~)Y``Xd-HbfC{a-tCHi0a~^|a~j=gI%PJQ%D zeW9^GgG|~H+9Ra6(Q9q!)d~dl4O1%(1oaW>RCXnK2&Yeqa|eCa^^@m8!P1vZz{hbh zJ?JrTARUsF)L#cpUlvOUDiTntzoevVyb7($ zvonyCF5=1IN{Q~5l&YI*S%H*_p-vmAQl@bFq&Rlar&K>_9%SUbWc^`THO#%H&yOP| zao8&1=JG7%R`l|+zgz!{YIbO5pr-qxk@pf3w`Xba8>!hgTx7|gt5NhA_w#;cDO?xK z?>)yYnc521tUjO|Uu>>S_$mfaR ziv9h64*ir18Z?#YF5|lWfBi)jGakNmEbJfd&EE~!L4({(>T<~6{g0pdDGg@&uP7k# z{40t-o8(_n{3{CF3*-OAUEwZp8UU=}QPGqsxcI}@?EYe|E(!9nZv7t`Bn*M6GYGEi zG0Yol(Zg!#Qo#{C$47du%?v=Ub^dOr{UcX@*8Pyq;lX{Sl!I3a@x{8a-L7#0&}S&* zxA~gGio{X(?$ZZ`B79GGgced&YLH3kj1S%$K&AfuKcP^kc!Gso;1~WLkY3L5_sNp{ zv(JC&Aost{>|11h)se3`LGXK&1lYS$QEXqsaYNby2e!tDw$3UP%(!p~GIVayT?e%s z4t?D7_`dN(>-4479N0PJkX5qqzD48jLj-^1rgzY%%K%;;Byas&U`FuE@*%fzv=_|x zz(h5)SVz#@B$k&hPxBE1KR^wjKyr#Xk_P6D&Kbztk9*44C!#C(An z5|s6YygxYcKXSNyFw#he!d62dC|YQsPi^o7C~>s+Pl^4PQGZy2!bR}K|IrBr9#Ohj zsb)|H#D@ZyZx0DLqj3fG0dinc_`@-0N4z0HbwHC+ialnEEB$~2>7U-noahIev|-=5 zC`d>DYajjR_Hci9>R?BG?0eM=1{x2&prrE9UpH`f)aJqJ&*-Uvx7HDSN-8i?O2sq# zRagE#gs}tZzb);Y=Ef)a8U(GALkUq7Zh$_jY=22NzUw(_LW8@;0ZG4|xzQbGp(w%* zzW>xw*jkAM_TUZV(NTUj!;G^}?tn)z^ez=zgLe<*Ah5dMqlF7WD4b*N@rC@y>?=%4 zQ&OcQ`uY3Y$9`-Jcl#9l+CI~wCLn+%0&0;emDn1G3p|bQ06y0SCu$kw_iF?df4Q?j zC-46?@I24;xH(F20))GOPdl}H5Awp)hjB4l58YAU_(Cow zwKCf|pq!yqnAsR5*skL{DGNp#F_kf7w_Zo7QL)X8j-|@6-*M{6H!TFQl-l{4aKji@ z$EC@DHsCr>s;8Rxyf`r)oS0~Y2*7&t>~Bu{a{_;e4F6mv0@7f|L(z8{mlP_0uf_Qv z?_Zpx7wX`=ddkFEBulHv>b*9=xzDrCk`v_`w%CX5M1U9~b^>Jngn>|i_pa1c!1R4z z4xYyKu|mLGtub6!FS$ZcZWk0w`Q#MH+!AhWxHJih8uEVbjg6)t*?(en6n`T7&$j#Z z(>VHnSG#Dh8!M1ohR0H^JQa5boh@|bwM_a`W9 za%E|<>BI(o>$FTASM&1_Vf*>KyS-&BY-$nU(ewj1-dnJjn>4 z87S!$n_r3-K^Ms98)B=ty6YbgTlY*&c#k1m_3o@jG z3W5`7oWDAP&|lIk-dX-~7(6n7>Cvq zE;3raMkl-pYiHeto@F$urKgD#qm&{~?<`neQADiH15XdroYgyG6zRIX-~|;?um)Ax zK}MR~T)@vkWVGWA%N4|B4kqP1BgL(CKvt*;VGvA`_*6*5Ep8Ynpl}b5tQ2JGHHx&E%RwQbON zdwroTQ%4w^>c2!YpQ%;YOhXtoG_VWm2iye+2Yqn!&X?Td(KOuw)i{E|<{ccIBQ03@ zmQX!XvL+EeG;-kQ!IWCIHLxP?tO zLO_V7&b*M1^sn=~*h;TNg+2Ei$0a?BNWVN+sZ`aEbDTUc#5JK)+a|==g&X301Q{q_ zwZ%nkr|&^ki3Uog(Eg^}8SOXI=sby9KpfhzO|h>;{>2jDSve0gLW1*}1Oj!dnXIg= z)*zR%G$A_wj&xsv`NZ=RoS*z!`G%a(fawQR)RsbbWkhFv(ycH0#<`Xa03nFz=4k}_ z1n*Q+mc;>9T?~sBoB>02;lT^eu;mN{7Fuv zxhjFl8~2LogD?SD2N3wzzPNKfPB~@lVsw7`cTfbz>2rmVv*y`v!p3@eBHp2zt=JWL zp$I+rimZxRC3M@{J!!HVDkWN{!>N?`>Etms%Vm(`g zjHi1GE%nZEKX5ZTf&?_{0;S2v>v$bdM?^c$Scqp1*sS?lVcS#u z2Dq>YkFCJwyv_Kj)Qc4WHaE<-RnL}H2kO~RaGFYGCQOtYnC0n~b~&L~1z;#-{?|8S zAGe^|_IUD66&afOZF0=KJ9&$S=dLd9_48#GC%^9>oI8uqFwE1he@-|=n0`X{2ektk zt;jw38!HYoVaHl3S(YZbtPu3wS$=1@_Hh>O`&Ra?=oA0o&wL7Lv`3sUloCy|4BLu5 zjp(x?BaB6~NmsgYr_=6W47m!C1V2I8t^?SGm`^HSmH-N_uXeWb`d63H*Ua()S2W=k`ekU5QMoj^|l+%zv z5bGV9*Gedz_Dy${+2bTMBCKC|s6{NK8{OS*(c4z5hM1ZR>TdQvTY! zGh=-~0(@?zGb3UQDE7Rmz4b8#$M}|o99DjQV|KZS>p6vjdu@vxnw8oqcd6Ks&Ra=J9kRpTov1VIOl7i}YPPdX1c#!JOcSI@a}tW!#ShTSbB~ce z$zS&3bZ+n`V9ew?77fxtp^DtA$$$tJ4pLAZj**de=dU^x1@Vu|admgCZj;mjZh019 zs6H)S17}7C3TjRSk=6kt4+dmDOsso@gg={b_J%LZw*57 z%NCVCLf)luc1gZ85=3_ z2_5=z>H>4bbt@FLt57>o()vKVtjfT6xygBre{KQ&)azc>MT4R?5en*(ssw~nQQuCx zND9MHZEOQ2b~al{`Jbad>8Zt-iD#NSHwT7vBMgZk<*_)cHfZupvHJrz=cS+-% zf1%@taylW~?s+(b2AR5w7{qs+nNXV;uG}LsyP`FTKNHh0*y(pPt8CS?aeda*8K6@Y zf>L5>=&k5ZkPtv&QxZ|~P@K&S%icfmmtiQF3Y<=uNpRV|4OR*)iTUUWdc}UA7+akt zn0buVGtX~;oLFW6`F;9D+gv#eVCi&QPhFtau?^Vb9Sr1{ueW^?Pw&rf(XXDjTgbCx zN?1A=F~v;l??*4J$^a_3r8sT?kfns{OOhC148l^P?ITJx^+rE(Ep@Q69bGKIFoe7y zChy%b`Vm*rZ|A%zW`C0A+AWuS-ND&^gpgQthND>3nxhsQ+aJ$4?t1*s`u2`^g-Pd? zR>~TALDF)Myi2vf_F(#c`j*Lqoy7*OQpCH^xgR^%uF(tGvcS0QeLEw-(*jvw)|>ZT zccSdf;&f}ms+1l5Ym6+-2UFIw*hEYPFe)G*E%7oy0~J?1SO%}2MVIm$jN6YIH&@UN zKIeGwSC@#!H$S)Pxoj{!D9s(_h#bxE%4C8bl-Rk;q%_wl=nb%>BX$X00c%}au_BH+ zc}A_DMLB^)=5IKau4%lb=}gk;>zXfq8R59|);Px=Bb4UrY=QR3q^<+8U+M3sYU{Bt zd-WPVZU6jo$~kB^j>q=LLd3-TRYQ8n04cJ@QHpMEYE|7nUygfTV-;5{zW$;I}LtQ&PDH*gU=9*O7H!22OlJxNUo zFhLZ%t&}|Q$YbsG1CO7ixq(vt2V3YpZg7pYCQPswWI6r&IQ_f=8iD z_H!p^M$<&wdSE9<-BwPw=9te`CACH%Kb*rN)Zg1yaSqzsirHzE+GmuvbY*Cn%U~CT zEISp*jc!w06pauSij}kg1m^I3!qlkjV8QdPM}$~BbBC=Q^S**$Py~*^Of3tTG=g#| z>SAUDW`(+KGZstD(@*8a(rLyA<-%haug!fA4$WZ5+nebvhbQ+*qpRd~7;X9(`5xJ7 zmS!^m7obzitv9=0a7)9IykmzT>iLx8sz1(k>3hUqGj2Ws@b0pJlq<7^{;t9{#w z%3teZ0Q8Co6m#9RwS=Rm)x3Y z0!_IYQ}bRUYJmdlzUkzSPdL}9LFr#;+%B0;xjbl5yd)FKevBk-fV9hJ6N0R-rY_@K z58P}RfgCcbS-R#}##ucYUsnKTwg#3dky&%S+cQkb%C!eS7Qza;4p2P`;(KK$b~2DC zjoRt^1+wQH)3pqpMLQ~Eq6MS@F12}}BMYH`Cj2;61n?ykccG{L2b+iVIIgNonZW-A z4<#DcJgNn^1EBH&GOgR?b5}F8?xM}Ll0@bLh6a(>gU!1?z4!U+>aDw>>@x$GHb2)d zjMet-5fGO;MXf$wo^Dn8sFdV37Dbo=--X`k6(8EJSdOL_z@^o|qnPZ+ix2p<1<$s@R&0rPA${6?Z+ap0hH3YltR}6tUff4K$JM!>=+2L#D-+4!wz7gt3Ja?n z$lmgWkPFRWc{`ni=z|~J8l@J|J1P?mp&=IEqi?wY+5W-`?VRv< z_(Iul7=d^|?FP?j>nqhYYrqffE6;*=H^(-Za&Vv!W6D~B~dEi!b4<+ z3bQt!MZXbaTGGvW&5;!TI8numG-RfN0qyd_O`?3|;+uEOJ#vAyhtPHVIJE$16udpZ z1Nimv6joVq9qKffV&vfFbETDr#>CZL;_|bGQBOUCSrl(s14MLqa=dk8*~ZZK zms5TrfvsHZx@GT{^Cg{Q1A~KSw+^sJ?iKQ39cAjh1=2fVCy2=zf(Lhq0Jhi6&JX)z z{`^PPm36-(^F&W5NcQ1>wlwUx{?_ME{NOJ}f985*n5uXT%M^Cak&I46yUSts`0!!_ zX9)4UWx#tz={yB5P~xRVjOXTAu8mgABSCa+=)AGdbp+xOBt`IsvBJYq&`0pH_C2q~ zYgcN1ir^JjQXBRaV7fkF@j!kGUg1cN0jIxLUqPsd<6;rIEwc3w2B~z0WD{@4Nm?{b zpZqlb2;~1HndHww#McI#nO8Ov76uKig5#pUS}~ z#4=KtnrDu3e2^%x)He7;L)cR0BZmLcX95EGL^^+ajM=M1~1U(PQ%ZrfH zIm+fSZf$ERy8Y;Y_3ioA2wv0rRZJgd(0=;IO>kctNwqw#j0En-;A5xtf@6XE@xEai zz@&1_7t1V0IX?N{xF5;z7l+xtTYo(wIL1m#dsWC*Bli7~?-XaQle+I^@iy98&Yh#dTQv9K>b} zm+$6^flm%6KwoD(X7mrJUnuhfL5`oM=j!@D$p=l zXcZpDFqGm#Yn~r8Wv(NwH@@H|ZU#ZxUdhQh;ljr>nD*xsqk=d&1jXXi_pK=hoP~v~ zDQKMi&Q#&h(Bj#lO}t4qiV+WxL><+@by-&o0HW;;iSZqX+9LP_!WcP?6mG(aYk|W!!ws3+E43>2;|)AGDy{keBnNqthJj z#eM9;!RKN8EGYn3l+6mwiH~=N3TKz-hwNWZj;aG!pFvU42T)*YcoawSNAPtIPAFDI zABx_-0-Ax`kA-Ie_oz^e9UPUbE{|JpfR>ifaxNfM1cY%WU-#kQ>yla5w;(s0=srX- zz%BXE@c^5G>>yM$Hat0Y2B7)0p-v~?wA6kN_Xol;Ok%D_z}2R zLIEhC0E79xi1bqi&tMy<(|2t0EjWq^cIu$dfFfEW=sqHWkeKjyDYgFq5QH5xJ!bvF zIs%~1BB41wQV!bZ#rd14#8nSsr-q&MU)CT^3j*%s7N_lm1Iyov>}@y_z(psOeta?4aB{!K*sE|HOVI5#{*1?%ljy#p zzdZPM`5d_9_p^fgwU5X{L`i>lEXlK2#an69?jJa@kqJX$=)hY;hvULOkq=PFK;4&s zH^zu{%{fD#(&l*!UgzII3L+G(ZTqMU z7kswP%{3$2sO6Ch;a9FP1JMcc`{LdRb;lE_ebEK)Z+rEQI8;9;OaxJrKpsPR{KY!m zQ<{*<8&z;~6WPsVH9Y)WcALRJM_Riy)xzbxI`{tTn;GhuUFe!_JqvD_0)Zji>u=_m z{$)CayLU&KqG#Uc?8lQ}d0;;nQFq~pTYQ=D8m+IXf*eS3PEuE1{s&%;G?$^+*xi<(kb=ASpIMt zzbr>EepzRq!s#iH83Q6L`1E0+Rk64?dX7aHgJ}JIuva;mWI}`CjL176ZTv4iFa zZ$Rp(P2UUc$y0Ygzc)FrL(2YdqY*!OY@OcMuq9@ItaIhevAW08x7w+oZQ4Z$4*SmW zR#Wf*fT9V^J?3GiYtFfIfDX{-kz7gzQjOPj zY^FVQ-rT354>4YS^l~Ssxh+XXW@{BQ4xD-&Fuy^AiAd;dE`dokReDC@{apya0OI_S zrRUmH^fciNyZXHn+ZpyfP*3<0E9>|9RX4n&27pN5eBlaKOq|cGo8pt4UfoCDTIZEn zjz~W9KzzdEQr6aL-vcfZ<2j}ekt{<@)USBKwWD5oVfpZT%zgdz4~l0X-@!s){$;;Y z@(Uu;P!JP<3nAw9iKhn5L48%=xqn}R(25mBPJm2}{JxHyE8utqfZTIQ|JotUmvfM* z5O8{$|AEywO^U(c1p>SNl)y@da?&ji?@JQ^vQjY03@(FD==h73Y})FBy1KjVcR}Xi zy;<{~oQ6dzHv9rXbB2P|J>!WMi%6u?ij^P#L(eeV2&<53P_XbTw2S+7HRVqpmiNmu zFqlGJL$fqa0QC1Jdk3iBtlKmH+Btk*gH3R7SN@a$$ZBdy8k{`;-wn9sK!ZFsNmm>L z<#!*?C;`aaBS7-d<@=w1%AN){C+1D79Uj9=k1_@Tfgiq z{fpmSR#5{Cx!zy!88EqJZT0uPFZPqJKs4uPAXIIm%X028u-PKwA-6hKf3-auG=yRZ7<{Q23RlXqV??57SL$!K@>k)M{OpX!+>8k`VLz*!Uzf=c1?s`OBGq= zB)xIiQiWv#lV@ym(Heva0lmmjV-yFEVUur)@?AU5mD#QUw%JbU{1_@6JIqb|5>R;; z-`|Y{zXIN=NWL9bM&!0}chY?&C8b*~&cFb?mFbrsqUki@iOvdn^zBWJCTg1euwg~= zH5`}^Q93`QqMPRjN|_X5L;_4i&-Z@gKF^tpW&UVaBewoC;d);uw@8M#NYjhmzBqSB zcCSe}nxO}vbW^M2r$1F={8LM#vKqM+ ze$C(wyH3e{V6DN@)Urb91g*QBG2Pj>Blyg(K)_8XfjS;_6OMSYqTjYdk>;s^B<0!M z0id-YUUb*@W_n=*)clRr5yzz{){VufpryMZf$a)@m0=<#84Fy7&qChRANv~Ja$i{A z%HwpPaO^n|k@!;nqSOkVNRfEGTVpu#^#6pfNL~kjOT)CYwT7s+nQUYg+y40iAa?q7 zs7)q4X)XL`+Ee->0TlB*F#3=m2GqljxsQDDl7Wzb7uT>uTp%!#69)ICW7R3Ow<}zk zAJ76Z=qXOaP9N#D+p6b$K^)#`Wp+Y0UO{ZK z0id7t&C$Y`3AU^n^(56wpR97s{@2%jAC6Wt8rK zc#dHU08HJ9bo!NWY&CsOO`w)!wdv6?jw{(UZ;3OvEx#YDi*E53HmJUPYmr@i zitVt)CPm@?fq2}YJ~$iB;*2CFqhkRW-sQ}@yp~F^h}5O46&K3gL>wY@nag!GZiL&4 z>4%H}w(vy2S2CXzal^R)#0C3w$at4#)3WLTX$8h2LdY)ly5(TMPIDMn>>Nn7*98?H zEAA|E394r8s_4Gk4%}Wron71d63(ib_bRU>%Ogsy@?YQXs`c#4UMN2c7_15dk zPnkK$>0cS5@lb9uV{7k~i4NZTu;{UMs-Tzeoao^EUN;FWmitN*{clkz$itLqaI*z?(LXi=n zY{_0_a}FmYD|VDk+!1ddClrui>`5Mnb2F(J+{ds^& zivv0!BW$;QKKCB=iW|jZLksT+i*U$!ZRa93hfug05hv-g*bw-ziF<@_UEW*A%r<;* z@+A|GUKv*y2R3Dxl`?2^uA;TSy(2~XPza4{ z?+nO55BfK@@_xfc?rlI6i=j`$&TzOu2Qn{eWCMQdqVkXB5YTbW>0rX80?=*Oi+KM; z54nF(JwF0`rs|1z4%>R&Bt)Ryr%J8JJco=|JB!H5mvK|1Jq7Jz zL$2_ILmrT9ZLwKKT!RR<8L`m|E||9q&+IcU0NJo|IGX0i@kJSMaul*(PIjDKO4Cv4 zk|X&@?5>q%$v05O9F+I3x$Wp##8yHlxS9P)6Gyjk!y@Q4JkxutgAx`shZTC*Eo@2^ z?XS=ND+jb=(hepJzTPkWi;lhm?~jbH>-cwAc9hlQ_5Y6f|Kn9FLVkeK$@l3D>1bE3 zDhI$B3y6(MR>2`lB?w$t*~iuXAM(P3U?9RAK(>aK+--VW^5!kYer~Us#b_8kw$0|2 zcjq8Ymhv2Wp`5fj3ug|p6lO`=+408Vf9X*e2-R1INFPpb)~-@8o4uS#`c!7ISU8$z zTjbs3Xf@!W5=Q@yl<;o2@TrjyeC;CdvyBm2s+*EUf(BLX2+yRT;l zvUk3r!D9pWw*KZt6ne1ey3RnsrMKDFC$mHP;BM;-A^=wVxYl2UUft@vfe2yBZ%O7r z-xe;p(CQdg4(-_DS@aIFy*<zGy-2Gd{sW&@yZ1-yi3J*F(GfI22=eM5pOCUy zi6#T23P20|Q@&yu8{E40ns?e%QEvjdHJc1@V4tCJ5R*yN2eNI6cHKO3%{3?4DQp}l zdQhNJPSU`Q{fVxpocV# zP3XNU)4g7Qpf0Kpks5FwbQT9R5U#oa9{`X23~_hK4xwRInQ078d2rxdMgNQJ(?C|D zJ2>o?SYqMTDNcTt|E$ek?zxrSOoZ6v*E*WRh-1g@K2rYY-ZLf1RRWmbv&9G_}9@KAjN!cTT77@3; znbbhJMF+zHK^#?Hw{7hADK#UKPe;nCkMYGS&!}AC(_RyQVg=|;kJMxJ6CbKxurR+P z$U$13dyM(oEuq9JGKtL))u=brv(nN9V4RX@tJEZ(gI#uqU6|wggwunGZFC7-%BuI* zc{GWX$|QC4dZzl+oTQC-F@6YEeI&`NH#`0a?*+tGaJ?;Qy4Q!q3`{ZKxJ&i!c@~Dw zN6Gf7iobtM<8^`KPB$!n5zAwrYs5+vPv3#CL zwTagXm-j#LQ_D7LV2-1J*M(O{|Bq`$YrzcEi!xlQd-tYFsKGzOK^~RZpdmwUVevU1xbjz6_f^mcb7^ z3*7}SzS9q~0FF2rge}~IeqCYP=kQ7t>oPbX9WyHT5Lk$HNsJQ#7%gvE^{(@L>rAmK z?B^HqBFEtNF8R&_&x{@3l$Z}UWV~HyPy2~XAK-^YaGiy)BD=D|NeEcxLv>n4t3&nD z)5y>?9pben#dl7g>QaIU=a{9xAKh*@x~g#3%f4T3{}&-}dE||Kj$Q5KC*dn{vR;N? z8XqvWW2z>Y945Q!ukH`5e($TR6uU7TU>M|NMa?F`hFDu;EP7nG7*WWy^-X#u z$}g51DKJH4)L%|8EK!Hnpt}}LNxL%6J1n217c8o3Kt!m&`{Ti9xpG5yH3eR+(${b` zt2?#+-y49;)^F!i3+2-@txMf$FycPRLTIk_{q-yWKqWKAdKqOUGt2=Q1>?{^(R-J? zkQz!ZdD(f85d?50P`G`(vut4XAme_p`IaRsujk9$FOh44%e>0zkPBoE*NGDMNLsvS zr^s;i^+H$ka`yp`9QBm7V={Zy-s^Qb!h5dSdS#ZQoaK?v9>WI8%aDGybpGIA^wtSZ z9{y^us|s1BmAy3{-~*@&ZXelwd^So$Ispbh7Jv&;NeBwhjrxWTCUc1HSB}A4JXjbq z^&nvRJmVnV59C|t!Sbqd2e7pw>k-T&GN$r-tDjMc>F1-kS6i9ksY#JLL%m%#YJflF z`eARZ!mi&|1>hUFVfFi#iKsf}N7St?k^VyjnEzW8Q7Ta^wt$(9bU~Rk_TrsLt=CGa znOtd&)RCBGBSG~|SBm6AHgw7D-#s zt3&2&=a;q#%&?QM=O^FKcaf|lrx+1yg;9%EP6?4QQtnu5KZ}v63fn>3t5Qc-(O2CO z-1YIY67aoIjV}*PC09d@WMfZ!;)EHb z>BE>P?mn)%E&6+TpH7K9>Fj`G=KW8}T(-qS;liOj0Ib`whFT@Xf9NS?ZqU>Zw07Vr zGKCMh4f0mIyeaO#03R&S8P@ZD{q0nwq)kuFV15VsUDu$~Eqw;Xo2(~&0b6Y>^D*EN zDS3i9#NfM-$pH6yj)7O>2hCx{qtJMZ%@Ad#*EXLRtomYj_3agN09B`njk9oTEUnlG zK9(6SGBEtKS8~?4d%a6){m~$m47W#LDW_zg4w2JV^ zqP#9lXR9TxHEl}^Kd+yDb#^U4N2v{oA2Qgf{7=HhVi6rrn%#Wi0FtjKMgXKF48(iQ zQY%rX2p4r=r&ETA=`Ab7tl@h%F$wWEd2NVUf$@WxcxZi{u5+X%Eae{`T(WE(v(` z_ByJ=Xy{Ybq@Rv#rJ`<*SPCpoc6HyQ6dpcgfIZr~gaguvYp#-fzT=vilq%?IDXW=x zM^(aBSf72TpE?wEJ$T8Jp>z7$0I4cUK|)@1c+SnZeUR$sc0X}J8$puGHo1UEZY$~B zHl-=fhg1k5h5K8SR=)>_Gj#NKvyd{>NM1A9+z}XdYjqisjS{)-Pn!!mQ>|3&e&%uR zk~_nf#!7%DYL?;XofWrlR_lI4B+`-1`dz0Vbo*2Oa`M@(Prw=~VEuz>0~8c5(6$CC zy%n<^cv$}K_1%+Hkr-4}0+2P8Oq%E}r|g~XogAz53su)=y5TYJGF9)J^*Bv-X9vJ@ z-e|VZ)}o7JqvT{|$DS!@7u6Nr%M$xIcpNXqBxe7zDud(%=w65a>Uf=&iaiAfdU1*D7zX-}0A=&XnFJ8L#c=dqJlp;G=0w8>`ly#D<3b3mnfw(@(O}6lGN0 zwRv8WIf(a8R#xhqazjDgBraGPiGvwR96C1qsa*I`(5b-dS1Pz`5+#?G*3dKn{vH4b zQIl?Nm1tT@$(Si-bF@gm-O?kQ3ED2VqMOCn>qL(2suMB;D={=P6^JlRl?QeM-bkHw zlXafK>0T0H?&sUpMynefB}VW{-zm@anrUy9I>c`D?jd3;`^3D79>q*YiJB{XvMnEC zl&A3k&`WAvyvJ#30n1dVLJ0m0H{W?(PQdsDRc%+7k)+Iv1UAQbD?JMyFkr z&q@*rRN_`7Ykx`3z1K@7aGMRMhi?S(A-HBM^~7-A|mjK74L%%nmkjD+&zU3zV$kzh_OE0lRa z#om4A_bS9DAKhkv&+L!46!=fBCrVD0c75S|;?PD-lT!zIGT_^P_8$Nkvqb)SvfB@Q zqt4lWT>G%ygYWu+PpuHGR-Yuy>8l#5ets=wV{^r7yF6sM%X$UyeQUU+$^~h}(tLOH zs`O|&M83FjufK7JrI%!8SJYS4dkB|teu%GdeD$2>a|Nsc!Z7G}BC4GBCZPQw8omvg z`2#*pBh%;_O=}8z!9zR^sh34H@I8;SI7>;`!99B$OJ$W-E?Xg6bJRNhNI$H_u~&bM z5EOdXrW3}hT&%gV>@V2)nM+??*aGN3_a2dD^5kQgl2@acyDs}YmOux|shw$6B2MY{ zGTN+}PMI_{Y$f8P7Sz;jt}$J=a-wbT2A(*nh&u?2=92UUk}J z4u%~w8~-g=6COW4g=2Fk6(&12Zl(;G1i$m-+bE&a2%gghECRLnszSv=V)~iyuS?XC z8bV8_4<5wG9DI|Pj#t3xdh0FiEu~f+@mXc~dTy8Kp4vfkDX+X9pG4*)biXDa4D-5ixSk}5CV&B)HPaxH> zME3OQ4Lzo*thn*+=yW<{XTXWG!EcgSw>A0Ph$wrwp|Ec0E_;`vhK=P=L&GRXqzTYj z>aev&C81U9wLmlSvAVijNs9fJ`s&tfIHa}!(6!m!h5fyDgPa2*{~4~uqK8Wu1uYzk zwQ}@X@y7kgD-PT+J@z{7rDW&GGG)ZDw$i${l07=y5FZ+NNexgi><)~}oz*QrLCJ}R*pQ*F@CU9*V*d;-Oh0W+^TEAkWdPMS zgHz|Oq{ByItJrFb0ckRH9QXiyKFATVnJ!+M6+~UModQ>kD!{XktAVb&GJU-i^Ihg> zh)KO6s4g;X%!Tx4Z{=Oz$x%zP4DIidgtaCb%d;I0Q-KV(?1P}ekUe+ZfhoYLxZPS4 zbe;35@;hZB2a_*G;hPK6{Euq7#$xZbPe~C>N}7H~Fl!RCf}G#&_D2OE9A+A^)&z&+ zPBa@8KaqEBid>35rA}@G`eH{(M(dQA2c&iCVFT-vfi}H;SmNCBiN=oigC`n+%w%@M%NrpIvh$9hmrCmGF7^eWy^@ zq0y0uUD1Z(L8bYmi-O6l$IH_8Lfz{vuB0v1gqVYMDLizWUxVx3Fep&-XO*~a^Z+q7l*8-6byYVrOinUS);%Q8;Wq_9-4$v%e^3$+unQ2VZ zpq|_V=X-8^TnWaCwx6hd8~KD*#6px~xKjAtkp2oG=+&vk3~rh)K$u%c?CWkw;mXPR z2Cn)+r&mgo;fw{G$mzCX}kZ2r{8M>F?w*Y97C341NQ zb;rZFat(g(7Y`q)sof5xb2|2bqsi|4yhB>C7xdz#WV$C#@pPA)2cBp=Yoy3Rc$P&A zKdqs>mOlKX)rT4{d7U>;Hy|r-msp2B9QN#%v|BF|vx0TWTKVfZ4s92&l%3y`MiQ|k zls2Fn?CRo|@o=q|VMc@4tL%F=(^^Th?mBq^hd*M~w-z*IPbY}aPs6G+k**(%gxviW z9&^KUeD#t&2tQinuvD*PIIHUSQjmlSILr+f*utIka!n;~m?aI))Zr-ahu*%5;La=+?#;mISAy-?dIG>1Hlk4#7pQ8%JyA;iM>BVojXL%)tTz&cC7>4`?TbUv)FNM0yvue~jR-K7)SZx_}b(Jg2 z_(E`pV~NRNh%pcx5Knm32@Ekp8oCR=gi#vcnW`pdMm4Tdr!M{a6gwH-hzU9`+3JcN z9|hb{fq`2(5XXXhsX(0ixzBl{$92PVrEGi_85gfLX9FRH?!4h`fJ;#*aSZMejq3{+ zKYtfgO_^<@QJb3x5eiQqw%`S?sQgV?Ak2dmLqjBy znkbvgU45#yTzS##=R0NeC-mgDX&YJkR;|%@XLV5AA#+vR?wI_+#2_3{k{`!TDV;oW ztV!NBmpBWhkG7)vI#`tV`rYc%k{V$ml64j*KP2nt9!K4(l;T>KyB zt-m$8D@EF*j~!2ohtMX8O}J=Q^+!sD)XtgUWD2*kZNwhC1I3N=@NM_a)dHLxPrFnl z6KdAMa7l;gILKL*_YI-a=WU3pz{g>LDDRBYu{ zIa=uUOt7EWtW8uZJhW6}p}aft;6hr|A7~iKQu#qj954g%_1MEso(H<(1VjmQVHd zJL$|&ZL4=2!Z+TbzK$gMF;QCW=4IE$z3okpVKOWo`O=`U4w8!Uw74b!m$zxJrR!rl^&klL9sKl4YvjrUFV}DS9t_O@_N>rfZ9UR_CWXf zJ9+6(>qvN-lS$;C_t?d*phni^2?no`Q+bYD4r6a``mviI!@H_g(qKaWE1#i$bq*zV zt*}To;O$gKWy^Z82E{&J6Na0T^pG~xcKuP3au^$93rG1fB%$3s_gZ^xR>A>M(y>XM zMPn^Gf4$?>{G3p|aGfPn^ubjBhFoV8yIf^MGo#49JC95*_1W6ReKQxClXFbD{iID& ze~3PHD}LFGC?5}%=`{J;Ga)InudRY4?zrAe?Wxt@)KMek9b~eL>-WEz$yq%AkG}o1 zk^MoX=mO#QiG0Zun15ovTlSz34K@Vf9NCe9dR0GBIM>oJ|zOX+GyIX})DfzOe{ z6}R)&dl4<6b?pKB2#=4>r^FWa$Bnx1m&WqNzI+)k*dOW|#4YwtC$n$ev(M8sI5l6Z zG^pBr6j|l64kT@yG8H3ihXnpTaTy5|XL9xf&p?^nX3yA9)gO%5mhERx(MrzqLKPTuxoJ~&xeZeTHG(^zl%Bl(!9b829~`}$={CnMRTo*w&~up~^u5Q2dc*ODMTlWiy`B0b9{ zgPc{YC|&E=Q{}E@WI95*Vn;J4x@6pJf9m`@pO%c~$A3?#S)ZlzQS3`BCC@z@9?RBP zg%&`KOYDN1jA~tYue=!(5sp%gw)O?~PHK&72s#;TRw#W7v`oT0EIE@XQ1^BBI)8F6l+DY!*8Z&HXqv`qH_9=wzz+|s$%p6H%@K0& zLMJJ!L6OY^DJ6%UX)y~~r{=Yz-7zK2rE5Ds& zWu*1_B-oMR*23=ZbLCcFq~4Mp=BG>~6pqz%Eu&9wk;%|!xGQ6;;B{!@Dv(wfTf7yepbZXdiht<$ z(E6fz-U`4q5@5+R68UjbC48sU$_)k#^fw!)9DZ^ ze5+Vx?AeP%Nen~PUT;gECh_2@QZtS0qLfxr-tRiyy;DQKv0BzloZ^) zs8iw)r*(#2`#uc!g~8Jpwuyl56b^nIZs$lLh;0q@Gt15QQr-3>8Syqx7l}=|Wp*pC z&ZDR5(NU zWw9ovVBg+M04^mek9{n5ClV|6=lcDfbku6OCyvfc*&T*;-28BO&&ca}Fr&bUs(xEv z0#wc&BKpoM9WxX(zHXZnxT~WMpN!ho5-*6b+fB-}o1cJ&m%CZ6$9=gr^l~J0E)U9?HtVr%Mv}z)L zGTuwIU|3kw^#s=cdE`NfRP1dkmyh){!#bZq((7bL=|SK;F*JAW9(-$ z$}YK*q)J_i3Hm&fle)HZXj@{~`0w$uWc(rG_siQw9$?w2kpoB)ah>yHl9YQfv!si4 zEKY!FjUW@7@3F{fZ5z-OU7E`x0wEDEd;Vxw`u8AyOo%k+x59Q!yapCVv**E7su zwWDj~#=5_hL$nm`O@n^VnJvfsXX-AIgtiLzCP?K!d!92oD|aInSOK{(_NlY)%ajn!1ewfeKO zoTFI98tT@NTwhiSWJhEtsKyr_sZpg*G}Fo_O?ewG35ab7RryNns{J87K2`=0Yr#{g zs|MWw;y0>CAOAQ-vSM_{Yy3$8SgMGZh|QlhCFVchUCr?G#Hx#GKa#IVh9Fa<+F5u_kYrQ6@6CO$H(~CX z@?D3wh@frK>P(0zXW~H8cnsI$*GBm=N3r}uXU!g%qsF{y0^Msc7|CD1L${Wf0S1xY zuL7Cxs`X@SX}D6HNCybc+yh*ktaY zUKA22Lg24Gu==<%}bdbyk%J5u_)Asg+yDXRxd26?g1m%?4)^bHUMS zDq*hQzBz2T&ypZNTWxmKM8o7FUt833|KCYJYn1Q@Pss&v7Qln|0!c%;*GRs>UQh`M z3)Q8sH&RcypbP44{qmkk_O&*>xKp2+bcJ3^$cF-U-8x;# z`%r+f-Qjp1SUpitWaqcOj&)Z}^|+QNgJt&l!Z!C*9;?H2YVDCq3C|5VtXz~mK9r(% z2ZF7>`+AO(ZkJv3hHHyduyesjm76|>GRJVlgBvGi<*Jw0_h+-2or zdmf9|#yyL=j(->3umGRf?;LIUU;y0ZKi9rOM*ZHWWO#$}GZ^*(J;?eWZFn_GrwTj$ z@bO{JU1%(Zy|$=oteq8TFrkfN4Mke4x)7*3!kC>+gm3VC%Hgq;#2AYQ^ZLapZ1WYO z;#S9OXIczkNQ|++Z=6Ayo7&#AS$Mb4PiT#WSbbjKD`xobn?IMV%3TpGLFV)Ka6F%> z3>>NLT9!!z!)%am zxAl7X_K0HUbj*@CQK@)7M=i*i5;O^yNb-7#Su?~i)`w-lhmTF3; z!5dXy#%-g~pbJx?d)eGznGk-D`4 zvDZs-b!PM85h#GR!ilu_^Eyf926tfRS_TWGZ7wmH`|<2f=4$SJ3YoW!*`cGAX=@7M zVR<}OU}O-dBjM09Y=k5dRvAA2Ufm~uC3rra53#Xi%3JeEFu2;AweieN>rvyMo1=oG z4WSD1Y>Ubi^c8AH!msChWh-c7Suf++hQV_1SeBUla?^8JmhI#`SeIv8%34Cv=3O=@ zzS}ohN?u=E`fot_|9%Db*@^jJ^AR!=^VvO)^FJ`#m7R-2;f4pXPW-#Cv*X^qP zcU6pQp-)A%F)woNM80y;VjTusOyCwQ`6iWKH?AiXF{5i2TW)=*XYFkkj!WPv(5Szv z^!us_koN>%w&2;z1Q;;Z74l3k#gciAt1Qm8RTKwG`HLM;RN)}}N%$Kufs)zBmJJ>s z#Ufh!?}@vrorTX?pyFLPV5d7{<7g(l5dgJrzmY+8mm_BqD8c-`KA&lU<(ZF2ChJ95 z3q9xbK(thjN()uIKrI4)P6h$Pk0J9m9N5R^P^Rvc<4zr|3SAS?K4QXO*{0fx&C^_O zRoyaDGoWr6{MnVYFrlSonGGh#~ z&a1*;%`TR7^!6sR{lkjZId9cho7_o<*q`qZw|O2@ZuT`g8gW8SccU1l)bcvO(P?$% z%~qMg%crZH<$o}JDNg7M%F9pM-d87Y#SakhP(J2}&iJB%u(46mPLRx?kG`()?rx_g z9Q5(~E32DvY0t)5#>b{hGsoPFi1`ZVytSJ&yCG#9`4~*ncy>>zO5R!}T8O;#?aMkv z`3K(paT$W$JqV7X0>zHE;D8LArDX7EDSDaISpg}zS%U#bv($PuqlwRGkzq|=5docq zh57kTz|rI0>m+_z*H{bwoEOYcdG(?e3$1cvo)eIAqn022pi?p{x=XRgQ19+BY;QE7 zEqetBxI!eo!dp}X{-9OL8-NX16)ZVSmaSkqA+RhO?IB|vso+#m;VT+dWzii+Q#$8r z(A6d7x@w+db1~XL3iZjT3iG$Uyu$%TkZtnfE@-c^>T{g0!KZI}A(Cm@!Yc$%F@%nJ zcD$Zz-X8gr!OaT1a1ht))KX6Z%l9mKmRG3b=2gvT!IKMakuSEKUOA@Rw1+KqqAU+S z{_z0EC^7fIyJZkuSRi5}iI8UrKs8x{cQeE=+S;Mcc9zY!Wf;#%%AzWJD(?@!kosda zb37>k+opnGSq(Wsmn#)0ek_Vz7A^x%rRl)v+9fbL6sWO;*~y@U;Kibe)X>K}xIeXY z=&a?yXJD3$9CN^>LPg17YsD@C^PUuC)#16-eQMS%0`Xl6mNdpVX20};d=~t+cgumH z5N>C?vRViHgtLM78>-l3VJ4H68tJ%3ECT9Q{NkXH=1<)b`E;2xVES^Zn(KfvjnSFQ zXY0c;2JBy6%(#E^MLP>d{)Gc;3+S;3?a33nWLf+9(;qIrPUz!a8sLgknFCOcXlAF9bw}2n2OH0Se-w_M8L+0M#`zZl>EzQhsdcA~ zFiY33lvLhY`csBS9}sOz;=~$!$>YIA{fwXI<`u~1$)4Yd!C+?Jfrug0Gfw(6<>`S; zdAla3`~zFzbIt;eSTqKSS>SSquNO7XE*0;ZW|_v126C>Mi!x z|Mb{gn>bm`GMGWNafwG;g_4VIoce-vHND5>uy~zv#O#SP#sa#WH*Ux`9%s84Mo%3) z+o!>U)fTR1xTr5c@#>@?Fv?z5ciY7pC#1lU`}JS1TUmDPr_gNql4Kwg^!ti24b8jC7BQ1tlQX9I-@ zs?+oG)1cjawqH%m=)7%y*kAq*cfAKgu}}6S=6>|?$u&{b%d%kKozWcgG#W}gkHczS zPKe{&&tM>!&#_kCuWmY!`!vISeG>mLqw_DHCTW;%a2y9HHLux8doA?L`1#n!Aq_!P zI?|Rm1HI_p_)}>cIyT2-AFY#)VB+)93MxCJe|x-iDCGoGbedB;?@W&ZZISMl%{Gh{>?`|;YjfoMZ0cW)79u@1rY<~I9A+W@ee#a_RIZptLQ74b> zUM1E4_m=?oW?|vi6vt1!zQUxf4>0qWj`q9#L&PV-g5?EIZ?6s(d|@hL?VK3fA{{Qt zg_<1U61aB9L9L+as#WsS((Y4#8+_=c<;s9Vpe{`dx(eM%g|5loqlWWp7bQl5DT`>T zI|_!zwm?@J_4SB=4h0W&vu@x;xFEyPP8Bj37+xosjEVmBW9h)h`h*Kso)#N-P>9ts z_FBnmD(jX-nO~(k;}R!tcN~lJs=yxtV`n}xm(8?dQ2XDh-?xBX`pYQyCYyC8Y@)8!)5LIV|bON_;HvH$n-{dYkJMqXHs z4>9BYwT(OJ!x{-+;?D}$grzaQ&`}S7p=wHk)n9$_P+}di z6#|^R;YOK`fZ)P3=`k1KVh%(45gVnqeqFKzj>VPF1Z9@MU!NfnhLMec%XF1N&;0$Z z4uP`ref`ZJfQ==D0IY?TwEe{-hyVaa*q~5Oy})oq^3lgzoIb$E!W2|9X#k)A+F_yZ zqHZ-|*ZzSwuNfH3eA?0S)SZZFk4x(xb&jt3qAtjL326X99y+~A+w|^vN@P5+1@LPC z;MuENe$$Id86*ihYJ~jp_~}O{YNa`%I43C23PeXTKpaZS`&4E?xm8tRLcH7Q+(cMH zTkOp$$$@Z+P2jn5u}czNxB*sl){On8$egM?35`7Be_4kg;tbhXn6YvkZ6xe%&h?`w zz-VycAuJ|#MHP*UNwCzeBw2J}8H1Y+@m-*AvK0}SG`t*GyfO`|UCEvSt+Vrv`Jdti z047vf98=!k$9|cw?$D)XA^t4T=Q-!-l>}Nmm2ILe8$fnHiWW2%buw!oz_a4@`ObkR z^BSmruMDsY=9yI7NpUO@_umB+VRpY=_*C%Toa(vV@C2}hLxJE{2oT|x_$A!`s5!I` zPu_ZKfUL%KkOtOp3}!@W^YOkDz%69UIh8LR>Xra%Wlpa&(X1tsh8c!%bnDo=);$Fl z`KX&=4R8wRza@P{hX=nWlx%l1?@Q z9?+A{vpD{^z!2||4 zI`-9oK3QPAUQkkRI|5|q4Sec=cz*E=@CPXIs`KDr;%;yaFZuKuO9T4It_nDX`)#hE z`Q$Hl1EL7sbp334aYXcnoWlph94_V**OGav_aP%x!AH>5yJy0g?ZNab4yrA-zdmzko4^uf5&Lu@ z;??-t>(aO@d-NvAEc7?FHfyZnuCgL~mH10Rq|Uj9a1=v9;yY^lhsKp7tCHN;VazcX z;1)$hzb)YSVvWaByG9HP3fvYTcid4vMdsz+Ga8+KZbNmmv?qfrD;DqW_-^)12Egl< zT58KhIACgUvr+!y>d)0H5kqrP?1IFGj682fs(G-}BKTlbf1ZDV+$U;X!s0;5;Cyd88>yHI^K!Rpz+P9`;mY)3Rug-w!7*A5X6XJMrwSkUO4gkChl((< zMEmRf(S?_hmxa4#>Kdu}pCzkvrC!QmdfV^(BOzSdU{;tbnD^G~_8q&zZ;Fzb2U$2r zzBu3`U*G%TN^t4Fm&ry0Rnxh6hXOgvOa|1H?UbzVgLFh8q0vZY&k>rp1- z($B_Te!um6-ni$ z;a?EdW%uV}oHmnf`^|5dde1nmEP;73cIxz~n|)u~92^^6#`%d|il05FfhYBaMTcP3 zugKO7*XQSYH;9=G49m^yo)m9f&C#&w)DSP(7#-ioT)oGJib0Hv5*g11OLA|06?;aI zx`K9|f3L(4W&th^U5tqoi^p^zjq-GAoSLx^ijE}If>a3JiSy^C@YJr)JdlHbrT@#w z7SS!2@@mgMyS_kSiTnH`Oe@0qQbOqP?Uw}%LqEfBWLoo|a(0wmwN*{lskx%yapT=IG_n#M{rI_HSpL&v}Ih?JRb^ZIL1v10e`s0-R;YLd2R_?Jr^0a5bpjnefuXf?B$ybpG^&$`ImQ$R2<*uquYdeXvj3xw$0i1 zsNm^apA@sdJj#kd9^p$O#zfuLMhdO>lV`q7LXBQL)Xz0Ilm@O8D6c%rZgi`6vp?$- z!}V%5r7gy&$&zOoz`kj~|M1?m`hP0=H2k9UXDN^a}em%@m$WJN~`UqQ3<4)^YL)C+FLwhqwau&Ry#;d9G); zrIKkimy5%3NTps1+QHotQh(Yv#HHTw)UU2nYzk47pxVu!cEqJV)z_EhBN=V2r^}~i zZ*%7=dDsbE!*mW zUIUM>=kvBM2K#4TqjMnk>67t|g1mbNL9>WEJg;T#E-3%O=mwNuN_#VI@X!_%*VemMwBEzIba6M_eb&U53oHL_v_x%22GHCAcG4ewUzxHk54_ z=e#BT3eh(p(+s&A0f`iZ!IBfRvQ(X~w@Q_g4D@d<{3Rp(aMp!znSb-a$AxGP%sx5TF5_#?v-LaLN~|s8#tu_Nrb{)d^>7j3}a3mu2@C(H)M`kvC6{=3>Jhi;p z5*R;Jlm)phSnQ!RUVs2YGiAFld-^~{)pI|UCE9)GH0~8lY1@BBU-oY^tDUa7#1XY6 z)v})tgfs9r43aC)u}OQ3DV`kywA;KGjW+9{4KP$8pDYu3C&(faAs}g!;=i=?%MTn~ zrE{Z!48?{Zp*2N8s|0x`6895S)gOsYf+X|Su%UC@8XG__iQ`(YtL{`-nt?9x8Q_6w zHmEs!I7m)d9x9@8{8Rl1D-J+7M@zs&TFk{W&|E)aTe75{V9Rr;oc-=h+vy$L=auAj zat_<{vaEAxSy$Dq9B;fpWM8L8$$}ueYQ8d}{nwoifd~RWgDr+-c7f3zv!5bCwXY^N zaXBW>MkRk_oMnE~h!VuWUTqSCqKQdCh(1p!hNLoF8udqF6d)gdLO&_DD%M1X{x)u!C_{gKa;A-u5 z0H}`kE-AA%&wvsYMzLisxvh<7*0O~4EIG4|Xcw&8?u#fM?d!pJK#(sVDt0FrmTBCQ zu^+FFxnopup=p2C&&F<#G!x1mZ+O$VOh)dJ@ZeE47LSFmi{>lgAdF11gyqa^q0v>d zW=lsI_2&M>tcs<_A8%zmg$n@>w-^IsaQ8&C{}Z(szi8ZdwL1y|7c6l;y#1N01JpwL zm&fN#_{!%8!YmS3Nntlfl>c(Mo}^3xmn-!yu(cd`#U~apW=#+$GstGLk2X)8DxR z*eo(ttN?(o4YqvELunGNgCuBW*8ZsZF~>5jCipEykjB2FX%@(2AoEnkBteOUpUBEb zN_Df8XIsjElsJ#uV?$Fw-dJKg{1`DfzSqYo5h@0X)C?1G6`Ve#q*&kr&}2KYB}>uh;&YsQqBHihn$Du?f?7$8~iewAXbDOhVpRDRQBYy}9$-Nx;I(v-koHaL8J*1_Eb&+ohXjPjX%v>=UU`w>>5S?z9XSyahrP zfBG=t{n6gtFgf{WgbQ3dGpz38PrnI)4Q)gqzHyZUF%Vxo@1!0Z;irEV-9fI;TRFrt zeSL2~cdGesQ98z+3yQuhb+c1-*2cQCQ=xmaBjh}rzN5uC>^LqGOS&)P^S8JpLOuua z$;5iPODZjqBT`tYCLkg{KQR>JR)YXpi= z!Q|~wxkJu1#8%geCBprV$Ri&A6>DkTYGaVhG=Vxm;-r*um51_u9GPtx`hd|$cp~N9 z^!Ya7D-p)85?`ZCQ(TA6fGy`^%V6Sn#pLT%G&mTeE}Pu>VfT|!|EHvu7lqly=n!E3 zEb@b~KoO~1wfCyaeBLSZfZQja0hA2B8o#Uv-O#RT9q|1zh(5u7StHxLtiA-;5r8u#QjKf@6tiw{%9hM^=mH6l!031H&Vyq|Q@N%xc`606 zBJfPM{+#=$MFLG5t$45zSK2qCW1btL+Tmh!&QwR&DrweR?{BO0THw%^6+uA*X-VO{ zeYPk=p;eh7wK_m0*hgkQ1{_t8VTbv+(bOtd|7bR2i(OI)ZW)R+7yvLIy) zYjDqcez$AI7ih|fNi<4EK8)J>+!9?=!xdix%=Vav3}AJN82)mxE)?tWMf3cKMWF;X zvIAIuf}J*V2HS3n1RwE|-Z40`;em^nPOjd9QSUxD zb)bD*UEpmto%?Z=xIG9{Sb_>VqCS-7dZPTw)5JZ(eO@MaI9IyN0ZOqmj z?ZwS?#l!PI*M|T*PuEF$vH;BU9vi5>H1B=|FqPkL$@E0|oL~1+Uz>_l+{o~K*odT) zi%pjR?%4m9JvgtfjAeWM^sJ734k2y@u^)0AE-T5YEEz^Q`&U_j@+_>cW+0Z)H{`|$ z4LA-|C3pA134n_4$XdE_-KF32*PPD`nIg5R?})|S7+dU2R?1=mAj?1yoWR5fMqHORzwsM5Ifk zySqdQkyMc`ML?A9MnPJ-Te`dZ&JE}NzSp1U-tXQq?zn&4?~H-}$YSrkYOXow^UP;) z{wNrPcw-{rOY2w$cazIpT9=w56BDQgrnS>FDyoO#l_j0c-rP$_9gwYldzkdea#5su z%q!?^pPFR@2X&*UdhwOR*H^C4Ecw`SC@uOPm|EfS{lnVW6NzPp%jPAX3w2@r2#LB% zGcRUB;wrvXE9LU|A29&81psGk6H6WOYP^NWm?%QHAD0%;9x%Mp4pzKMuAzR7^f0Fi z5~%XsYyWg|U&X)}<7mOYemgezBwP?o&b!D&TB?HSN=R=AK{^+uY^yXdQbJJ2%Rp*$%2^E zNFjoOxQU!iM?9j*_A=@kdj6MN<3$*j;N;nhQ`G1pHa$A^*Z+Kg zP;3#5>HzIxDTc*$Rw$}uH6QuGs!{H~whByWZ20HQE43}06HV&%bPc2ke+=2W91d2qPe5q{2HNsEaT659<#}l|;kW zuebO0KKyfYBL5SHT*Q!m=bwMMeg&8?iG!VT{}5JzS+9XdU-I~;ycGQMGaVY5y}){I z?mu`CkSh?nN=4OlE&0!g7x^C{fisvjvq^b5Kx+A?SKow33v2wJprbRQLPBa%B`GQY zaPYtw1;V4*uo(XNmye`i9vo(jjcNa3A;3Bmhs#Bz6{!FDm;X-j?@RFCDgJHo{=17& zw#~o0_`j{+|2Kx(s^l94)P;~zcz|m1r>?j`{1*2vwVx*ht|IsU`V}m9?0qnZgg5B# zp|l>m2bS=i|6>U-&+RXSsHvLzD&h#|+TBnL_GfapVjVK!z^+Ey2NG~7+;uCy#Xpn& zl^x{Lf3xg*cc{`Xc|9-omb88azlvfC?D7)PEy}<3do!v)7UY5GD^AW$?ZJP!QCvmZ z;m)3FKsCy}^nqQzcZCKU{cU;S8ND<}L&x4ZmET?qJ)Ar<#xZ(6aI<*vBjE+AKpNx= zE$$_MK{+9Wh%k@Z*?teHU;;z;NP;R2H^5*%BL<}>pQVn!aA&X70B*0NLDnopwJJKI^I`6>VR>OrQ5 z=VFav70g6Sh`8*T^HcUnyS3uz=F6x(7C3(K{IvBXGR09y!|bdcNz_QwAyi0>4F}C0 zclWXZ4jTIQ#hTd)8udB@F_gt%m3!Py*Og5L84U@{vGw7Wd+*3l{jr%x$CHdxmK-kr zTxcF?C(ZZ50M-3~6?ODtCMQQ&QEr6<@ZvZVKJf|^2gvR9aZd<}+9aWqR(?WiloYCQ zqs*vtr-rc-ZjM>Ax`Hhug^uCy327&g$;SQ~W$yH{j$a&6)b9hv272;_Pmcg~?s#R7+sTv>GDCRemT+=qXGyP6)*sa?T8D)U%T>*E z=*c{-P;3GQwEIkd?>*|WE>t~!U|$(8IIBVeXOSUna!!~>?Xu?aDy(wW$A#rzD2}Wy z<`Wh)D2suL=y(VVwj7e6s6ha}7dIffm$LUh$|PsQdcP!+cxnvzB$9Rzsz%^PSvK-i zK?Xo=JvIG#t{RpDg!49+uTNTIJlrt#3?gD;GX_Pw z*n+z~O_}p?{=C;Cfj8D;6aQXOH;Mx?;M~cXb=Nf@F2S8mD|49wA=J2!mHRo6WOfJH z?lS4Hh2G^w(=u!j8NRbJxxKXGer6cdkk7=p?C2{+_2@(}txdP_G3M4>*60}zJ|l)h zLhQo2zcwH%yGh%89ogMV9S=NER}uvajDMUAMDjLVM1di$Y#>TTv;l>TD+3%E`w%XB z^qCYf@k7=~BOy!&>qfCjg9TtMUT_v9=VRBU>*HN41eYH(D|D+z{? zTxG5^UqRKd@gv-Z(~fREdXWz;j`n|T>Y%r zHJX(puQs8UGhfy^`TcW#Io{lH(D}keV(Cs#rL#4OPHWQKhkHKdJ}hOC9T^-9Vk|jY z#F>4K8XLEql`O+EF$$j?Eu8xnY$yvScH$)LDJw;xTOeWWHJe(L`iZ7YBB22oNL=Od zA!;ND0Q*+cJbU9ILq{j(FMbR}EWtVy*gay5Lm)uG-VI7|t701gm)H`t?m04@919R! zev-e0OXS5;x|R8<(Oh$(U=nM!kxeS}>!QkZM(v^$&SLSdNhrI%MO>!f-Y{+j#a$ch zdODdj1@e^p;mJ|Bj?}U)*tV6PT_$8};MVNc z8ucsh6;x&WdCPJ9!mvz&6aGTUqMO+&Q=TU%8z+3y@glPldPP&mgkf-WS}0jQp+p@< zM`Re=#f2zDc$z@aTal%TmACXqrjs{|&R)Ix!A0OXW^-6y?3IYVD>tVR_F^g9x&E~)*6l<^bBU+> zU~N6Oa;DQ-OSS&D17NSbAqZf(E^v^qSq|bP(eqh%VhN9|)k!{UR1nxe1Th*7pvCI6 zeE(IO27WMz40825Zw`1gtXaZE^3YY*}-oS3m}65ztFBPHK!^0x~>VtYhL zj=%@O5ve7mAM>N{evb3UqKmeBR=HBim)C3XRY$#7@hqCX(I^;zXjuVCOSS>4w)noB z^MRCHEAFhz4P03}DJ7lZ8X-}(m?aMDbsITa;|Q~LGSXGUZZ1RLn4J5m5cgjCOC^iK zoBt~-X>gUQY~(dVIl?#7iW1PO2{ikDl1Qr$$w{%A*!p- z_cw4}hNv_5<=%-cLe7~f6Ru6vGu=Q8nUTEHW9AsEV67=eB1bD}D$+_edefnXIHAO; z{0yBt!l(FDJ&L)?ol^&>(K`vp$Jfk&gXxvisq{5HtWl(-l2F+lN}}oex)P;gj`*ls zml7@Pcjxo6G{qP*?!D@o>Pl132eZ(+ZnyMSjQI@kD;<=WqivgSw+Yj*i?*g5yas`_B_shNG z5+wZIk*D8@jX;*_)K%s@)Ck`I4BaIWAY5b&jl3FPi9gRxv>4F#oFCw}=~V7=dw}nn zaH^l~z*fY8%M9x_Q~}Y`2vN1E&DH#d@G7pMnrr1HRs4#{_059>p!_i&-DzhGUs38->VtQ znwO^Wq_ga5iZE8DZa(tttP;Zf=aD(hWNQ(T06WB9^MTaj%y9Oo5t&N{fbx7N%=|0=BxRIMQSnid|# zd*jl57nPQ4D^L@D*{E>Yv&WnPqwvUwUSPL0MYXaUhy&RE1Rz1`!;Ul;+14Ub<80|8 z3+jBMAMkYjRZ$`$ggsFK7v6-<^Mc}j1=?w{Kog>}U*-|Uxo6~6t!3p>?Z$&-B+np< zO1ZP(97rUZ)6eOd3aPp5cc;bxOzu%HzsE?qPcci;_ecQ;2`cjrP8X1>RO#?R0j8X$PpW3&8_%(@ha$y!Atk>e3YnWDJ^gU|UJHIOSqbh}7F zk*XW6vdEl&$S_VbH5xfvY0ZdRV+w$Z)=uDdmKZ`+9$u=zZOY{WzU=u;S*1cYD#5LE znzyAp*HSur4O1@J3}{zBlr_p;tiVlhXKT7eCmzaugU{xh%kCAH5f#ope}=>udmG}z zHMVImwGmvt6gE5fIBWDSrqNX25If8LqQ(L*D{);&AK0RYKnWZ=}kA>FO-fGj`WtCAR6HP4O z&?`oIc6@YF9q8suKI(gy`>cU^b42azGcgsSNF!1|r=;MCLr#J)qX{?}T&jeMw#Gl2 zo2+wJJ8013*2Jl2Gcy8?bN}syo=O&KXGAa?iy`^ZwYSbTxka)!oHC6i`{It}mBYcU85}6FQ_VANC?Adwtj;#Ar4hlX}i}(ArjM!%Hxd)|YQI z`|YaH`6*_^^DV5ecSiMmd+{M`Y)L`0pQ}sB?Y)jjNGYZT2V5QkoVfy!`3Uku-ZS@f zLp3~&iqbENBoZ}AxWKXJ4i{K|^4eXWCAumR9+X=BoYy}yHJi+Nt!3C9PqGH5lE|H~ zOd-1jZ5zCH$_ZFU(I^!7L`OmT30;o4#y~m>DV> zNt~~X!8Ux@r62|~^ELZ51OjT^E=9&?J5siIc7D98Fcdw+XJJTAcj|Ub6DY?`h$*$R zEp+LLdCSwdxZ~o#8m1dgzh}}v?N`@*8_GTqlaIr&i3&y-$p7aoWVdwE*V3S;yzkP; zGwEKkIap-I{X<_-*f*4O$cq2`-PM@=Io)Bm(-bvzN3dJXbiO9AZjNZ%Hp#6%x9G0Q zSRrwXVj#uKWdM^>w60i~gHLB(-O`_;S|{ROt*`lP;I&9WwP6*jfWHn)u?qXbSN4%& zTC{D19H@lva^+Rm7xtr$R|N2KW~uAP6I7Jm-F(%NT&UtQ&M~~+HT0n0g?NW9t}29# zos}_j>5goDwZ1~MlZBH5|G`s>)~VLTM;&Xkk@Q zhjVZd*~4JEe!dn>Svr=WO`0JI#TiU}eT5D0s-iEdxr}$LeJXXHJVzJv#GWLwQ`K5& z`Xw&k&8UD|3zyi^Y; z+>IF8yBg`@Fx^gO{%ISjKpB}@Im=SaN@!@q1FlXRC0aO>!joT67$vr-p3R_YtS@v6 z_#;-=Lq>98>#^X})GxNR+eD)40w>w%%Y&@y)S?5H;i`y)lIPZI%M}$)tBuO*#k&-C z*GSE8xvmH(IOoWI3627fmvv-IUuJU8Vzr^%mCltB!{9a&z$vB6oJKtpZH~^wdqNaN z<_J3ih!g6aag}j3qm9HLtKSS)qM6EV2|`3zVwDf{nXmpj2W5x5WoFH{Prr|m%PbkN z>GMEj48=h)mv(ow3JOrNE{u+!cHN&+en3lZ{x7T^l1n}P^?@*n#+b)%n638(q zQ6>qjHiYZI`JK)6hj+#;-`MI39qf?el`qFhU4O!qjJWGvsy07!>*a;7Y7aeV^savn zB##RfIFelq`;(EYEazMgcjJ0KT&SiXCsfk=Zq$bbxG9n%;%n%SK*{KQ7C`bK9C(2t zWm;_hIIiGyK6K#qx1-=Nc#=QtN&0Z3TKX4Uj-tscAxh>v*nV+x*S{^1A80wWm4t9Q zq(d<7j<`SK>cg}F9_5G|&<$j-td@F@{i20JLm!bqqQ(LSd;PI|^U;TDkCnIy1vMj{ z{Y8MLSWRUUB_!X}QYXTVw1qUe=e&#p+NzVVyoXOsl-3Ou+oa0cxykd=%@p zKQGQIoPr_~+~xRi+U8d&|SL}!F%3LhH>~|aRb{Yq`+!ybJvfDr2>QXjMw9F@}!@o6Jvs|)4 zd@)$f;E_g%6gtO|t0Y1FwUlV;^VWyS{~p5QGzL0X?LAm8BRz+GU~#3H6Z^jO^C_TGrZ+ zp0^i9EFmkSV3cvcBD0>6{1o;aWda;^6D(ga+(^zN#^Th}lRwZ3>f@v-$bmT9eE;(C9xc+ zT|3GGVb=8Ztv13lgpOPmJ&|`&sHpv#(D4^iY-zV&LC07*LN#36JAzQm-W1kxyVNb4 ze55Gq%TcO5p2r5YC)U@GS5l(7Lc>*|Sk9wN4z@tfADNu{a6$q;BUgHR<~Hb6U?F+u zkf`_;)Sk#5zxa#l4Dw05>d5KS7-Eik`cO$7uP^(Z;3VKg2uLFBD7r3?pdwA#NWubM ztU9ou1t(ax6w+`br?&_7#ZriGLrt})KXh-NT8K&>**jxcirT3Dt|ZZMyR|e=QrK6q zg~)R2P#(iX*#VCNj=yky%eX@a9m7Bvd2u+}lMATl=+$Fz!Ej_Y?joRJUyyOz$U>7kAVKUb{A^p{Wi=m9RDmCj~Xk6h4C%FwS2#iPUp z`Y*BO_wD&Fcj`a(39)_uThrqnS)#*z7)b@ZNr;*M?_DDln}QDd?;^@l^TDe?LM&fM zfZuPa2?;Gk{P*x7f3|i7NO4j#62^cS3(=j$M_(xx;O7mi97jF>A`#z(^##AKI(r8o z&AJG5-IC>S+wiwH!<=kj9T6V#@+PCeKXq2(NEG(Ll*;^D}bEK3gu?yT{V??UQefXY&B ze|?PBr#QWAe~Ad2AkSmw6N^Lb@|WWm%NDgGpJY{voX}A$qd310_H&`=zi9$%OfvF7 z6Np6G38d1nBaD>)ZmSn^{Nkd*fm--~y%?n7ykSlj>Wia~{lCGsQt#fo&xKME@2Z9% zvkn#gk77QK7^c;+xnMsCStH51FL^b8_!<-lS_s_8<-RYVqtfn+yqM#}=ga^7b;tvN zf`O1)x_)$t&;sp79(mqOPFeOJ;V}eBNFY%l2&{wWLLo!*n~uXX$XGu{R@YV1ajM^^gzAy)@g!Ss%62>jcMS*{koLIwgb4M4 zyT=bKIxLDi1w)vDyx6*5I~AC~;!*%sxfcQ;w&&nxkzh6QMYsR6F3m%U>!#+h&23>evC+b$4;Fq-`F-*ch{$?=+~< zY>Yw?O!z_1)0fyt-B0&a?m6d*V)?NOtuDix|3k5`P1)IFO=)^Gr*A?A@ zdyCst;+kF@TB6u59qS_Qm-uz2^K?gSigWcunfMuAzFszB%-`w2BJmU4aRzUHJx8m z<`CEgvh{%6Jwh-T$vn-WAGZt|S^>2*12tO|h5|Ka`ZiTkz?vQ$ zg{XjEbbh?PwjiLqwVvkkevc~su}>Yty;vYn0b?q$p%E2CbcjAS4F07)fI$=GAAkal zCsaaar-vP!e@vF!bw}tH?RM8$xc5}r4lutghg==)>GV`Bml&&L5X2X=sIDo=yi#nF3?WzD;H*v;tJ&Hkqb9=vbILdYi8_4}8PXII6)9BW$ zN!8fanjtvr)_4>uvoWfmO8C^I{immvQRIPK95Z~U#zBEX31J4YWf-QFa2i29=CsW* ztwW1=vuHKbq3V5@PUH=v%w=JX`Iyu&aRwRtds{*Dlpx3CNY(FDpE3;i6Ftvi;7c4# z^LqszLLx`zqXj2b1SOUlQwDn^l6qog>uqJ(-Z;yVfCiGN3I%zL3WUEg@?dKEYq8cc zl4(ZjSKCaL;ac1ZVNzrKqiTkOEnO(L!$E+nS1bvPYQPGKLP}vh5E&#!V902I`i0b@ zyh=i=!@1y5t1HXnp~zKO4voLQzieA|T%VDh4*IeSpjE*8La1vi#{m)S4%5Hkv1ywW zjIl+DaD0E#8@x{29rB6-r?JV1Jr_8xtVya>^}HPij9v1?Mi*#-B-=TCpI)ve#{>4I zwPmg@5x~@NvQbjVhHG-6TDe4IQ&^A&22cp_Mgm^hBPQfiW_x1{Vd1pJ7TdY1u;7Hr zE9u9Z_39_a?>_scd?e&Q+~j&-8ER5EeTPhOADAFXTV{n6i8Pl0&5GOiAP-S<&_`so z-X`%~C9u(sK_L8m*j*JC^-Ll2h9>d(3mHZtWuLN=E{rX^ERRQ4uKMFb3YW~b-+cK| z!_V6;{WW;yKe&d(XYmiSHk1rKRc!{Tvp1?=A!3c6X!ao?mzw$uAs^vZdxk+|f@q#LO;38mJ!*D^c zgz8GQ;K2=2X|=O7HH$@u-_2SoU4Qlyim3$LT1j6M@h@-W=G`mRB6+_{;dIRjBwVH! zxcBRReXsX2&Ku0sa^Hd*eegU3$Os0VH%rQ=hmZLerrQA|J>GKv3-1OY8r(Riru|+v zeBS}l1yg(~pw}D=hKwaB((*qX46Px!WGzB6#R9l)?jW?1I;K!OsAQP(ex3n0Y}5@Z z(^;Z8$G@MZJQt#3Au5*(@Z2W^VbuV9?i|1l_7~F!<++0-Yi~lm3h)Q^=1pD267}il ze5vWrWBWVZE5VZpH$n<4|*F(GJ(tX7j%b`8<(s7rRn^!-_e!D%rmHY zvychQ1rE7a5~^P5+n`fq=ZzfZ=BllTt^oW3Fqv{uwCHB zS>3)HWC7HU=c-39zU<~^dT4(B5C|Hcpwkdv4FlDe-hAh*2Qfu8ZRd?()u(PJTj#f8 z5V-~@cIWKbfRQDWTsfeef#kf4FMP@?0`>&@C^(+k z-3-~=6g;QJmvUY&4%rpPMyYMT`2^rfQ6QN*8hNyBpk0LdYcV@{fmY8ELj#{PtBHGY z6E2AhH?F=gB)C5<)iys8$kTZH0Emm)<0ExQU9d$1(5|>5tveNIwfzM)y3L>pS$7vq zLW0qcEktB&WN+MK9AQ|Qm#(d712ar2QQmuZbZTdFFoM%h#J;E8#(ZG3)E}qj?56$# zTka&F_bm1o44{}eMAclj)z(&9R&d_6+MU>jh-H`D?!nHT&ipi~#lhMDd}Ys`-JcY| za2$U7ms0?Bktc_oItQQivzxE?@;^$CDVyfp@3tDn&Ki}&%sz@AHnMyEa0r@?&`WJJ z`*Utw{ps6}mRVf4#X9KaID7Ttkn08$hN2VhjDtbAnVdiJx+iZAybs*dWM)Ezs$@6i zv1))56rr`Y(IL`@5griXy)8aF|U_6{b^=3bZ$0B z0bHsuA1bAO${aZ>5ViwLy)R3Hk;(MmjK8*L<1)#}5!cgx)jVheFdmg`uoVVuG>ATf*(08tY6vV8sUJ5>km1ZKLVKB%HvRGwmLDNPWleq%8B=2C z5H=^&@^JaVdOn{d+7S(pw(jmcqDrIX?!t&4MJE{<8XRzF|2MfI(3nWBF0G>jZTq_y z`Go}W?Q1EG|KWVx=?LH3TY5=9{4=ZR4BqQryT2J9xnsp`P5gYmt~+5?h&UFhZ~&Q_ zMgJ+c+@o-$bAuOEP+}_V=(r>ZOal(#@G@LsS9Tjw>3YS|k(~V_h}_#_@6P5i+uzp9%_^@uNu+WfZc2VMT$8PeDCdYJ7#4uBsen>rz7(Mz zrM(YNQwt{Q$`25Z4WcWkm=x^Zi_~H7yl5#GZCM2MAx0ye7^VU_xqLjijef_nr3S7; z$ut%HgXI06|}C!j85z8U9?`O*3>VPxpaU3AfKj2yUViQ_I|jrXhCvwf5|J$ z-CQ?P?*CqQO%Z88|4wE`b-Lk+BjK^1x9TS^1Prf&1XSxNcx;GMP2H%hrl1BiHdeD= z-^z80pkDZIt;!ybk@`C^K==~YF!}He9LX>VCcMqc3nR-U9`UqLl3k^pQAx;PFRdFd zJqA4+E{$5|u^KvYH`!37Q{A;Becz_TmHU;paEzUW9h3dE;c{j`mds)5Lyz@!zssi6Ihj6IOLfB{3Et`OZA^L0EVduq9 zkWb;=-7yAhP0NxyH}{->Fn^1}oX=lP5JH!7IIak}+IA?xOQCM$bAb>jzAg6CX{Du@ zt(QXQe)xoNOKky>_(-0& zdw6FhU$8L^&Sul<4kK@wKwvTpOp9@y<@2eSeJ`g(k6`m2cYAalLEePrV#Nq(P6|e_ z2rT$meyk>Pl#Jr#h$5t2@DYnFy;547&04;{ThehzD)J@`igO8ORriwvg9GN7)WA3; zs3r3lw0}N7X~Duc9#>PJCVQ>xZ4<8G9&0uiS@FV7nLDXz_(p4IOEoU;Y6G8XN8ixq zx4VkJ1YK7K6#tuWftNX2*6)`*)!f{V3l>%#_HbxpjXr_WFGnE zwmmocnAIG+#sJPs~BF< z21VMMMny!a6vSK9)vl16t6>h|@=nDf>cPQ5!2Omle~xw{)7d;cNP@HHN*{AF;)QDf zZG0H~WX3bWqszpvstqB;Sn!*`Wr-YWp9GOjB4p1aB>(uhnfm&w+(pnFnHJ-Pko#bz zV%o<3^8nC5rC!M-M4FXnziCJG>heKuib0KGpr`y0p@8g|go8C9{5J!&vCgqoIXR8UVzjvhs20dgjMzyXuvivY$x*-4w@?kZGCLsx>!3{;+Qz>u&nZH zzGMiY&ke$9*G_e+0#V$3g@e?bJF-N5U8QsxYcU%U~u1<=1Th-Qz+sOkzyR(!-e9d_N~OTAy_Bgy_xWm zzoZ{cka;Ke{ZXbQGwe00-4jqb&hdJ=-k43Xhrs&w&6(Y;rAi`o9SO>2SMT(uW`Wk0 z-JZs)aw8m7w8LBOW9nD&gDLl?M25F6RJ-@QqjzG&;62I)`m>ucp_~>}hsdX6jBOt- z=-&?eh9~8RtLw*?sI~Z$ts$tm+j?}VjiU<_E+A*-k?RhP6qmK!6xj5DwZQh{!dPX! zhS%OzMzHD2%(EE0_mE5?j()KZD#RynoD?mtb~?%NN#WNrT{ITa!SKl2H)9^7`#TNf z6Z2eBQ+&2V=S$COw2p}mtDBaPBQE5fH*;+MWJ87l)Ed+Ar?hvI&rdx-Vj$kuZ#k%5 z|K4#vb@H$%@uVz90#dKD2=zK*EM+S3_u7i&6*RWfSN;3E20@+6>bA(4?7PU<+OYQ| zp;{l@<&Ca%E31yy;{1hvjy6`h$c^PG?((tJ5K&Ry11~_ z^%QKv;wxg^_A9_EADZJJplZHwlf=Xmx6CiyDX7Gby5n4Ky<;mPxm|elfy`Gd7=C}85Y_49=kaV=B!RAl z01!^0x~`8#Nf^aCWdBXCU@DnM93QJA4h(s_QwVCd+2Pf3-VJvf-2f^|AtwkmnK8Wv zsAK~xgh2(ho4U?4Dj_f$Oh>2ENeLNu)S-cbSEQhpcjv9abu@IOGQdk@Ui8Za)S-c5 z0%S)GY^Rt5y?#aw=}O@u_g}vaAmp{Zk6$dx)r$ZTT}Q?x=ivfskK2KFfqYKfdvcTXh*70mk2h9;+dieWDzT8O&j{(j}yLnx^Jy72g)6bzvQQ`Z39 z`}eO=<_GhClTq7p-Dt!(h)y~YjgD%o%ci&OG z6a1vJuJy8Uue26D_5A+Dhif@rgkqbn(eF0bN+;RaY!^x_1!MR&S9t@fC)q4@8+j}n zS0j1qcZ-`G?+Ic2!%wU%#uX}4S#$b~%T(z97e5jg0-qPtzkMz8PY?MYPyY95sAKr= zxc$Ch{#_yeKVKp5gnuCwlG%*(g8yJnSP3zL6?MWg9(Y~6F$k82a&bj^2F%jc$mbYi zcxvD3qA_1R|NZBiFPI*us^+|z?-v8}zT^ifTsZG0O7j%m!n*gJTxMOs@6F+_zlmRH zR&h+I$y+UezsTwjGJwB0LmeyWcn9lo&v~>9C_luc=;4L|64Ke1siFPH(4Ys%8)%{Z zV==kyi+{8arHA%S_}ezO7TvUyenimpF#gx?#ey% z6*>HO7wMsW64KtF_PY9??b+@_`>@MHjgt>S-o!%Y%Y7Ii_vgIxXXqbj-~D|*f$4o} zgsGP*z5GllkW-y-P3g_~Gd0T6;txMT`%f34eG=_gCvTMLdXD)MAKL4_cH)a+ zK(ptomq;Q+yAXOFeI}dk6V-cc)WJXRmO&j`u+fcwpW)(VG<$~EqTcw>{t41PN9#Sy zpY6jgLwoTzrjwSmRHnKG^c^sQcz1-NHr(+)Wm2PVugk2Q80=jX5g+yL-+`0v{tfy! z(Ei;+X#XiVd4=}R_A<~unto;8DHSePWaNrhf+4nSG=kfcBS{O3m` z(>8k+Y^ZJulHGD#P8oKTZ#?T-I9ytn>CjIi=1wHAo1wAWAde-+k}DLiJ}?&pyMLce z1{0ZVJzpVG-ubGVUtj$>cK4o=VEV4A?7ngJGF$O*#nM$ZliBJzfxCHM^GzkuIaB+2 z1C52?ukJjn>jY1t889CRsiBGFA%nVoto`B|X zs91N%o=6b>p78S9X6a_h(G^(;Pa+ahHyR)x67F9;bLnz3mm6~VlLn#*8({xaa|h?^ zUu{xC#`SA#raRua0&9;zR)eqnu^1Bpp{i9&Z_D9wfa^J=NN=mB$%PP<7xU7gK_^ax zLot*$l|$VQ#Y9A!GVOd`*}m}>=rTtMjK;pi!3uEQ8*PitK_o*Dan$!lh_dWJWVs-H zbI`F1i_1W2l+*tsGjdPQCAz9Jpt&&e`NGegt<%njIs*COe}%3x7B7`M=NNOuQIMKK zJ=RbK2sU(Vb&FbCE?qV;NENWdO7PsNks!Uk@#>O*e;>Dj3QvO`_4@V_?`Y1| z!Lm%PjP5ms|=5GGGm5Q-m0BtW81RaKChXb$s62efUdxDLfuB2v7keDRAx3Ab!Z zi@*iqNFkZ1jeLo@Q*<{EsMU8`Lr$IA<<4F|f8~=u+G<<$Mz&FGF^GY9z!fGpWBy{N z)p`(25qEI~*vX&_xk3zQWY^`$GAdfCZl<)RCEAxr#9+xb+>koOEcb3f38ve=}Q6K9gBDR5@7&`-4PY z(*C6WchR(EC7DI_OIU}Ezb2=pTpd!mr3tH2vlKSk6K{36=!g)us;#^Yu~MB~ugJrm zAo@Q0kRe4&FpGwd=N0AV`!pw7BY<^!wqp;wX;-2*ixb73s^E8l_RETq(~Vb*316HP z1GU^fQdWONv*+>%R?~1@i=*@ay;9n~^|pEz%`fjHdFlB_FEve#a})_04KQBcRKjIm zT0>|)iykju=;_Qoma&|_dedx~W#Nn?xr^}>dmQ&IT2V{FP?kumZ=M>XbSyW*f)CTZK4ozKFBf=)G z-np1tpVwk9nS8Tsvt+R$s|6#G|7H}JL8}XTU)oG8p4o1Xhv-ggN*6o3dgZ~JDNS7!J9MmleZdFbew-NSl7tUi zFzzqCcvQF3CK*Xi!;grfuiQ({s3Hkl!f0qfaKk<57Y@QAOT@JAruUr2HWjWsd-qfa zmDEWKgc%hQ`#EWcBlX45)oN+`4}_8J>}TcU!*ZN0_lvfFd9nrv{j_4o;7;YBnW*463ha{y90|jL)(qED7Qq^kuQH zxNFzqR^MHXuO+GHa1Y|j0?aoygU%DG8hlvV4=}w*sHj)!^0H14V??Yw-$xmZuf;}f zW3#*ZR<0$mCA(azfHAeq&uH!hkZTHgZViojpu_>9lGQI})Xn>;VPSajV0W=;u}i?4 zA;g}{{H(;lQ_aaHZZqHVWqxM~^Ew)%WzonVRviu3#-_s}uCdQy;myCa@vw&jWXO7G zkkr+;uawu^tj)X6g3mm?#k+kUM((Vv%Fo9Y}N?>wbsE$zfMVfE356@X0Ka8)fe(I zZ3iN-1$R=^v)pRACvHea@=Et5;ThxDQrM5ZF&F>X8gxE%-dOB~W|gb3C8oVHr39f> zJRR1{700z4b1&g8=R+FiPqw#~+;FY71JErNX-O$B;<`%fwOOadP)f(P@qo|_pY#?x zt%A%tLpT*^)lG1f+1FX>%-nf5`uPk0@uK@u?AKOuyy&D$V=-8{sl=oOIc>7bcPJr+-(Ccd{*N5zg$SkU3j+ z*H-VVau#Lg-_x7bdn&)I(276ltQ}?cc{%&@Ypp4J#@k)pIf}CHkBNPkAGMD(?LWTS zVkMbq#P14s`dH~rI$b)tlT)K#CDOmJ*76pZ%L)BErL487Nd+6}zJsnii+r=DjxE`H zWfbae=5K);!vyR^*S-tQRAUu(+(zO#>!WrDclq@K;-vizU&|2o zoIBK{ecxe}t+FkI*?YL3AK*NvsT#g+^tBgo^6PQmsu0banBDLGU-RhlfL3Rtk2_Am zAenx{TeaJ2yr*lOy9U=>=<$X6p96(+)$;C^jl0iYoqe2hF=Onln7eTtcB!s_VVu%I z--JFHN6pY$0+)xW^Q`ivI^MpSTX6V!a@#lU#a(zo*m;FF`*!oGAF{^pN`AdKo5{4a zHL9tvH(=itL>L{AB{8|*nZ`LTp-8;tf zZ>YYjR3z)*izU({slZO>x>Z4W%JgmQ8SItT(}E6hh;a&UX^X$DX8V&pHPWHuoq7I{ zVR7f$u6&U0p#H>bZX3<+$T!<0nf_{=Q6;B%irTmJI2Y`j9M$G|7>AgCjW%%GWQ0~N zUE`>Ud&1N=aA*I1g#+9d_%m94Eg;}*U#g6FVEg;Hwd?B9;~X^^9)tDNtFhSWoTN-! zpKCa2iRZ0V7=NZ|x}?r2-z!M8TY*E6w{Kgs>(;Wjl$K5Hd>(mXoXHB~ijbOF4TIgn zr^1AZ-ioDy`RLOX^_#`~{41=5IdQgU!Q0-o4VT0y2)gU z0}6QURNt#P#bu&w-r*IwoD8+5bJSGE%|V6N}oO9~_i6s@v#!L|Y8mV+)1+Qo;w+*RgfAQRBxZha)nbL(-1K{${5C}ypfxtd-mR*iSa_tSoFh}bt)i+iAy^@WU} zxX5@OHoC>BN7EuQ-S^#|3g%dP2AAa#EY9xapQ&+mSmzjSZ)RH+o|Dbu2A>ITp0LD#{i;dKNJzlr65|C)js3(lu5mSG7GEVHe_@OhGqQ4uwD+r<$pqR3yy)`HxPeN4aB~f~Yx8Hp-8=~2@ z`29|nnEw26{@DAq{%LwH*9}=Fe};{gtXcaD6RvyWYQz^=%-qvj4<69dv&$G|B2ek@ zPW*{WSwB|t>p{7@9=pg!Ir^iMl{sfzqD;#5 z#pp}pTs)2<)?a1U(Li=Ck~Zm!%MH2sThhWMy{*^3r*+h7^wJMX4U(=%lzi=D*MfQ2k;J`#4g>3T@i3B)7e-M!m8R$nQLXZ0}JxK_Isi`t$X4jLA7zTRvi?tU|{ zy5WpAVVdU9@ikMGGC_Z@fASVrR)O(>kAKp0vWDfn()EqxFj#;ypIKEF4Fg=gPGvvam-2b!MI3%F5_M-hM4%n1vs-SSLYvWYljLg^^s}WDYRwKLT`$I#WN$2vR zkNQ@MlO&mz*2X)gb2cFvqrAxsS+{Q)6%)F&8V4%hq8la_Ob+R)?$+Okir0&#!<7>7 zJK+~?{`Bt9412ZSvv>w^c9C{#L9FeKDiNk4!l`LWvIrG9YHZ9jI=la61SAhFk zAC>ZG|))|iL!s-o}hxbh|K z#=d*LK0iDLM`GDsUBVj55kpcyx#F&>u$XQr$vZNZUr*_x@mNdnX1HVR{xjvmkk#oi z(sP3?!6i!`EI&ut4!V0jsrsN_T@+xzTa~fS-svFVKKO>S!JxifN8c1Tmoa(iB(;9v zo-IDv`0xl}SNB5yfcM#zf+#yN-2RXDk^CwhqU=2#-#tHhVqZ`il}m;KJw%j0tr2&7Kt+@49woA;Jzzez-dU1;R{xN%|x zqwp^21HQmq+%a#9@(DKMM{mSt&#j*&{iv@}yx2}>DQ%dK*AQ#HU2%$G``T`!pDW>_ zb3gCkuxfk>=K)zp;zPbQ5KikIO(5InQ|`*Brr>Ioa^-5&~!%uDhvLzi7b zZ)Q!;teQ>+-R%$43<$SaTl@xTh@kg3_WknE#8srf$Kwg&hj<0NDDf@Y6tQ0|z8-9k zv2bM}yrG`ifTob|P#~x9#|7nz3SOfc5*njG+}_6f-&F~5tJ~w(eH`vl#ssq&uYA!sw6jfy ztAaoAEm$z^3rowY&8d|+7ka$pnSZKu`T^^bdTBr0=Gr=LvExtV}VU7Yt-B0_e8_@$!4&c!;a@N>0keyykM6~kYrw#h5= zW{9X&chIYkEGN(W@S&_C2&Sm3;0PMQ-Y4C!X>)Rr8vA5Z}`1S6nFHN)y`($x^9X8 zxLqk;gyF9U52JD4C?mpP7WTO1N%`#(Y>QTf$}{uC?97r%H2={8cxk4WTj^~q$C>=7 z+MiF^qte$efU+$83Zr(xH}tbwN-@bFJ9D@KILZ0Wu?|!iFeq2EHoe4lOe-j_V^OcG z`e1InG2e32@8#4S?_o2u21i+)gqJsgU10FdJP=aT8Q0y^jMGXC>trIK&F8+xlzm%r z&+M>*^5DDz?c_eSWHZX__wR8kf%(3VRvjuJb}Yab&pyp$#xJ=e9f-t zdHc(2+$`9dTtBa>N6u-_zfoo3%Z#YWz1Hek>83tGAQ8$fievs!k&xYxDBVNt3D-Vn zWh>?gb~K(Kcd%ec?g>7xiIGoiOD6I>gINft@32*1UQ#-|Iktk6$3K@xatA9erL~;q z8&DfbyQQ`L58gP1#+AhmvGxVq2nZn86)I9{clEp_B|4oMlO48p8k#%<^{U!g3=%eJ zz4Iei7#@|qW*=?KZ^joXI_1~oG8+;w3!06!(bH-1lWQUEY}xrTL?;8?Ou2rs#(N6R z?#h(y`BR3@kOX{k;jdnc-cJ}5K{K8Xt=@|l+4+v7luG!=p&#QR8$=SNn$n+%B?Z>o z=X$zNt}iwOVGyVgXGgb62?&~;O?3XY+~BwMVTNAnm5*})CzhH(dIk%d6{d`F{gt=#KsaLlOex6N}fzzdI^UEQu7o_kX#cr5Ery#&@|+We?Lf zS3r+h|?u^pN$=*`FtyC2?k)1V|F1 z{tx!vGpOmV{r455_o9HbpeRLp?*v3drHLrLh)9zT(jkB}kuFGYQbmy72}Ej;-X!!+ zkX`}=0)g|(y`TR+v!DCjd3R>^3~w@%g!Nll>sr^k%I7QAvbmNvZ?kZY>1O&hL%a{q zLCqHfg_@)0eM13v61+0g9`r-*Uo+LD#1Ibf?NjXD*j4))5|f6|z@EFlO0yeLUd*w8 zYYbGRf~v=B2Q2zF0ByBQ(T#_w;S=SH4;`ZLyc{7v=Yx>^%cm*ROIO2xt}j9LfoNA( zGcVjNWn)~~G>kmf4Gh%Ap=LA1xVopkWnpQVFN&w3h<#5GS+aU(M_4Q^5Yd6R0EQvbo3J zH`qrvyQlN)0ZOVD+@@`9ZO*G-D%2$bSe|iaMtQ4{xtn*E&l@V5#Gw1V*S+~;aEF8F z^$zAz{&cgwZ8T-WrGkmO$6@1Pw2IU}IghrLjFW6j6){mp^HaeXg!HCRfF3J_*Lf=t zb{B<}t$ovBko!}9QMJ?z$!~vA5Uf+EJk(d;t#YXvThXYtY_T_d>HZiclD^x!U-ifs zUYOonb_3q7!cOg4lfE|(^f3L2oTY5JG%yudd#S~_Plw;7iWo=HU@f7nHI2F7{BTC1|P~b!|ODn z=<9h*$l9eWdNjK8TdU*MD%{%r8tU$um5&%XHL_#??^@sD`Y`24tLb1hGwd11IaA;& zI{$T2+O9W8?Shee*Kk&4#+%E?$?PVl*22+&?b3mU-;+#-HFTfW8_TQF6<&AS@-%R* zen0D`SYV0uJ?0_LIQiTXB-=i|Q38^fGDW(5K)h!Z@&q-t=XwbwNivDK%MfOdSc(R8 z_86#Vf{;+_mOmm}`01&xd5Lvm46(lN9ACjs`m@zqE6=3AFw{=}<~!-xh;S2NCiWhD z9u;*OEBX=7CvULUk2OmC;2C`jJ{0Vo$Z)AkrKn(=if*Bgh0=npKw;8`a|N_*l?R74 zDxsJiQ{8)VVJcp@XjY=_o6=JbI)ox}&pfcO@WC zsNob3>`H6%eDd=a#f;verH$dN&H6jL$1KxB)1yJ)+RW8ZS=dm3r(i+(U&C6zT5el& z(aC6XBGbdJePqGtJ6P!qgM4)JmFVOF{}KiDf<*0=_r69A9s3+Q{ba)3T++6$9_B0e;4=uh zlUKafocS%cOVUy}#rAGbj$6 z0_!gu{7DfNL^8WAIoHAs2eI=O$$CAvI;Am;QBhv-7M)4WYgW5>KjwS|@G5a2?;j55 zN1YkcFLzxpuzyO!7r#jw$*wZW@%`j<*U2bS=ZKK`at}U9C)9}QP+(RMtk`00x8H(} z#Q(1Kl83LT6T;3^re1O-)w^8vE|tfs6aLf@6V{IC-XL6~K<@fZCw!R6dz21jH>{RF zms3CFD9#sJv0@WT0{FyJ$AAq5?>bU_UcO$g#ki31n{*k}5!>ub7-<4t_&28Vm* z*S?jgHY_;xqI0KjxV&l7&u;ykhhU9_HjiQb7W-yylntW{7gW+kJ#1X{%CJlPnk2l> zM3y&p<_lYSVeHRuwcHHcn}=s|3uhqZE4k~2w6Pl z#Q`aEelA|njuz>Ph>V-4V%i=LU4jCaB}<*$K9M%G$Mn&9T&*3HoMdqDm5NTL9hRv! zOYED?sNeE7ugIOF>&7lW?fu!w@l?&68pq`evb zfkbR#7H#~F;NF`ZwxwbdDUi9`fM%&>ok?pQs?-Z01>RhQL5rI$UMI&O_%#yubGn~h zPn7Z-{FMT$21IptOHW?F&($#|3Qmyu|a=)s_hM_LbkW zpXUkP<#Al>#13~QvB{PSg0n7C>@=VKxFvnI$ON zd@joObAfuJD_D|{xj+?v_mz@qUHp=d%JDBXxor&t^O z-8CdedlTm)>SSbHTZR(Cel&&&j;Du;mF+;Xe&=)>dp!PoHsrORMP9S92Q{`aWBuzB z;o_Ks-p#pqwi~_Qe36|6o;y$j-G6U3H@6N7_L-drwx7mUKWm5sOVVx9I8uH6AiH6v zch&j@*!PF7k9)TWi5HM%pfyfm)`bqo<~2hb!lPCqMjb=t*FG-1&t!LSF7PAH zYSSSvn@(z#$eg&cRekYM8~?THqm}9pg@{%?yZ2%Xh&$QF$PwxKTHz-=?vTR=xBp^n zZj;&tWJbT<@u`j5eDqvr_U&?jPi)>*-r&}FGID&(0{+T-=nT=(S@)F2)?)OPBy7{5 z7j_eZjsije9rjWAeIVvuq)>Wt^-`GH%yU1qo!m2ZAvhKHRxBfO3N^-QHRM!z53jO9 zt+UqMgK&*U`L&Gs6OuY8%1YcpZk_O_aM>244wGn+?Q)Svbd?yIw`D@&{E9-<`t8uG z`rKg;KB+PDdhaY`QN&ih)~UoEC<>E#)lV)r?;Bcu1wBIZqYJp^kYST4qx zIKSV+&yu|g0(68I_ypx>rf;JwuOzHgh zxl~Be@p)P51(fn32X~BW`bQE$sNt>4LV7YTbw$dB$E0lW)N$b2=F1yV$`7-W;w+i) zd}?8GVf2I@?*_mjS~GIBLb|BCX1(x-<=duo)8%E7jYgivV8Ke(CE4-GylURU*i%PS}{Avqf`VrXb7b{8MwLcrJl)R9~ zV$0D`sd{2+pe<>wSZkLI4U`=eZbN1v;sP8(=~F&jw%?|6DFhPl;dX)raoSGRFGT;{ z%j#ND83{G;TLdjnVWJ5N(KV`l)L*;q-9}k?$3S0m>x1t=Q1ln8! zWw)Gb>oRKoHVj=Hx7XNauAm}0W*T+DP@3<1a!X+XuRS8!$WHmiy-QhJzyr<+NZ;26 zdv|FpH*R`Fy1I=<#7E{xj&7t?Vkh#eJ!8Fh%x0WjD#{?PLLRjVlh$9XkF3j`?xY1S z7M)eyK#OeH4OooypNEM! z*oDpHoeH1p&2!;>kgLu{tKl8<4Q$Q#>o#*f6x_pdhj6WH9nQp`o`Gs1eK~mLN*nK| z%H|p2^MImXZM16fsLr)Xq|Cj*y~xH^LejGXwm336>{9s(p>sUkxGa_tL_$b zq_RASb@_+3ZrVtSxVPU-2wAT#rEotBB0zSeoY&3FFP)IcJ=N}Td1ENczM+V<6{bwM zR;fjJcFHj|C3c`+pu-4$|K*~+BFCZnz+!pa@023emM&SdY=(ma?ZYo;gU69IkmzNO zXjS5!R}IRAIS4p|lJ95yLIlzn{~4&HbgvZO*Qa-V;*4l zB>_*@Kb0E^&3D+5A))MJEs`8CpMfxh9X=weQ0?~HW+WP~gt!I$ic#(O0jB>DFUj=| z2}Ls2`?Okqv`K#!s!}>52dhiDD7d4%x}k_}DuZrmeAt>060&_L>a{Z7C37;-$8p=~ zm{a;Fy{(OXFhfcxE(fF6o#j_slbrp~aY!6=$})I&EXLH`^CriuLZ?J+=ZC-0iJ`mt z2d_?JX;T;*k2yJ8AExRq*oI zjz;Sj2|O)WDDr(Mf9iT{^y{u*#V9M7si$!6yuKTdpvu4UwRjBcxS69k(vYXZ>BnVSGtF;kNu=}>thGTTN3Y`E4y?@*Iv3S2#sNWzzLD9E zCzVeP$IPn^viEFHHxC>!mDrYUV=t)n)i(QfTVU75qVg7L%XHt>rF5wS;+wM{;IXXI z8waiQP6YvOQp(ZOjm}kIk?eyUHM1bB^&v?W`-!&PeOK|+FT8vhy|vO%Tn7GrdPvw- zv%X%({bi2woMyvYFoHGJS5Mq`BVf>$S4wT;ItVW+#SLl|EUnsmqi%@#X-0`?>gbxA z+grA#Al6r9T7DLb0AnJHzUwp%rhoseQ>+xNK{Xm~^|G@dfhv`%o!orf@r#+Qfxgnv zk4A0XgeL0#8_0(PLYv=LUuv#ZzxLu%WROy;&%EdV?D{1OafZgycU{9?&BMxJ+ef_B7XHP*4PMt`fR>Dr05M-^GN@2wVFxH?r&f*VXk;x znkD4FdAS3~*<7VwD09x)f3eJ$_pnXQ$jJvo{C!(q!+gwlv)H(nt>|0CwJIc}_VC?8 z>x!&89}mcJv3U-MT3>xS2&GmU^`h8}ytW|onY$GQANd}@oNx0T^m;->5nc`-D1LEFfWlmlCD zF2sR0Dp2()Z=O{DjzooX^pK~Hd#923v1E3V^_DJF0VYH9kM3}{ePl=1z=#j{iZwVL zaZ6K1)T~A_(GM^=v?fM#ukqck??P(@Hj5!&)XW)mW2_hJz44l;EjNGql^F;A)a0cr z!&LLiqBrxe5UvaSU!~AG`e)BsEGT{utX<`2yyky#R(1B{^3QMMG|*P8LI~%6MH6I8 z$52kqM3byHKyD8X^XTtA()*)j*jg$_j0!MmR_U7w{gChnp*;*6^3gG63JaRsw=NL_ zi_~`>Z@MVcnDhj@vf%50dzPBo%qV7I#)@09h=FSk_z!g4+2vGG^s~{E>Sp5y92-86 zyV*zRO+L)ou?9I1H8Z>_VR;?Jvo_qK+=n3V4KNAGzPR#;XM1$j1j*E|S6acMWzkmD zmWxtJVL+vFv<8sXYrFn7L^Nyq>S33ph37b&(yOr%g$`7l^AG2e@3EteFSdR$V8-!C zOQ?}g%&)!cBVGgP>E&p|hA%z-(S*aZX2M|vsZgW9_7XI3Xd9SRnhWTpg3a4c>Ye%$ z$%=bLglo2sF=yM^Voeeog9k}}=uyU=ta?gX8>{1HR-NNxLDr$viRt&B0@vxew22w> z_j$ecqcdVVe-{abI5rgN=>r#we-EVze-)UV65C>5Y(=*<-ME+-A1Y%hbKhX|@=)|2 zQ9G(6X%F-q^w8}HylkX*178Fzb?WXlY5UCb#jqRBcw#Cx>O$Vx*o=?aN=++6$X7{=JWN*GrgXInFIZaJ3xezWq8!*G$K z*ao-vjv}u+zcpvBXz<7G^*B|oKc9xdH!T7Z?A`F^Lzj4N$4@LfLF5{8z2TX9L)oHp!5(Cu zL9k|);R)Q9{tU{(n@ES&D^D|>-LVlX>9!kBE9;b1 z*Z*D!!fxm$>o^m=1^mF*a#knhd#nTGz2lx;Q>S$C2p{JG?a>P{Eqph=7QmPyDS>=~ z(_}>oX@ApR*f-77{Q}sY_u!H2z~~kUHX`u4H)mfLqo77A?9ocF7JaAesx#G9ba>q; zVP#N*tQT93w;)ten}C$VD%hT5M0A-qIAGX4g&DXql0iPjtEVsK=q>wbC|_xYm8-gv z*%NDVM1$gi&A1>(AogxPOH_3Dn0J?~0I-OXC7HX#Jxe&49cL%1Jmv|!Jz_OUdTDzv zY!1H-h)O-jlfkFRp{i}zzN{)_iZC?>DbBxNN3CHc1UQO04-PHqinSC|Gc!=9zoUqD zkri9gWkK>kqx^UBQ&7yO!T^(|qX_@#gR$s-q}1KVtX6V(&>NXrq8{=rcfFZ&gfc!v zC^4(eFFMb9piP2I-v!FY5Q{x*fnwQwn8vBVJD)hoC4z=Ab23ef^9zUN;>~*4g`Rs4Q@+~TAJq*t{QZew_|Q}k;5WkwoCiiB+_2&yC0t;3*)@` z!6h#Fr0oeds4pqdF80~KqxKcEX@54gQzR!DYAYoHn^;6pZ|d8T`cL`qTBTGGE& zU9-#pv$@JoC6~NQmBouE6Hr%3+3qs4Xk^wG8jBNO zS;CDJfpJCF7+R39%0;F?ec2FM(iWmSWqH)NF#k zV&$PJ5()#XmVBZrtzQp7r_@c39@|b8t~_8Z-8oKH6)g+O4F0TrbE71{?An3g#R5!# z!Uqg&kuyy&kuev~&I1nguxk7#2Fm(*?1hT&Mb;F?d)Gf@ej$y#vPx~CpY13i4tb?1 z4Vfe5_F(@Ul5h__J&Z;4oR=ROI(pK@r=(;rikkePV(eZ5zx(9#1Ck|$2tFw49i^Ji zg_XaE@P-CuMuIz>q9QB}%?uw)23kQrj~RLn&Vp@RV;!>@u7!;iH}?gy@9oNfKMB#t z`vpJEB7Lk2+AI@QSgK1U-VEvMYcHQA&P`Rir}WU>EXvLR&*zWa6Sf++9ktog-@c#i z#MEyFCjM;x(nBWX?u1I+HHJ-od#R z!CTFYnR&u5;G=q6#TM)Wu^h{;_wQP}1ZR1)8kOlXz7(tcGZm-q=)?KWM(K;}ia^vXd2bXE z%+5p=hRvfow+JqpL=F&3Bj2{EDmzi z+Y1+0)NA)D5724qTBPr_7%(+$U=)oKw3CoFpyOIZrAWG6?fHoD$x>laOw}`6s<&VXw{GuaT zQO$yYqfVu%TFHP2a2c##XxZvay5y?nFU;f>(>g^`GBJsD}qP!NEDN8HU#Tbb$n;&ofK7e zX|b~n(cLvMr967om{~N9Q1$HcMywh95(*y=FG6-*y0>z@9~CTli06~IKg?mtReM;P zY5w>xoXF>Orsr@&H+Dk5MDMaA04K9$(DCT-t{++q%}8V~m` zwjKuJE|0|DKyZZNucPw~|M;Uip!8KWW4Zt(rzYa4>%|}I-+_7x3sz#zD9?C(e(E^G zVWoQ;-8gR#G~+n5DIQN>XGr3mTbO@ucp>GG%_VYKOWGUU08q=kNLQdQVu>nyeelHb z7e4#~STj;y*$1KAYllVjkA%d__&B)PY$q|tcXJ=R^pf{1F)oYU7!i6Hj<<{Ytqu87 z<|V_$Q4-H|H)`7F-R%Vib{^EdyE624IYfGOgCaUVRx9uBtuAE_?E$i$16u@}%BeZ; ztyU|(-~7qR!ENf{kZ{qQP1Za5c@5i6@CZXC!h!O6|0slYoQLbzg5M%`D|felxCIR{ zXTPQe$rV*pt|FUCv|8T3-9YU)T8l|PbTf;pkCn{KNwx5T4V*s{xv3{krHe;xk#@`5 zk#)gizxQS$-;P<)$@HaieQ>J6itgeVVgy5$3Xv2hjr%Due*3L=_=1FV* zRUud1r{TVKOuab7bJ$1u?bo2pMxp??cX`f-nm1o%Yfncr*HfDKUOOzu7Ioc17b6xnB}X&^4vW6v?*CTb$W$% zx7thdWWP-?B?fWnF5`^BqFQ`0^9rkstz%8brg$>DWp$LVfnKIt@rhiR8|nZZiP#mm zn=8P3@C&{@iztGDa$dlv1q>yrmrS1x@_v!*aWAoNb}Bl&Sj4#l>o%$*XiT+_K1BD?wBMv+C09R(hzk40UN) z^>YR;GF_BkW>>JzCu1#?Jx)RG6Z8A&eV+TII-k-PXWvE%bWlSMLUz1v-H+hO?A`Bv z=H=bZXGWVhA_<$^zE4)7a*O%+h#_ycgeGjH2_{`hf@=jlTD%Iag$J8|3hBReyA}G6 z!ekeb)Z?Fhm%g_IBhGZ{DuFJjbwu8h^kYpVjoiScQ3d@N17USj!$U1sms=|w+$fga z&49D8(F|D0v4!d=2^FteKtI(9m?rdhQE*UQfVYoV+vi zu;&i?`)#^B7fu+Wl)^nuL95OGO1~4)b~reP_(mMaQ^wlgG=M!A14Z!k_B&NnAN5%z z+BlmaCFZ=x2bX_^fWIsTp75Eoo87*i*~+WrP{#lQr(&-dytVDrN{C(i&FxI)@>@Gq zcmL858tPs?`KJFVq-h7En|+j23E(>kX6zQmaUy_auq%wIg{{{L!^sZwa2)9EP&RFY zPq?(s;Ea{iQF(XpJAH#K%ehF}6(!jbhA}R9SPKpJNE&$N56=q~U%pH_qWPDPs=o)rguNO1R zbE~i44G^cN)DfmxdDjcIf%6zpt@9a#QeCqhLgI&rz9*Z@e{=&$@dC0DcXQ9XV z{bFLf@jJ^?SRm}M^TMt?(raMl^T!OR`8$Ws3oF5g+yoeM_h_9lFTYgZ=dHzTzeJo4 zETzg-6Wyo1gWOFr{WoBDq+U|HMt^)4 zIT|0x^6J=z7=`dL%e@Rf2`(tnwXal5%)YWmA1DXj2(U5FF?vsH6-m3sM!oIY+L zGM%~#{LN`pZ-n6jx|U6*6^YLI9p}{snf5?FPf~vM_ID4*@iIh9&94_aC};k}vvsj5C~WoV9g%Cx z=#Q3W%B|lV8r5d|F5bjSz6Hb~8*?Getli1C1T1B?EKQ&kuEkH0#ox84W?=}niaW$|lIK_RK=r+; zr8I`get5^{wOUVa^nBUiD%JuTlb=;v#TP&npq6H-_^i!g$82pF{kt|F%Do^mnWy1W zJA3M|U%Zpq#JBp46~)|hB1!c&^A;h@(9dr9PX8*D>9afXJkD=!wc0Nv*_lE5O6AJJ zu2QCGp5^pu#_&{szXel6x9UondZ>X8?P{uBTl%6IyhyzF8Fi7^CH(HVO(J7X4$ljo{wD~J@elQ9 zeR>N5H0iHwb<#rHL8I^Z#%^>|xlW_}y=!Hs(pIt% zz3t9FOxcN}eVE5971_tdIjL55;O92C>ZWa*6FmFXO=P_;cM)-+$~ zhmXaxKhu5BnT|nX&zn`6d}G*ADT@93!$CLt> zX5r%4(+o-camo@Dl6Am&RD^74EO*-Zn(CF^e4drVA=UekxdwrYUBtDMjCepkjg)po zVe!_daEnoBJJ1B3CF&v3+iGoN=o{6htbF%eG;ioa@+VJubyZNsjoQMAlDvZ}qfw@w zl`3EL*%!!gpFsX0Bkn<+6$F7j1Gnw5*pWgvn@Xk(_OXW zG@*g1nN^tAR$qhy>aPbg$5sy9UBG5PMYA56_~%f@U>_plh?)wE_Yw|cW%}}*8YgnD z4~{F>w`cX#jT6;8t#fn%LQgS_QDUWqJt~bb z)Iqh#ys-ygxdL6u*TD>i-bu<@Xtv)-miFdzCuVc=7Ccs5$zLZYle#?YnFDpWI%GA> zx3`j>@}~+jp+Y9^N!A)m+FH82LSDY>zjS{WT+w@XLLhw&qpA+(wXRlcOM4M-eN92| z(V?QEIz$upYjQ2mhA#W(l@-3xwT{qrY&GRrXhK(1P(`oB%Z@SioXHB*BV_fd;O#GJ zgRSLoVPQ3CqolyGC$np)$=g#{jySUy23Fg%9}hKYnTpnX+?$>lhup~RJ28f(x)`!a zowD)@5j4HtZId$9<*KZzeF+kPY^kJmhEhgRBM~6Ovt@;EZN)=ER7s@rl{V)0c~qZ< zK4TsTnMRFW0u2*78#^*~3Slt4n-75bsHDa^*<^}8o=F48>{$ds@yQMGyTKp=mzRex?dfMi>L zht7NQ4fgsX^F!{$vZktG=GD`-k`yc8A3yMaEoXy#{<(+F4x4A%vzDyQy#ck1P5=rN zpBys|sFi6}op}@c#t>eLF>T*_rM}3vlD}XP0_`&d=_YxBoJP=axQ@+0lH1YpxI2b& zH95U)^UprTOtBc`&6VMV`5lz($8BYd*Md6V2GD2Q$BR|ixv<$MKmX|q9^b$$roO7F zMn#G7e408j!*>IjPs>&*ul)@C-LEzYOH!aL)6`;%nVWf5;7lA7Hju}L+XVou}e&Nv)_>b*mfXPnf{QHj@a0F z<*|gAT6?n309)#2ud^4oC$rJzk>N^6UT>EYbTo1Nr%zsXtcd)-Z zE47*{V=s01cs}Z8_dsB?Y3eBl8N3?PtG5eb-F#On>{#16Y+awn&wgoH)S1B^`Dm7A zSI$P6N@gv-5;Y{pNCaQIX|3*(YVg;F<-J9yib;V&2#X_4?9US6y|00O1&i#_kvC>~ zCdxjY3&@UvYJO%_7n|=D?*NV7@;N_e9UKd_-cCLcJXk*|>Zue_coj-kY5_@hlC1W+@_gE1CdMme>c(c}LGdwXu-Lq1; zBVt_;D;mqft#8E~(ySHmZ8s)rDTdoG=_IA=<(`9Maht9hQ|ZYMKE*`nhC@wp)-pEF zaoc8Kd-XkLF5|pC^ivua6H*2WlkAC$GPM9yoqKzFwEb7;?OU%q<^57CbzY>WS*3~D zXjy-@OBG!SCU75!82gisk zo{y^82Y@~GW`Kkhh+&6=9|xB-X-fHcdT$#qm>7H8EVIYn+^5%;ePH6g8|WCP_<$_> zbbezVSy%^m-K|x_t9|Y*^?Bojp8F%KvwFS8i)dkH7G&f1QI;X_^(BN|%o9?je@7y$ z=EL2yIHMfPlqV0qD+aql;9DDSJqJ`CRqID)zp@nS5zMIM?s@-I zka_uB0q<)?@i!FLKQy6bHQD7Z@@rN_uXfGlb{Ke0-xE}`_R(T*vNWOr>axybyvg@n zUY`aaAmYb50l+H?wIu*BA8oWq43A5^^s@T!QzD1ebN0lsuFmJvYA6pYY&5`@L9;r9 z`clcjmw?%$bG`9;WOdFCos+Rev&s*RvZgwszP|^EznFp|PMbc@HlV0)6oER9u4Y_f ziKIaKDk}8)$2u^{H5I!{A>3nKWjxRF!N-L!cBR$C(&hoW3PHab<2t}LDvD5=Kb$!( zc%p5W9@N)hm-o4X(tTI%>hy&)int+s-&90s4pV%=N4`s{Z6pf)kO&Xol``k$ukZEV z{amKut?mS1#=d-fDXNfUJAu%2{RoY>YE36SpT>` zC^Z{M`?>NYpzejB>G`B}iHG>zEo`5U+gX6{{h z{Nyg4&&B5xpxI==YZjn+lir1-d=E%_@mV#ukYFnsVU#ZMK#uOk`447s&0I(KUGN+x z?x3INK1_^k@LWaCfBFF%rS(`i-=aJLjR0V)RrW{sSrpMBOMu%--d zUad2{`L09s$5B=#K3n3>KvC0aDu=U&D1bBbIOILNH^PqZwdk%by{=mvws?8kaPxt0 zCnbW~dBLKka*ZO;#51=`pCDZmK5wwK=FzuR(9)qxjkl73x5?8TbGDDE-=DB+FYKS(6t5IskvV4 zwK#hZ`z?&Wn!vF^bBTk2ox z!)4PdB?5HvWA zEw0Y3I;j*Ly>R2t+U6M_)}p$2v4;~k!O6+#C~CP|Niz*zn+o;6u_5Idd4Pzyr^H$7f1UR z{?9*0j@%|smgjc^lsukCsM&!owRh@)iy{-mXu((`}gT9vi|y6AlKy^K);z;5NA zo!~6!G;TN!6?lFBhA%3O1Gr76^uO=m+<52jQYzel5ep-Mr;YQ$c<0e8h)!S~Sjz#!sc9wRx&HV2|L^4g|7r4-W|1>eLl-uUS49_{Y-mNMz2^OI z(dcXbsy7BsC1D-oddBkd7U!x)`b3V;f$ib=P@wQNuzpuaYJZMrH{!nt|1X;WyyS{i zVnNP$&0o8m?R4Y(VHu&z&G3td{Y$R){Iq7rJOA;ATc_~9LOf~$0WTXptZ^6jMY$sQ zj(=-e`pfS9pA=w(zZtK1Gd+NtggLeR9Ak_sgSjwt zy{$m<4K2S3AN|XJuibmXWcaJHh_R|-lPMNd;pd-u``U(j0vZx z>w?SSq%sp`P~k6LETB6889`>CDsaXHkWY*N@?iF|?4>rJlokoDaOz-y-lC*DQY3Cu zt_(jKgjc7qs?Rl^mIwHGLh=W{Zd!ET*3TS*ZyEk)w8EAI>FM@TX~DzS2lXz;B{0f3 z`3bxGGlY!U^i)$Gy9a$SDbDxXrdcBi)@quLzj^_We207FRdFkjLJ8fjfGP5~9*-@1 z!7k>bH|)RA<1Q9btN?M%kYVw(z99_Jy5|u#-AxT~g@Xe94crGMYEaYwA8WabUVC8Z zf3rY0Fx>d7kKX`{8%kXCmx~n)2()!@WjKgx{nj-i;>fMHxnavmPPcrS5K z)M3bJO=mDM`_vRY4zd~w$l}QXprl0-oFjPTeTN&z__N55ktD-GE<2g+%MupHBmd(` zY0w1&kSk3Q8SF(vxMjYJ|1Brj{3O+^yiJ(SV+H$_q#*yvTXmvRo+Y8Ap zpDY11H$OdR0YiTL>AYCe@u~oViPLEFG&VrVY#Au!9;EZcNs2nwPZuRTu9maA#-ag| zudAAs{|=SRc3Dja3cyZ24{Qd&)BkvNj`v3ZQfDO8{4&a~!z{v_MRt%UR{PmlCLxlX zo2}Af>g>0=MpH5Q367u}m-r9l`PWkt@jrG_yd1F0qkb`Wfqy8VILbjKKt9t$VgVUE z3UdJW*l}CmgmwW~XyG;r+lF7SX8q}Fc!YdMqpJC#J61yEfN+ejh35zp_-rYp#xH`+ z(;!vO%V`e~v@QW+GXF32@p4Na)|c&bwj2Rm!q)}HG5!i=KKM1QyLQ=sT|{u$umOgb z`v6Cxa?Y(k47fj)+XNbKL`JZ>J(UI!kZu(u?ErvsRm0vN(3u+Y%=o6GwdlBq(*gt* zK^AsG=RfWP*GDA)o@rq*ci0foq336Mhy7A55VX33Zi*7DdvSIJmm-bgFA(xN?Wek? zi>H31Y4ML`xQnmpF#}X>|8F{+A=Nc-kqrZOU88724nUIaBnDuB1Jtuk&-x3Rfg6mB z|2m)f?`nAc367|$3&glM<3e#N&o#txZ&m=UNe`Tkt$SdN=~Eyf($~KE&WLmlNTN-5 zFd`YvfSC=KaYt%wqfRQ3qywk*#XfnR-QEDgHXM-nmWx^o*!MG^0(isWQNrQK7QA7f zA@}Ne^~a|GVplFoY8Sp)ES&$CdEjoYxn1Gco#NR4Y~T@Jd>{)DL0GkT=bBOQawsPv zrbzFMt=^Dx=VS|xIO5Zu2b`gOw2vI~GXe8(8&kTmRh_bZ@$bD|1+{HB=%m|UA?X@t)Z5C&w$5p-?Lynu|A%Y=^F zA5y(pZEj}ix_z@(gqUUOz|=rn%%2K~hxB@q?hokp|M)|-pZ{4@q5=OyRI|wYE4P#F zY{<9M80HyvLav2>1d06pk2@s(G#98`(onhruuFV3oY209rJ-=e)2h0c3l%qNNa^_s zP(Zq_Zwx15hIn3#4i$K<@#qcJ+cc}>!v?Zp+BX2QGp^!iRm9IJTvfwty_pqXa-7!@ zt@a$B+2QAdv|)eiK2!$7OI#<=8!}=sK^$(gd7w$d$Gek}9WWsWvx`?7>N~hF<4rCr z-zFSWB`z)-CrY;L9C%^;D_IbEBU}UG;txUz+T0=kZ8 zD~1kL-DEY+R}6r3d6@~h!k@;ctry$o*XeauKv??eFb%d|LC@yNYuGbkP(r9eS@v-O z_sNOk_!Y3}1@7Dg3YO;{kqb;Ds-zLmBm?x-){T~M81AsLm)auv7L^T$OXCiSok;*U z%{*!1K6uZW6L$LRCiOdyuf&G|06*rmhBL5;{eLdIm+?E7T^+3a+-L=0w1 zHhN!jDa)-fbpFCXD1zYxh>Lv*_;7?nJ__vz@6KDV-nuLH$c`2C6L-FL(yFo2juf<9 zVYX3?Gz+@`41C)CM|~!$U1JP2o^vJM(D3|zD4c+--ZTmTr@*c+V9UT+204{ie1zGt z0m}t?PDeJ}Zn?g?hya+t6n$eQB}i4;+zwokjS!2#SE;vKT*# zo+K5TAH4`15}|;W{e(GH%YK`mwQhP8G24e1QyvJ@mc3%IzigGrb*vYZ6YaLwv=NRg zjGb0nKZ-ge2$leOg<@CP(T$gSuoFJ&v%kN++E7_QypEedgV;Ml3DA{p7}l8PVC8fB z#{&WCE;Hs=LP~JQYK_;8<%ypS3+cqcEY_rE=Y9G}M4#4ocU&`LX#&KA=k%D+8z7{+4CWNwj-(aMt$g@U3 z9u$Y|;#Q<>18B@Mg_phXL{fEm)gc%s4*ga$uLQ!3L{wvzW^c9$DPPwD(PHb-Ul>c! zg+Z=>WFRzdLSz9hnyhMfhTRX$nI_#>z#a1PAd&@Mb5=)Q@}^dX8>H7j<2)C+7=Z8q z%f~d`?jd&jAHR(k=eJS!g~Yi#ouBp-Q7XDL5;COIm!A!wIxaDxFU%<(aO{(($m8yt zN;NoJ+8T5jbd#zVeEB+Q29UuO;!wg#SE6P;2vbR77Ot<*HjjSc$ZK)UlP6X#B1O{nR=s#b70mM}?qS4;TcW1qZ$d}F z4csRi4;>+Bm#11D08p^I9kza?B{YhD7UGXMjx6>7TJ%8Z8@*B%7wyy# znuW6-DQj@fQBpBM{Up#VFNTu{H&OBfAp2q5fTI#j-Jcge`vq%Ug241Vg^=?Y4))z4(G8j8^KT2lZ(=TZ``~Ms zB_h~-;vp*HP3)0M>0ArfqKG1M?>!c(yVZftmtw>Upjvj$@BV;;l0%+K%j^t9ymh~{ z6H{xlB%_1ScoCA~i*&+x9-dWeAjb!eFCMy_dXRk+CwJU=!sAxU2JK8@?AR8K0MFFD z;@Qdhf7tuVs4CaBT|kgfI;AEhAd=EdQc9E%X=#y^?(UQpq#FdJq(e#?=@giBcXyv> zuC?~wUz|VZ{~6;QbAZ6>lXqX&E$JqCD;LlsEbo8BoI+^Ib8hAqY#e)=_S3vJG^GBpWQutZjCL1HY&%Sc4b)h+4aM% ztgFavcB5#iYW0#>Y;bnPdTZ z#nsM>LfXvqz~$i})+U6zJf_sZ{Qn~Z=UR#vwzxXzy#HRH+2A*B}p zT}v>qC4-qWG#3#d5 z>r8dd@B;BVAB7cgsSfu@72*bYZ*l){k#kJDE5v(8?QDdIBn@jm`Np4hdZnsE zRCpn@^Wy$y@cv`H@#MH(1WFO;OEKVS)RkpC%zDInaCc-Q?bx27Vi7S*T`B#&(9G=s zx^;!Agk0X~TfVF<@rkt9_KFaOqlvH}eEFi^l`97qR0Wm}b-777w~pMoVC~qxcETlW zr2UIFv_s;G$C=W$n_7Dp1B&ImukNl_gJ1#Kht_8>7mqzX8sGNFOQ#i#kkq?+zOUY3 z6i-wV{~Mocbr3XR?PIg(A_LtOb{Ymmi0mhyB3Y3So;Xy?lONIk;}9*&!_|JqrQmUv zW*bNPorV5EdRo=x5b0dx4b(r}v9^}|&Tp>D(z$~Zm3L5`WHU!MtQCp>Z{Pn3jrD}K zlfwnsA4#7##()Zc@Gk3DIX;SzQdZdXnw`i6i<{+*p86ge27YdPj7~75sP9(V4qe{>Wn(ek=nAd)dpeNhnB;p6#@JVZ|a{;TwE=+z*a6h)uNwM~iPvh!DJj)@^X#_5@DVrPC zd5}%pAA>ZC{)?5(P|~GKXxH^_@3W-u315`wmKnMBCB$xH-L!&V_6l0Uvc|qS6y#gx z?pkQ1hP2+^XyicUl&*>Zr!kM5{2BC=n0Pby;`t}HgIC%|Gf5@$E4?+(t;4Ik@^U!j zKUOp$>e%42HA5^8oyKn&QwH`g@h{XHpNbrkq2HawI?-VS)bAbpaa{-pUtkHH3_ZKc zvTk@TXA^DlIsL>}L3fNs!OCgrrGr|E4YliM2De}fWw`^8tRnM zpuE$8)=kELdE4~B+je{V5OO^do_bo-n15h3#5ZbfcnCSKN>=)DHS31}v;Rjib4mbb z03dx5E92`-KF|57K54nnv19&-|Haj4u2L! z&Ds;iVR!L8HZ$+It7WO^C?+$6pmU6j!hnX!;* zC4JrfRp+pg$m|sHdFq3xT;HnIcpg4}gX-Q(?n^@y&}YSjaPBM47tfDeNQ8R$ZQxOu zhTbzcWlZfqlrg7p0FZ()AdNu;1QP#{#xw$;S#yZ1>~;CNfW{DQUvZs|I1Jv4I$NCP z_sq*Mq{d!{LcQGylm<-S6oXP~miJWjI4|AY94pI2o>H=2oG6wwl9MybdwtCOZoXKf zmh$soG4Ov2g|+9rgcx18X1uFCaEg|N++0eIPfOANBf=vmI*?P}BdPS{l8)b!mhI#{ zAT#kn1^g`hW|8N%Xf@_Ekq4bxl~2-irS+7bGM2IXQ8s3DeB!*iaQ2+(O#gz z$y(=CKwRyF|I;*h!wTW6o0;R_T&tsPNd~kg3m-(EFsETz(E+w3tAmRj# z{a$C;5cayad7fpWA51eW&uw*<`(8%R|rGA9Oed1 z9*AtaRotJim3mr=mQ|$RMw98HDWs?T=tDEZ(^Y7fly3R`nDY##UHmpuGQ0SR&z))!~ zsi@A{%Jj8XE`HJR6x$p`_@>iXU2+Qxn>kQpXtmOWGJlzEE8#bcY&&ChTv=@0w#3N54{eQo`q=y!bQRN+-)me6s?s>2`!g9$gi}suHm$S5cclFz z{*19lHEwAJ^N{3?93Sl7l7^$6#PHp}B2@U{EFZ~v{Q7k`nnB@h@F!)S?&N~59NhKo z9YBVI7~|BQ5}B=b`&5>oJ;+4(*(0p&+97>Kl=(Bgcp=s_!}Hv1&tsO)wHeVu z*qXDOs7A*UL@5_1r&qs9QO+Ut4ImQvV1Dq|xQBFu$WF?%6A!Z|bVLi_|CR zlPj?M6Ui3YDcPc0Bks_KIEuCi-jk^UyZjKb(!}T~TdCck5HD4JIL!N!+sk(-4twqp zQTQHLd_Ql(rIL^=>!HIM3*r0AQg?eSeewwGlEcL+ru6{jwaCcfGO7g;I+OFA4HSCE z;=MxwvuC zE;vqPrSARgdf|KI{G)V$^NG`XJ0I!IVOF;W5e4mMgb`jfJ+x``Hxiad_JQ}>X~J3B zw~07NN1^ghuGL3bQQ4o&M+O`j%a4C$-)B|3pNXm0cS;LfvKu8XYa0x=df`l7OTHR_ z>O(9kltn3>G(JLXpYAas*4^iu!2X;ldAUUVC@jsT-~@sKka6huSEUWJRM@C0GQ z%Dbl`k2gbMGt@~>ww{pBRnqbip=9%??aVxCDkL(4fDo0tP3dd!+Q+Mq*`g1)79++G zq^GzlcjXeDUQILbmQjKE4puckiGB#1BG9mlXZmRs%5` zWO%$2)zD_}0d^4aj;(3s42Q{Mzkp=Ud-uf=1q&j|ih~=qkggp!y`%4#~rcn=9e-P;r^MYS|TFl;Zq)5}7JA8qyKAXbr?fKkzwugZiN;;c4kygZLWTv9Jd0$e;RR4Qjx7{kn zSAT>*>?Ujn1v0ujXW{QNTAA%JHtc#%hezHzlWlXst|~k(+!~^AGw;i z>B}>cKkl8BMY64LNuAB%WLBE7s*hfTN1gcCzE9Z8c+jzXThkAGk276iC>(lt=>L)l zt-kTsgRY!%9gtj!Zl)8lxs?4GD0po@p7N^2k|6l_b?WR4JHZ+C05c#vsg;~&T|I*3 zfHo}=C&M*@Y-1jC@@ml9i?Z)j{?0OcAX^xGq*x3%Ln6mti>4Vv|BPefK0=dFQR2G_ zziO4{+!nsSIo=RPUNx0J#|ysUiSeGX3`}&0k+;rwxUPmc+Oq|4omlKGW(V}^@+aG8 z4Z^T(cB!c*RKyiETf3r)20UvIAS}fTBAZjO+@AY`XJp*Jm)VYAA<>D#A2|i8A9ar0 zRKhpruKMjI=X9328pZ5_kDJ#Lt})VOlNIISG%?nh&HHsk%I>^eH<6+{GVJzNFqBQgu(HlBgoh_@n#h7(gM}KVv85npN;$X@?`I z^R|aB@quBRC{n!W=mzY&Ta0n z+&-3H0jQXE4XW^>*2g0(pyFN!NegB=PYW2(@Ik}ECx?|tUGE?Rq>7>uDi2L9Z!QiM zr&`)E0&!U_cMcE7e|l(IxT4h;tJVzq2^E;)UXK=+9NuC(5JyD*snGt$?zp4w*6ZPZ zbeL0I-u=dpd>MZW$@FD+xUmxdfTwyXr1F58?l%yed*YwIW22)*-oR@I73*7=m<8~t z5aG`TBl1zwC$-+?jpw?B$=)~qWb~@ZvvHUTh%W!_l&(Lcs(IPU46}3`DRB(NFME&3 z?~jsj5rjmJ9EkJC7n!ZyTXbB&q($7hpt=J;fKHVWL%+gdC@ye{AmOmIm2cj*v=04z zGgXanjBh%y)}N#?`o74lf|rM>64OLKOV=N37jqYD7khU|pnZKCk=}YK#j^B0 z9iuGh)u8qDj3Y<;wJ27xhe_uW$@<8i+H}OQqsP5C|85X}80VeAgS}0uddaEdbJh2z z-+W;#Y(Lk%^d6Td{S40%i#W;;H%af$S_TavNva?hkES0APB%S?ps2E(C+RJsBD%Si z#?-_~U_QLXfnYU?dzIZP?CxJ{N=nz`(eH_8d0>r;i&~~f!t;fSIu$o)+k_6GBJPZ# zc0Zq(&_$sdxW9UBOA-mMUd^BVCDO_EKX>r~>Fm%BR#oHiosjL|^&?#mPBs1-x&^ zsst}KoRWvwnxA3!~S8oR%R7Ejcxx zQnSHP6awvbfR|FH4&ux7PzO+^C5 zBU+a*Fc_vB;kjShhGl!TnfE47J~itT6Cd%ox>NIC>nN5MM#}wK*_rM~(^WzKf&{2k zFmv((dB$ka-&V!-bc-;chQV=}&r3TXJfiG=H!_$knc6kT-Lj&er)2NyZ1;HqE)x@i z?ad;pL*dJJ`$7V+qj)o2$74Q;42YCO|NN`_yW82$BBq;&Ej+Tig0t-DZr0bb3Z>L@ zU=24|zw5nMHZiv%PAyc;%^#Z6R`N;d)C_x z4OCLZ)0Hwy>kWDPJ5ZA2u%WgquWB~wuCUep0zCDOYa~S?(62N$%;z6=R_ZCgvsmX3@|6LR}pLV!2!>y%8pOy9}7f_}cximA#`Ld(pfJ|1Zk(Kf0e zvU{_Xo|J%FT0S5gG)C!MsZn<7XLqs=%4_#)cp~K^=UL@(RC7h{#Il95jec zx2ogCJw1wD{J2{+5z$HDtU#)1gWbR$amj%;!5*<8Z9FlDicPu0Q{AEd4T7&Vw}5)t zMn5`9knrDiw#z~7@sz?!4TDhe>mHZ(KRgiw$M#>2Bj3yYA^9mfo!syx1Oa6wk$o5+ zJqiJREKpPjdsh%;1qUW(La~m;w}8M88bX9xdK`8T%Wpgu6WgHb3p_v_z@Wc6-Dc3P zcO=x7f;b?Rrq1t!_A)Cm07uIikEXq2F zn&s3%T!;hppVQV7>C{KaxA9ddt=?)H8GCQ-*|3-dbM!4pF@9 z=R&>1iNGf1$`9v#u2-Zg8quv%ulZ~I(eG(GzICLdoZL@Y*b=qwpt~vo`il9qSmVp! z&qwcr9L-04^=7@OUgJlpjdn$OCV7+lN^{%c8Y*!gQ{a1Lj(J=ZR#J!~d|wmQdPhQg z;DZIheVJ$ELT;tig&K6d->gfd@e!BTQj{bdTgDVi|Au6g5y|#M!98Er4o9>w&gEpW zv{)}Jw6Ji~O z+mNn3w|GLTZ1cFK*RUP&;p82$lyBLUKfXN9ED28o;Wb&&q5czJr+-V)S8!dp-m$WO z@1`)&bn>+!>-p#P?#B3-;)zXN`*4vSx*;>BV`et^#&hBMG{kwh*dct|Ay!D@diGu( zGr2-$fc4S&o1Zqac97^EPL zqA597%0)>}u+$UDURrP2t4c6=;~j$csaiDP^P1(&CdhW^AheavMX{EA541;2rp~Ri zohCMae$BI&xPT_bj)HWIBQuax5;Dy-7AWp^au`IfS|pNCT2}4jK|-#di*Sp)D`cOK zzt8)-;la+7=JocRNMF}4x^S~E(l-99zbhg$`!6fo95Hwp^VqDq3&uPKzs)#?xt_VS zat!!KMv_UhpWU%oB}D(aAfE7IH`)ve&64{xyjI)`%Pn$wOsEQD{5+r%MY?)?wf%$b zCvg(m%W3iZ>g5ML!9vJGUZb)fKR$7ba%P$4UFy0B7yi&isjI+F{IG(LsXK5&9d%T# zRD7>(l2+EX<)PCTzeJjST{Y>aE_Q1*4I#s%(NS!aT>OtY6IBy zb<9i8xtp$P^)VhT$MQa3IbMofz*f>RbrZX&b*B5v6r%DmqKlf)f)1AJIt;F~Zq?7E zCf9xCcC62{KK6g+VpL|hob}Kp5kjz$3b*i&=!Q&mt1PB60v@u6B*?VPe|f|DOg$}i zrIe1M_ou_r9v2(xW0HlwAp6ac4iuTu89$>MjOLjyjwUW~%TWpm_ZJ)4kttIcCM_B< zbT&3EB7OGiqqIzw4oRB_UNbgbzu~+D#ABwZZRY4H-1B(bb)CegyB#)ui70#~ZwnWS zK3;lM|B?-W?J;nD)lZ_~dcJhqNug4@#PweBd}xg&Z?DHf_Z7s*S$Sv$M%(xEF*)B` z_v_ESy2U@fTlXcdB6MA@r_Yx$cVOQ>SPRm_W9ZLpcDB$@;CXo)Yg@HtV6&MuFdX?7 ztLP!R_V?>QURAr3ohp0$TUfa&dYum}YK5kiwC0M5Y-^E_JP>!$4eV-1=vhl_e~g!V z(6Z$_%`&u{qE=>RiU)O8f6H-SuJa!!APE|}XFuyGrhkQ@+q?az4_lQ+U-F%sKv)fl zLaAAyW{qVzVB8RhF%ADSHywjJO8>Ws>I}eUCAT}tqaW`%(_<8J-IWMW?n2LFE3ipe z#ZhC^7?i)h4MooNZjfY3>HvFbB8wD$HB^F9%(bY^GV+{39?h(%c#=+x!~ddG9}CrO z2HwM&|2J2PE^|xJ6iV&TOjv!lmz-oq^tHN@k9kI4E_yc^V{+eXQG~VbhS?)N%?%%? zyMGP|B3?eCk5t;H!4#D!yRK$hwOK)#Trcj5qLIPr!osjAv(lZWVNsPZrss@8c{mx~ zf+$#4(Fnk6DMsfy5{K@W^OheBSVq-?cHO_>kQ`0-`W6UJPy560{;dqfV9cEcp>005 zjsEx5s{Ky1=VhwJF=oATh%Qyg+mN}Ez80y{(ijEle>^9;w_Zs7>=S8Mmyhdi27cRK zw<&!*njU>e&B9+9Z~np-cgl1OVROphjA?fap+SQP5u>}eRdNvSc15xI7&)7)t1L+o z%~^oZ*>JAMq{Iu$3h9J%Z+<-UgkZUTdf9}isaP4E$H!r)$;BUzdHGBR=Rt*%i_~B( z%tsUq&hghU^_Lv)hRcFO(5=cQnU;2al6~CLf5!creuT_lj_|7t7j&*^t?Sd@y1mCt zboZ|U)-L=Uo#%CCbJESie!TdIsL^0G9ln`d<4DT34&wAWsqE5VfGiRjQfa|irU#zHPc zoJ^Cg7^z(YIn1h5PT__QT3#+gW zqIAtCcRy&+@Ed`^l?wb39r$YyoIm5jB9M;aiX4fOkY8meqE(KFyZtHBkl!XHbsUo{lH@6XMSw`>e& z8x|HN@YKCDqzu>}B@ivjaf5OCUK z1|6$%Pj;tkB+%ymzLLOk%qR3wP5LzPS^s&*8$_4q<2=2uK}5qE`C`bjmPtSG?sIZ&CmC3jAWbHmeQe5dXK- z`p*|bE67(To1-gF_0G1%ypfJv-x`s^e;OqUzCmVhko}k5_$%)A_ur`iQP=R=n*}cg z>4@mN2~iA?HXP{Ai2M*(xQ0j{;s2V8=rQhsR{iBxk&vA%qKn&zF$}zjMFHH*I4dON z|MKR)-%l6Ci8xSY_Yl^So$cM=r^X%%wAauiTu|&>>F@pL2LG)typfY4fuQJDGu}-! z!04Wi5r&%qCN4Mo(a(ZJ)AYr2h#C)T*GZG0qN*0XRs3J!S`th5E#omGTOi9_@7x*86%~9 zT7O*WCoPTWg8D;`C=#rH8Tvuy?Br?N4BT?~?++IQtY-3$0T^c&I1~&XxpH7NkKn6$ zNiT}<&yxRpvZ6@*_zPN4;|vNx0gP^_GfIUs_?2)uuvwiYkhOn#zW*6c#w8jMqwWrW z()fWvCA7(D0ai1P0jy>(FCx-^J)D1k-JJ+Fi~qsy!my&~T+8!>4xlkj%)w^yI^ME7 z{qN!QOM}&1E37!{<3L&2l}R@RqPbEKtfpoYI?jKeDSYBU8S+gGJTXCy1iBDQr-hjk z{F?!Mv&y!E>fqY>zdt6zU^RJlQ>%<4o>5sTT*)Sb)m(+I=8>E?!hfduuYItk2b(n@ z3U#(8h;ll>FggUkLYfRVD_mQokK%u>eboQ^(f2V*{?yFBO6KA?!j*V6Wq)}Q`Xr+B zCA(qG*Ls<#t%m@Bh93QKD1GN%|$Xhn0!8 ztFtkyXXr{RyCJ$caW+FyC@a#R*l3{Pq*Ra2QW(bBtl|{Fy+2CiK!(p`MJnS7C9IAx z4)_1|Rthw(BQ3W-PBiP0UYaGfE??ZQSNqCMP7<06cbZP$y)ZiYFjA+aA-2gM0EE&r zYv)k73O19=h=#zbSfR^sS@u63xZyNbcUEB~ALDm-f+rgcK_vWz%h5-GnUZGe!}Q~(@c~A=xV*q7oy$@P z!^fS;GB~@yAMWIudCz(+_rs2(F(rlH@c4gQAm;Pe_W+s_-Q;oI1I!pr`&1Do2GLv~ zNZX^x)5*$INudg(c?wR0$4}$6%F8%=X1FPQj7TG4oIk$JHBL%zPn+wDmP<(+aIedd7oHQ-nbW&qob@8Yh5!# z!5OW`2RRCOQ_UU#z-dUJhwCwD*LOyr^l9sZ?B64suh!-9Fp^C3Nw#j?;2Y*LzS6kOo@?$fEDH8=YhU2?4x9~LrCy#!OcJE==79Z24Zttg(RJ8%tMuYN{@2!XLYumb58Kz67 zn~4-_WK)2QlU1q6fD5fxR6l|o-Q;OVqd&16Bl~oK4D;R>Z&h9$nE3gWar*Dq&x`4F zy>6?RtH$2F&~Nqs%&j{@Z>s;Z>LQhn>K^0;w5HsXF5WJnY~GPZuHsUS9yxP|T15Li#FyeC~$YNyb5T2?N_0o9Qh*% zsukB3{kMhrb3F$@OJvVn&6{t}6lcew5SvXnU*Hia;-~rP0D{xf7_`G3u`%ysciXT< zVnK#!2)+FfcNAJQ*KzI1A7S9YU&(ZyJfT&8MBaz#a&DL2BgP>7G{7BfqLkoz(Xn9a zN&~H040m&lXtG;FR5-YZQ+&h3@XI~s|5qH60T=K!L~%~|%3L-b&AT7V z)q$h9vKR!36UXA00Px6S<@}V?orpwedl5#ehJ0 z7o{7Q{h=8@rC1?R4qyv98?9JqeXMdJxB@w*^L5e2Y%96gaj&p>OLGT2Dru8WS{u9+SG>G6~DO)>nB zR}-riof?)uk5BjLol-d7x!P|lTfV(g19n)&lfRS>6c1e9)O9)}e7lC5wnhNP7y*Y_ z`6YmZ`H(w|DGm&BPiWD0zmaxGxR|pal`C<+Kli$4_*7ITm7Fz@#QjE9(}LSrgvhZD z0DvTG%&Wq?!id3He3G5tqnYFlKVG&F>A=TNXvi!oqwV)*>Iy1J)cs|lI?uc9-1nUA zw{)|Zmz{6zR8nY=#T0}ub4GW_+4V#Z`(t(c-&V)&@U9lGr+euDHeN_=1BepN*`5qb z^V;)+_0#seTs;ENr$1lsLGrti+`SWtCYhCH;LYrd<|j;WOJ2>7-MH=K(lPX7+7SR>%uE z)eOfek3rTjIl7I_O|J9)Jk>1wH-YE6MNqexh$c;pt}7Eg@Txel?`Yss{pUZ+>5Fu9 zL62}9HWcRD_C|nS!DbK01)@p%ptjs39Co$k*K#8TA<#FIT!=1l2Pfi4#qv~xhjrr` zs+xB|fK_c)N?$~1Ri7;+as!Cs`j_`0Cza)4ok8x6*-&V+%kNlyUa!VIQy$n+pwNtW z1nP-w&zdR%E$qFj7|Hq;Qr(Y!TLxegi0@q6@FUqqjGtNvx%5{R{yx1}cG8Z{A?LdQWn@g}&Pktx|XrGTue<+b`SL{Y|WLP7C z-qOo;f3HyLhc!wtw6QUo7uiVY7V&NS4Re`<(JuYuI604bf~I_;ks5xo(}z)t+J4@7 zt-HJP=C0hxP>ti6xfm`eV)q--y8-Ba?Uk>^?{_?V%LtwTq#ZF2X3OCB5VJ=s(#EDk z+b<3WMjJ6nj_siHy)Ce~R5Rr44@eii&i*dLm+-(9Bq3Azslu=)t*gIW<#X9>RH&Cf z9ushG0iyZG9Y; zBrbN2na|woINm}({^-}dM$4^6C-d&wtfmq1X$%`UoUB70w zri<38oZYeS?$b#cT{bJl8Bh~&>)iH!IPK4|T7ByapL#K67p6p_pl4g?b$?ek@pObK z&-?&;6k#Ex%zojQ4-zNqctST*jRF;RCV)ZFmVmHKOI3f-zOb+e$$92&m=kW;^taA- zGvt+xH_nvui7z_FA6&hAdWrzuaLyBT?`5gcDll!Z?V=t0HinyQvqXW&+ONvhnFh?F zwy(q%Y{hI7Oh3N$_+D2zpT87w0XZ-&3a}aO8s&;B41>niv(NL!C||e;6)GnDj$wFo z#Q<~t-lU>@@xn7!srp5h#qxL)#F#KD9;#X+!`0QQiu~hWTmX+;s`JPzzPc&@bz3Ia zq=i}c8xAG86=NwJzo}gJ7&eF3BA?U%FND7RQ7{I?ju& z=S}BA#N%*trqw9fVy$FR;BHLEQEXR?NVy_Q&K7O`_8WzY9V}qHpM>*LjU|hbB{fP~ ze~iaaOg9Q76Wz>P)@SoV8^{6YG)k(3r2ewJa<}$dmOWp2y=-S()K)8yuSJ>V;$l@_ zByPTtT+!vF3qQrzX#mX7S?tCsHd4;etKwW08ZUszRkxo_SN^gN;9bp%DP8~t;;t+y zT{W!!_LwcY`T38Aa&M8ru|ML9qQ}c%zzQpnum&Z!f?)?jzKh;HX8@a(JXcm%wO3$> zA8rg~{h6P^mLK48K3l_>-VND$+do1OAa?~!wVUh%VCso{o`_fA4P@D%H$@?oc#0y| z7-4NDU7m4fpnZLKcts0KfT^(L+0_gHxilQ4`3K)U9u7J~(|B9xtn={teZ%Xj2;(1Kc1x6hVu{rl@^JehgJXoxwKL!pAcXUKOHJUQE{%uVgu;A1^UXLZvnalu zt_p6CMcOdxCXI$iKLVK)4@DP2;#ir0MAMcxbK=h9Tg>?g#d4%SFRd2=!4y+Q#FrEd zq({w{{Z=r4NlGUGPxCEFkimFgx_l4#sMfYf1`(l_)ReO~wQzc8slyDGEH(|Ve1ja~ z2x-L(|Co2nd{^hdPMTOt^mkvh0uUdTFo=jS_1R0r$BPOUmjWUre#63T)mVXS%{R6m z1WBHIBa=qHYMY!I2K1dmN(=gF)Z%ED2n&z9zrVaP*|Obxi_f?7}=zKpGVo)+p#x?s%Ld$^kE&>#7o zk^F3!RwD7rz8&~fqHS9DSGms@poP*q_ zp~*~>C7Cu-@CA(SS;?VU(|ax{og2iGcZvR6C%eQ?eUbguL)QW zB@M+3J2kaDB;&*6NhPVDx4F+)4e5Wm%+>Rw`OX#C>K;qJgxcZ)a8`?0R<5?v*fA(! zd9eK3o59KKCfr)NuoFYrY*(O|jf<~p*u3Z~-Bw8yT**U~BxAzY-SDc4r7J*{z6@7hBKeyHvm%hlE*4hlu(Nl+IAxX&eZ4G;u%%bIt(1fAj?bNAe zj??d_@!bIC4x=|U@5%KQbJ7k-jrLG78QAm}#F(egsPpE)@DLeet+2p7Es9aXFSs}X zU{oB!ix3R@ckx#SM(X@jLa4tpm%;j7y%L}w3)o;WV_hs3(_Jd z0^x0Zk?K=Ej!Ws!Z9H$P-YwU1V=uPRQE)soV8%)>s1jC+;h zDM)y`u#?r{?9IVYA~b1_D}~~<_m1zO;O)uifDz|h^a%Ofr9A&Cfe(eA^)S+LSliJq| zm+EzY$3%7w^0Puc{fjEcZvH?A-++{212WZLc^IGfaKn0+0ay5k^E2O4TsAcw9DUul zD)rsyzZ<<|->Gc3-oh7*mJC)Se%c_liqm9Hc5q(t|Ip)SW`g?2)C^2XO|AYn8a5bya;{TZXFE4Z-(3~pjw9T zITowQ5i&HiJ))xz!f|hvKqS|ZR|%bHmZ!F@JLlI zt@tfXN{*PxH(2d(uWhZYF`=LAI+-%*5 z(DK5#m~Gw!|BQV-+eq}#^qK728$iRrl+*_kI6{Y0*6g)m^mTiKCW!(4Q!&aAZm|CVNMu^6!>;)#Y=^_}>2ceJG zJJiYx?lV58Mds6>NcfO)0HITKTHyH5j%Hkryv*NebFYYcP95xhNyvv5MMZw@Pyy^8 zj3b`|w9v?Bs@68Zj;yUVJSOJ7vjRmzzLAX1xfzjqVMnX&6^KX}H=U?Y(UK0d9$>qZ zAG%0tzN~wF$jBX$ZGQ19WF6;<)p?QsK@4N}17T|zPj;ck=QIb#!XL)Eme$C;?(7|$ zk6${6JxHU4iC<7-=#Tna30t}`tMFWOJapesnhPiA9~q~<)Y{%UKas*gsrE;1F z(q(t{Y*1gX$GMpBPC{s;_XXnvVUqYE))p+IsRjG5(bw##$6QVQ%QIfizv{=+-1ata zuDnd@=cp-nl#qg|_oqd!NQGMsn2&A=c9#K?-dqVA%+>EF3^8wDk(RCVvAc3R=cC=e z2=23~Ex2^)l#0dnTpSvtChy{lT{CwJh$7q9`{RPhP@+H6bxkVl`|@qjx`^=1MK!S{ zfrhD0%Sf|ZRtgDkD|U%+q$8&^+SMVd@wD5;o?$P!XZvTeZTI@tN#C+Z2I!Nn#-pbb z)SmCcOyhsIojW z)kQMy-7XS@dkC^{aClWzF@RB_dLsWtQdGyRsCrp^28z>ZR>`}6Kz?@2=7kAa8ERf= zJ@0#8#gOip{g*HPzNvRVi8)XA#MbtOE@ffzkS8zr&Qfu(? z0F7Rxn^hQbF8#a+&Fc)$ql5Sv1g4nZ2a+YB4u?n*D5N~Wm+)I{P`@InA=hCrL?H|# zR!$(U&*AOE?i0H>+VtlH1@s0kOmPyekhtQ36#j~CU2Ydu#MiM+++|sm$JJUgAPcR3IJCUECF;2WZiShUHMgEQDHe3s+JxOTsCC7=!SfYc=Sx5t{ z(NJfzK9ZLkF+v8*FzE6-Uj^zF;fo=mf!?(WCOK6%PCK#9C~Rjhs)NYGb%*frNtS-U=a+MuoLSu*> zFoatsgq6%)uN1jHc>nzDmKRhBTmEn4=b5E31ntMGtH_CGP`g-|s z+tWmA{_VKxeQJI6J63uG3xU6@0eu{4q9(+r6=)7xkr&0j`0eW5zawN2l;XQSB588& z=+F%DCTB~NdHEs-kvhOjM#8gE?$wM^@#p-g;#!+<`>t-p!rA~Hib2urZ zHCY{{E4$|Lgvz}oXF#5Q8TsC3vt3(hH=l8DXsy~JwDbvNogMuIiETxZn|--~e-8(z zC6bW9Wzgl%cS!p3NZDyPwv!CjAYhKh3Mi0SROS>}3OOUfC~33KJeZO1rk0fa7t`Pd*e5l$z$;Ke;hp{!*bV;}ytkP5x zQrK!Nm2}YG>37U*SaC}@SU?wq9l#jRW^kd%IVc>hR)5|Xz5aU6g9_(#HT+sMxxGh+ zO-&M!A{>RPXk+FFfIA)mR{NX_jV#h zoTi$ql%YJ{O`!ggr?H3XHd9IM!Z7#sOz`!q)hC9yQO3{6{7;+YvL4%Dy(XT+hUtB& z*r)r3yrR)*mSb@4{Bpi>pt*25(!?b1&*IefvE{WvF)T)|UU3i}DSh8GYLzr*iFCwi znWk)7t;s|FKj17B@-$z19`?V#UqbE==TpN*bm7Ad=|%N77ygkT>xRlg>qXJp&E*r0 zt+YAZRLfxci^{{~5sd;)SUdV|;cq)WnF<*7D>j+88@=YE5%|iQEOsVZ0R!>~OoUNE zF}>`VB8W)hg_%Keh#uO;rktt|O2$4u?Gyh#f0ix5h=8@7udG!a2}%i7tz(U!m?!BA z1zwInoxnZk!&Ei545(h;?leoINi|EE%^^6o(tfX5c~^>_Gf3vICW8fOY-k3dupix; zX%0zT!Uj9uZ_!tQJ1-dWAAQEYkDQhoZ8qwU`ajQ`*o4w2_`Bj@0v|9hBqJ$jL<$HPN4>x&JU&{2R z+aa{f?+z}f=+h+y7Uvp?42{>Kc5}yKItMM)TJbgBGZ-yx!q&j|lDG^fORiX%ebIJ} z;a{7Ec$Mq%F#a<(k}HJAQU#6Y<@q^vSRa$~`KD@&T75aTORyVn1;kNY%<{b(kWU(2 z0Lht#;g_+9lDLRAZnV1PCJp%nO4OX5Gd@rSg$znP_ z)VE^qpK54TeX#>;-SnTI!aLK~hbmUh%{=seUR3H^!M3q$>?ZP+T#*-~QSHQ*d43++ zwEkh(;cIKTd%=TV4DoWAfS!a7AY zkvMc%U$7K&ZyJ5)1&u5}mgA{DKB3Fd?F$|7mnQVZxEpg10tr3A@<~unXfD6qJ}~8W?s%titud6>IEpQp_430xr+MfccElfu zxu@&()gSBu{d1Lo&OKZ1U1HpvaS&E6^7YPl*CRHLnKQH;5IOO0*s6}LAe$?&mMyhH(fV^Ze-&EsV{0^sLU-~!NU240Nh%J+ zmLWedKBY985<1qdLtsZ4Vty&Nc9n{ga?*8ry5zePKXY8GJ#YAPb%PZGO!x52mtCP- zOx<>yi&9lUqT2T*aM%KdvJ2*RN)^7Xp3vmX@U4bU{(mmY^m)j^EUMOx-GfzCpOonw zqUDF?msx2eoo^;<0<|q_i*Da>|DX2WGAgcS+X4*)2_aYr5VT3~U9)tR%7KuTIRSK_?nTL1P0rPSZs$cEnP;2}arGl;@G+fzU=IQBYB4406NV`X3rSeew+6693nbqPe7EUFUvUrg0G4){aS@EJ> zn#5`9JPg0iZLJwVZdxq_4EmNpoSK_1Z(Q4NT+Bz(EJ?{PzySX+$EX<|pk&mxlKvG( zZ%07@deKo>IaIzpXk^IQ9%7GG4I@E`i@yUskOhnC>DQFRDQI0BOR|rOzD3b3)+jRi z&VE_px!WoCra0RS|FS+qZMo)M29p70+|Z?)URFC*1`%2bp6S@@& ze=$mwEgc1Gv(jsK*~N8fMU%d8_*R+E9;!^5@slu*-uC-&Y$mSee-yNYYwb-bs9U^Qm*v zWSr-l5Ex$jCfX}*?Q$X< zaHjTt|FeEq^*v>MxY>#6%#3}q$Z6UA zE7*s}UJ33f(sP(o!*mn&n^0seGyCqtKybft_pr)3pL^?Z$LnABPL_1_JlboEXeVW- zIvR)iz**zHm6@WE*CQ}8X%lYoV={M zZcxr6v)@n6leLht8<#&>`%K=m^E}#~wWxt{-S98Kk2@5KH8ahR%pocgstW>w>OViJ zZjQp-v_OLblh1?_f)Etq_QqzAUqQiw^r0u}SyU$k`>}@o`ei0HN@HQF@9E5RiE$8i zeS!6_&Jeo?o+jP=O)xb2uF(_^2HBnPHE>WwntrS~WXvu3(gp78=V7alflTu|TR2EM zDn=r#l!OwCxUrpGnS(ba85Ev>L2XOh;-To#VyxJ?U1ly>Qo&hIjEMFkak zYwVLm^a!D!eT^Gn2`s>p<}7+BLkx9L)09a(L{VSZkb8?&$y7khPk2x*%A~0C(~7!S zSiaDX%q(mH8+sC;KkixHWXb(FL$;VlasK3ONLRN3U~wh$To99FxC+^-IfnSSaHjky zK(Zaf(h{>Vy)O`HyPD1F zYcg_}Xp%-?N^%AMy&myQA|*FYQ7Ot-c%s)h3f#rug?!XB2*_tDKd6-`h}qYCbp`?+~8=?s!# zRTnHFSdu9vDqt2s9>u{d`=(0k=dkfEtfuHlZzv(`o-AP0%;!b9^9(1fzbsNc)Vo`* zetXDC)67Z7e@tn*wZb1>8%9K!A#o)QR?h;5H^bDp2F zULp>GY-FwGMa<;F*rn-Y1@4yk{io1p^>6*=!sqKX6hX3|jc2P5^mqZ<@ay@mi6hND z!tQNJBA%x7r&YD{PTF3~(HO3im6DO$SbgCpPf{RPGUV_^8z-WeCj9Fvc(?3LMao7* zxE}fuo6`BsatM1P|2O@SZH=($31OGTPSC>9VQaWEj%n*`EW2wRL;BQzje*=^ZO~Qr z8~2MU^Grs;U5@^<$el15s`&n}ay8WI(W!fJv45u3(7S21B@Tz{*{DtuaV*{xBiZVt z(~IgRIeyDpbQ}r~*>7Bo-(@3fnKV^W05DO=^4le=sJiDRhSLbPDWl%1iI2Xu535U> z10Pt*Xoc=%A7_w;)h=~hywu~woSM)PwX`XErfQ92XR*lDYwT&6+h6a3=w*=B`4BAt z2oL1j3|ig3E5YFB+V;@RKWJdAGt(NBr0Tz3;*N?aEkkl-eQ7py#N@*DAl)4)eftG< z6RX;~MT97`w&EmYg6guO$c(#ZY@5r>eKPzJLA3eaPp!$*xk)z_P~Ghwzq_sjx2+?oi-R`$CuJQo<}ZkGYN$Y1mdN`(0n0fOtIFW(4xVC=^peP*2>7VYL+68K z4gXlRs>gJ(QI#R1B}Tm6d+zleuzPwrG*hL>H$qV*y{Bs4#3*fTzPl0y)6{5NdNuPE z7OdeZvbeucEDXm_NE_T;;q_%gM=}NEejxT!I9eRX?&b?&XQVOt%sM-`ZopEjC)tzw zK}+{-*Lr#@i>+JyFANTnk+vWAlfhSNq+jr$nmdS|qWPwu)|n7qmPfo#PPg)E&Z>oz z9E&9c%*a-qc1uqaLpO@_Dg7##DHh!mZHqUmV0j@XzSc@htpc|~+_IdlDU}z;Hr343 z<&U2X)$Xtqy#TNnQGm%+z>Gc&iOin9Bgp^2>)dTWBUoIC%6m-6hkWxh5C&|R3kBHn zUc&Db6=FCcSwrUBozd;-&O5UcpBjuh?4pO_NrvF$y&L1$H#0$($O=oiZUH zOqei(o$oSKGjevtG^gH_-$$|L$MwZbZc|zLMr4%*#^?APuV(sbC4cglT%0Yi85E?p zxK91@jlG`ByAtm?;gF6BR)zERqqrK&Q;8~)%yYiwFo4#bILTnyRG2%TcfzVI+%MSM zoN{6f9l^V{N8O|4s)jaa>7`$Njlnl^(11&udOW)wrQqJyMUcIb4X!p@~YEt zF#6pUxB~P>{Fy10Yt3Ot^@HuPq&I$834A22Zki&%ZsyMWpz~cfE7Om30Jy-sLwb*X z&!cexYX0&;I%hunj@;dnXs~MwdXw94dUo#pNL8fOD*$dR{7nev97T(A_sqMnqhASw zUcy(qM{OR}(W|}N5qfw(I_&R{`=|q&OCq)VS!sTtxR`&v74=QiaO~et2I6FUcXA*d|0m91PJc9^1li3AP^6R`Q zE)F2%u0g~K3quwY{)9&_Q-K_=3To})O1Jdfvrkpxud_7)aS9?-U$cVK$8n3uNxE*y z3u$eC)uR_9j@9f%1QzH|a)jS=XL>io1Wq7N4Yf9@i;r$Nic;Y(!>SjDH_r}fJc=eL zNfZ@8PjqWRlW(2zjcNJkZFuD5eAr#SMQKM{x$A_s zzbX6#l(vHru1t=#C|&FgR*+vb_tH*2GSE8Tn>7T4G6WObf()5L;pHMU`+UabAyzA$ z(ok9x4V~YH9{I}YPuTVvbt6CBs3mKF;^*1#^OSmM5_m;b?Fs3;iky_L3Uid^ z;w0aiZV`z>YdFvdF>x#i7dxfoRMgCh@djJWyag6%c#7|5)*)*v9xrv~2?9H6Yr z*f-TB1X8FoAGr|x;y9-@2u+>-vHI1~e5LVT71MB3r8pSSu~@WrUx8oH=%pN@hT4g4 zKGDF1lXROSEc-t)m?eDBRy4hzBW!x^w5*?R`>Wp>hBl>m4+h9O^oL*5E6j~p0Yx71 z>nvw0yZP%s5VBndd@CLnhpYsO&08nrSNOJb;n=MzLrlB64CALe^Z}8k9IT!Zy%DPs z&yg^Rcqh;qK)&2sX5Y(tcqw#gH!bo~@4_jZZx{2`u{E*+%F?~ z%zp2eeS4$%(e}#2*-~~BSRomCSF78zq5pDT;5eiF{D^K&a-IHZojDIP z>Dzs0jO>28yn`xjSxD9hpb{I5uvcn9ZkEF0Vh&G-khSn`TU3Nyw>gMC%)smQZF{)= zfeHG`!F@ejwzAVFB+fAIV!8IOgcK4%mt8Hx#kki7vCR8wA6q%zFaQhAr`8Q^86!A@ z`?S%FuDYOxjhnfYs=2;bw@lQ$FI@mOoha%FJ&=SJGx##_u=(0r@ewmj;c_rNSq-SU1y@0cSx(@7E!!Ki0F+pB&;M%Dy1e8Ogr&3xD$mZ^$SOk){1AbDENy6ub*9#?>m$%Q_T8C`D zjXkYi@hU-I)$BgZx7`jdqge&DtCX3ASKM3cw(pB00E?Tfcmurey1-P4kwEzZ;TkuhKE4qk{EL?av z>g>daW`NVcvOVbCG5G@%%f+8Bf`^w9jaGh4hw&W*ZNxZ#8M~AKovj<$;N5h2f_)bG zQ23Xt*0p})Q4EtDTKNL|D8z_mT{6zA4D7?qJ*^;*>LdE80t< z1+PQPI*;d1UTZ{aq+?PX7FQTj^aWlU>d4vEwgg?s{YiXgC(~1R-IBro4y=C1XM6bl)q_907D-@s zuk{)!c8D_V_hY|2&FM@F(#m)cM8cMBadTn+hNYz{8l!m=CL4KX9qz&uMtJ%WH6bPq zC%jkbyd4rB-fqpfTE*UE@LnI6Ldg1;51)L4IUjc*zq zGEUYFt4Eo*=?U?L;-z0X+mm9grRPH>e%_Oc4?LPpMQ$minMDC1j35}82iFVtr~TY( z%!|ACgbn3;v5n#9uh!48@}NqiBUW#*6~y3pOE=PJzsyo^9{72FmBpukW$Sb7t^^Ch zS}7+Dfjqlw&n$*(4FCtgj$!RxS}~knUq1h^TMeQj(2x6!$IxYKyF%9WIAH`UmAfSR z3GviQbu>XdCDMYGx^9maLQ!YrUtT-T+52$%twzSt$s=4Ry4Mu!{z1QmhH|hLUoa&r zby%rCjc3r2AYI{FBna{#R-0vTPb)M#`_jP)1P2H4w7{x#Z1zmy<8s-`{frwMf0vdj zo{|!ug(i1 zTYW2f8|C_pJFbonI)n!fKRG&qanV;#^(HqW6i7UNw&Df!Q7%~XnfQ&WBo*Zf<#due zTkIr$#*y(v~&7Om+Y4ly!V(Y)7TZD|VZB)y%5|pKm!@lg&8xM!|ZNC`g7 z75#i_6&OEB47q%o=YF|lj=BS_tX&I82k&t0oPTpN-3X*hT%z zfMH%3=RxKH3fEQ#<|vxHsm9gI(+6S;0MUWWnDr8elV|}6AU8;L@sz%AZQbNRM$KxR z&H!vFV-kRbfAS;cyZ-luaP%Y*G34ZE>s+CH1l4ZFaiCW7zHEyG$ zPXF&+b<*h|3S}wx6bXGLJJ(aJWT~7zD%sK3mM1GSk9p}caVth&?9VKZ z{n+MCC+R2JFyI5rFQ*x{hR+In-^Q&Tghd_CiKjh#i`~urYwakpYrqpnOA@6RX@kf!ahLcj&h$M}p+tf@HU|9wa*!`{|FTLsg?l zAD=UyC5FDVagsEuC;LzvevX_FE-JBk3Uz!Sb#uPiA3~FMbyvR1oqk>CggK`DRLg8; z`SB5bOhg`Ew4|+p&0ySE>-~*a>NK9;kZBs{1{7yY!)?2q%=?rEwa^A6GepVWu*V4l z_qKyG?0k=Mf&03NdVBK4>kK^CPEZW1>{fTT+I9jkalR8+n`_%n>kXK2ZV==ZyC7?t zSFLw#kFl#1ZNownSZe$y_5&NmW3$Nt*h(5>+sdOWWM|AoB^V(wI;&falJ5g@bcBAu zeu4x&z=bO;V7$zvxsUM(-(xG1O}DZBY$$9CfH}TpJhEyHVaS z1!`4jUWO|?+LJ=}%|vyM7x}g`-{f4U2ipVh@aetqbD?Sh^c6U4e*f5PP&lo-3fw0emEMi+)AaDC<*PTi#s75k zf9N>(G?x4GG4@UQM%NgX(C40q%*E8Q9vbM{rMlneT8y$Dlog$fC)pxp0Mq(eZu@dn$nLPERRx)qxI|(uiW zyJi0$GTfypKkgCAzd>7}ufb0$2GD(M<`XQ$H9xf#IrjE7;*a|Q0SsN?m%-|gb0?^( zbdPtcOAsm|>}{($%l)0|3v)ntMXxA|4y)%#rmUqukfU5_RXKW)yc-3Rf*L6KUBfKO z>8UeC#FPo^V?#EM$^(pW1o|4c6`U8sH0Ea~9K#nAHNMo)*ck zRZQ&j0^Hj5V&4?OTgry*by<#^qC_LL!RD66zJwBt{(W+scb(;xzN7XJ0fiRT-v`8h zD1AwyG$PeUY;H8zJ&qUUyVYZgr9A*NJw6|2WqNtXPHrTpdq0gWl&FS?Y^O|_U7T4j zzOivvIU-q~RoNy6)Grkr>o@)Gesp3Wo=4eC`mo{ zmAUjOR#I}#x8V-~D)%X>3nq${$RU%)O!AZNmxm8-Gme!99|jbgefFv$j;J%4aVL!q z3w^21uvVg0#@nIew!Oi*iV$Sz-@{Xi?6p>sFDax8I*n40V*A5H7IGWV%Sk2k=IwtD zP{K6pHxuu2y$HJDM|gY9a&tYX2mlwVLgn%sMPvMqGJ=6aHl92EJ}L|=6Lv<(f|L5v zKwMUp4f5Qq8~Y{p{iByQ!%^LK+AOxYMQQiYr6)Ko0qR^{s5(u~QM3^SJEi=ZaPu%d zSA~k}H+zX%=a~q?)8K}a5pxrBxTXUVRbv;xdrJe!?%7@CvuMBth0>RQV4)!c2(DRHSs;`Lm@U>dycFJRa z->fi=isG^L!ZE;OvdXJ5qqx$<;LJ@kn9qDB?PG(&BBY?wkCqC1pjJ^t#yh_&rOt5qkP}?Rp2>}^=-n;)w0vA1Sq`Qc{$iL06-}eVh-<6j-sS!T3Z>R^L-hPI1pM> z%ML0h8ouUX#Z&<`3|tDv_fixaxo;AmPk(=iD;{AyOeL*nfK5Pnry-_3`Q8wxPHY3H zsjjM_9n&K%?F>wVUq6UH2vVp39F>sskk_ERwKlF5V-#Cts#W`|+|F8{k+0$&`g3~ELRXSelU=vQRF82E0 zs6n`JFQCGuBKK3~haoLi2L|Z-03`HrF}SCLTYP4;Z-mj%QHx6ZP>*@AM!UlEh>a6{ z)s@i59Y`6_zQq946#Bg-%kJL%+bhBcP-`Hi9+7_yte+mBEB)DYyamJ~N=58*|0&1& zuMZ7Z+-d4m?C`=zl||EdmAn3^X&B)H@(YjNAOFcv{OjaDmm&q!BT`Wwx4Z^8li_a< z{O`v4_MJ4yC8x>Pe>)d=OPLI`hq4K{0YJOs|8C&_o&aDI|L>TPkKyg{28a6?^PMOC z_m`;pA1xmBvg4_Cz)-E1`baV-BGF=O?;p0q@maJk=~OiQ>BOn)JWr zy8iRq?`9m;JDLVlVQ}JqtH%AiIL05XjGkoRivP6xe;ccRd8=@z<&&FFZya?up8nSt zwKYIt|L3IlKep@7K`wM4$VkpiN|5~z#g0FX`bSHBnz7#gYtaAa*!{;5z_$TC_|^O) z{Xg{H{+9+K?ik_!*Kz;q$Nvv?9GTybWYyM469$Fof^>!W@V0~{j+$Xq-5h1kDZ)P1Dl=M_IW^5&G{e0`Oi2Qzthiflo<5?;{{zGfUnlM_&3Ygd&bY&3SW%f zkieNXsH!s`Z7Gd0tNDET@IQ?fpQO8iPXC^i>Fyd_>^?HV6NYlp!A`{1&L$ma8*c)RNfLZjthP@@gjxR^!DN|#PeKG(TqyA2xEwGj5){`(uH zsf!+79DPwPf2ExXe7n>ZbJ z&xYgQOF&|!3tO`<7PHOxQwR9pW$#4Y2!YnNSa@&U{YUgZLH%)9uU+oLZi^*cP*aL! zlNC}Y>vn!oKawF0Io>%_WkS-hH9cX^yJa`#kGGakmF8@=+j=uv2{-n8!Paqh=Jj4U z{W8>H3#Lic_Pp+n+vnW zfvA)2xs9)Ny9WWQNge9u;hF1c###-C)p)lqa_aUg{p)vOx3kP_49Ol|%KSDVbbWmj zR3~4qdLHuxp-f?|n)XNd zIZQ`k@_GRkvae=mQ24M;3VtH0JiLNS!Im7hso_up13ig=(kIYTx1Ago*!?ntN+x(6 z^=DLehck~=DiP93V{t9X-rd`jzp0$@MVxeAA806HzE#S@%R8mDyFALEIB2BJ<6W-e z?Wk^}u7>cj?sp4^s~e@ZScOwh#>>=`Pg_QA{jou%v_g%JW`zF`Cw8b6^&;1`wfhCd ztUCnwcYhskz=o&f-HjFEtq1i`s7jr(}3zbLQdJc7`KMb=G>^P_Z-<8%@kSU)#y*rFdy-}ty>oe zS#NeqGA(2QK@BIF&UOy+jYdOIOqp;pTdDw*xRzvm9CHq`QObPd2Az0ZWAK7M&)c1+ zh8Bf?Ul@5BkAPx|M1*SSzn^@nb@9*#xFI^nDMkaiSY~$5P!+w^mEK_eo_%Sd{mSYQ zJj_1*E*4eg(``Cx{ZAEXi1g?wnSm9D`RJuILee3oRR9yKYUI9Fy?cb*##` zbKwcOdH8u=%300iK6}gf?KyJa1$uMasV?7}q0Ju4T^M!CNSz+as35BM_@fkF4q7j1 zmKmQlWsQ#6cflm@x%d+-qU-QPn)B|OHmZiN#dUMOx3zCYsWN_J{cYCd!nq3?atjlb z@20_g9G~c}7N)7&qrrhl4Q6U-@hiA+apB;Olh6sj>t%2O8+7B=Ky524S=`F2Cu^T~ zpdM_qFR%{|_X0*st_+InT1hY}VttCbuBHu%&feQ{)(U64xi*pSGngH!;Z#IvpzE~% z+k|V13j{a3(n7D|?`-`oCu%b2qQ3d4izIOcDk0iBTIjJ>xmcROudqXKI_g!wakC#q z6gWG0J_L3rn)wjWpq0wwK6=5zJ%Z@MT$It6f8)>_;P!gq1pz?Eb#;fYSbvg1x*E0a z+jFnAwIMqY7vt91@>4^I=^XOcX5&|%x{7_@xFeY_evgrI$n<|j3{|{~PKa$rUxj-f zm;6vX;kdt{>ULT+xVp+J6F89?4EJJ<7P(N5({s=xt$%wqjmX$%@Hh``<~-5fYWtu; zTDj>yS|e}dgxFDLOSp1bN@ZW0O|1IV(CH>u=DJZ}*6`zyb?vCOcAjp%I(`~$urmxm zay{4cochFnlNqmTHl>-@mQrs@t1d9lODu{S7B= zHm0^kBQNMjaksLJpTw+a4P=xOt-AOSj;AF-VvxLGaVi;IKfr0_PzIjBGyM$aKG!!(FPOnpg^+VCyAS~7BB z-4ofl5kIR)rW{k#+;;+7En9oCM-J#I(8=LpWq2J=In@v1Vf0v$<9r*vEn44O=J()i zd}{A|5UyA07)#WYd48=ZOOq7!Jluo#0s>F9?i_0tV9o87jF$XFhpZuh?Q6rBjf+pVNRMKLEml{=-Gof{!Dg8(2Qw z@O^P``Gz~@*))r^-*pd=xI{MXK}d8NzhRG^8(*?$nhfY*IO~+Bv1G5Cgbm|i)@@_@ zv+NJI#Kr0ti)iOt-Q;2QOf7CaOU%`R9XV+H+~_<^I-Im>4xLJ4&w}j4^6%yApj=q1 ziF$TZpj;ec;>6QHU4+fCg>jgvgSyUnv`jeLQd@u$QikFAc^0?LQiT!VaJ!tts=4qP z6IXu3@rETa9k7^ED>Ye2p>IMS24{u~Nn$}SDt?W&tnS{##f#m1{&M3&v;nZ@m;00z zn5h>JvrSe5OoW{}XH-Qp_iB2Vl}AR=R^JztBQMycLFJjEuNbDamss{f<{ORAzv2_7 z#sAwQssy4lupsm0UFhqBEBzzs;REcmX`-wbqX)Hz11d8sUsB<<@tH@4fbMy6*$De( z>e{FFqV!%pFZWSb%H}uL7aJo*#OSjMkZ8BPRegAU^|u?@=2~9V0HdIWI09yN+9KRN zR`#OWlNG4fb5QG}^WMjDyP_<9$IvPrp zojek|L?nfusB6jOdh+N}xgDw|=DSvxlA-W(+2&caJkrhxx8yWe-kL2+PExv%a-!n? zLH9~h#PoW=efa(BnvA-6(^sU0u;sFrvzj@Zt0s=M`5DK>788wz!&QQjpr^^*&3&<5 zPs>OiVO^&ZYz>leEW-Z*ezz9!OMsFgT5QNg-pV41{<1l_O5$1 zgriB)Q(38{8uesE4*km33WBRNJ~kI@Xw1DNs6z)OdY+8Y%{L(HCVB#H%(g2=xO&Q( zkoWlyn@O_3$fM($km{B(wACq#X~K2?&c3KnN5^idBDUs#c- zs#jjA=IF6<3_;qzr7hmQ0fM@(Bd@6cj9?KT)mwhqQZkYtT;`v zGoo{Sk_hcL40qowr8rGqOOeI3*WOnVI~HRA@3Dy)Ua=ABb@C^u@h?|5B-OX67+fyA zYCi^6Q8Xkv{IizW%UDN`LWc*{A~pxBM0-4sWG!Uc&>M;AuF{e~Giy+2&vd;Xh#UB=t+>C`y3K0He+kYlG#iCaQg!_h#E5tU)S5EK~6 zM{^$hit0{TClZRAh!-T5`PPn`S^M|zk3Ov*N3~l4X;@_SUKc-f=uW#XelrzI&YQv~ z9`_c~Ud~hXxe@kAB>Mu<%StBJboJ2+F z){#HILOKM$ocXml>HiSzjcn5Rr12SDcW8v#M@w_qE(YZSGaE4n%v4p1I>KsJNs+)u7w&Xc9?7I&ak{*B(gBnlCL^TTk*ZbI{cM$ zmZr1eP@A=5)v!`q64Q+vi5GtRo-tv>tep|Rw07%lzFM|)-A5iXRD|Q{=O2j%FN}&r zl~YKdYF`M7pa(cfFUb6O{xo2Cuf=NnW2ohMk( z_bDJf`)_od=Ox#6D>}*2ri<(lv+JAlKt1jKV?CnvfLY zy0)|4>O@3`5ZqZLUaz+9dL{qk5x3r3^7u1a6m~>Qm(`h6uE4F^LSv;(S#A5fbWC@& z0*7ct?OXfX^1@4aFBFSsD2;rZC07bGdGHxdhAx;q=5!R1kt?f!kXN2&@Zra z!hWL77Dh0L1i?r$N?x%uciDK$Zjl4i4?@yT0ed(2R-opiw=*9H8$j|beq1OQO1N$; zS`fSTpUiz-MT0z{h8;%$g#NGTyYII^Zm;a0TLU-}Kg@B05(`i0Q_aK&Ub3I>p$F_= zW8VK%Z?4ka;4E?}b$d}ZgKGDH2sEld)07;#|G|7U23y}wtb$yo5VqAUh zZB}mu^v^yHs-OF{9AlNampaw9MKJ-&jVRcT(R9RV8tP-A#3ahBAFOd1*ikO7(mk1z zcW>~ch!fIXf6qD2xsJ9qqph0xT9vi2y z7F0w%9wE<)YYcYspcVKOg@jf8^_yX*cLn|OqE>^?jJC=QJRzVKRo;@ltmMhhCOZVR z>vi18kk2A~dT#yMJ;zYxt72-i(=lMW-fxUal7IC%wewvDH!Ow#(B1;Xrt!~@`)9N|hQADM*Q!@BN*B65*$Hd1%KVm0 z6H_}CJtXL?{A_=5GTPP9zuwlMxSVz06ISErDZ~OxA-FC6e(^&eCtAO-TxbxOxHM)j zsOk=B{VnkH>#el7_Vu_?5e51K5hLcAD7fvTLF{@WmBA1VRLk#9kA$=h2{) zCJH}~`3}Tirv$D(-_IM#85$NR9`>|!oisNxZTM|W0X485!R`905ChRw0n?M&AvLic zH83%KtHU&9SKG>HtK7|i#d93iqt5sweE=1~0@}jJOjo?Xh^L8<2Z-o8()qGq+x8aU zqax_u*$WquS8uvUHlzU^RCm`wn$aAZq?%r?SUnX#_j0d>^a3I}X`==^<{Qo6KK49& zp<`d5F97Z0Sk_tBYI&tl+IS@7`eu+bzyo@9F_YWK z+m71_dVV8B^>6uj#9eNo6--M0Cw!pvsSXubc^Dn!?G8;1?9;KgYMOZ?SjnQyu3V4HFwkE1&@SObIs_?O_DLu>Y!c#d&M%lh4MI^dU3~C zJUmVslhxr;FGtGk!u>6c?37KD2V8T=ZG`0h?w~ant^0F9+Um%CKevf!;`sPM<$joW z&qy>+?`u{R1NrwT4I*=3HM!z*d8;?%vtTF|Bjp$HE%sG&`-7eizB1K{Uwujf+l{t) zJ*i};wokIzE^S)yPfPXitLP?IYlAUXYumi6{2mtMtcGMTurngWO?WgOA*}WhUn-?-OLHlLw@iSG=a~^3e#=Ky6EM4O= z%i;xX^Fm$IdcKUi(5YV&(p*YAGW` z+Y;~*=C#@CMlM8|m4cdN_zC4E89sekcB> zlaXLG!FA4x!r4~jtKFlAG}L%2&ASvg(GHjFEEiCb$PI%~>N-QA<31J5dg?KL%Xwy% z)s_te`_0ESH5iy%`V{{4TD@_>x!~ARn_GGc`*Fl9ZLNscr$FZs$m$VJ3bq12**M8< zn}&035u1nL&}4 zqt9#WeX@nD%T7Eo=Te=S?jye(v7PC>2)|u^-E+Eh3_)D+#%pta{E6(=TF91b_&Bt) ziDY@#(rWTa1$s@d^KD@Y2@@GfSC~vZop<8TjJAgH4~^S{d9NtuJ+4U}3Qu4YsK0#9=5!dr&nBBE;Ysn2@fcxs9BOpHPZCyIKYLH(t?`MJT0INnnIs zFU$)^{I>1iI?&pN;5m%n8K_@*eLnR=)6y3du0nHGd2m+`)6tF?<`h8n$3SVKR`PbB56jV%Hq#aBpdvWPs7 zHf<`<;|BTc?_SI|H^2E&A5eSW`^PU@Hc`>EdLr%Tz>&NXZ-vR3%L+}!;iN@b9!rPF z<`WkWoMakpswtlhyzuwWLn_>=Mvm8lh8h2)do(;i#3YBcwB#bF5oX zQNs|jDYzPSb61ki!W~mCsGW0@#&KKtrEsqzgrEkMW?CDmy8)2T&JSIQqW@cMf9EO7fuA~)MESZ64WU?#F0&$bt z&X~Q;(zm2lFpsUxTX@V)4ay^-TDbaH^=MN?J86n}XK%#|v6>KzIN^;H6lSbgj$iug z)0AWo71eX7t^Ni)UE0_mQ}Rx|suCo@p$=?c*LLlpQ6eX80vg|+z~6^kxJFd(BB&u& zim}h|)`iTu)CF&`_$?+c65l)>9-iIBBjoKl$D8_G*1y@aZ9$eTLpX1B5_g|NX~*Mg z2W2*J1f`*+&I794a5KZNS8hT0VRA%P#=2fWl8p~Zkyzy(KA>H%ifbb2t3S&iBWiUe z!eQPpkVMJ^GAx5!?A91Zkr^I0#s-?6>}*Ab9wTR~KS{7ta(me(9{rkReY41rC9DX2 zrEHM^RE2a2)H>}eO|>f9``E3fkt-vAvQUG}*Mdh%njecW_oTEuQ3JxfS@aMmDD;S4 zrW&@ZKpq>dPPUL2BcMcF)Jsd5*ygJH39T~V(rY4^G%LP*i20j0c;HB@fnR=qM{uzz z-(Yd8Q7b7gM__j%#0*fG|F$OYdCd25Vzm~Ve-MLx}s#^(A>nqGh877A?$H&NOvCp-*{qidI z1-wxivQDFDKLDZJmJ*G#0dP^#zh9*#?@%uENjkoZminwA8Bn>=Ux}m1frxgvfH%u9 zCB8O4bqhU2`=>*?b4{pN_z{@Ph9-AD>G<(Mkf1@jW?P!$T9^eh?Od@!wX1HOnVyVB z!snh58ewEk)_Z1){B<^HbBo;Vky`dA-z&Z2tbk6dw^Ymk>tM=CkP(n!+_S~mLTIt% z&(<7$CtN_s=^-(rJQ$@~rLXDt z?a2*prCS8EE;}3K&*mn*Ji|B`_JvRRkk+hpg=7(4jsqLX4i0V%EyK$T6$JJ)i^;^| z;d>qp_8EO=wP)2YQ7!~hpVUd5X4g-?J!Z5j+f=B@4r|9X$=J zem3hUQ|)xn#dD@nj%jHQus7YeMF+K?$ zh}{#YDX}bu&CwWu){zh$7yYOBzg67i-I(=?RrG(LaM+2oSk?&(1~E>VJbeN?(>v*Gf)u% zGJN!Pg9JJsMrnby(`-q5!Av;0!Km@EyJ}I|^+sW#fcv0nL5E+RB;|}})464~Up@t{ za4dz*Ma?eLK92q6Ie>D;=0{fLb8O--*jah=2w$pWy-47 zb1jj(2#WQ=ebPRp?qq1VZouTUKjU+0p(Z|BLx15}GN@Da(UvK;8!hOWL&IBz_a)mo zJ5`?+GYnOvCB&F(>?%pH`8BMCevem_sTI6Vi*`yY_5{kbh?sg=Mh#bS*R(WiVHLbT z{gsLle7fQwRAzXYD!b?;q)aVr(OQ#7^{NZtZDX<5Y`doX&P~J9yMf;B46$u8D^}4xN7i zzjOY7`I5WnVCf-#M9_|`1#tL>8AC+FS^hVqfJE*9jE$81tXrsUyQWVj*?4X3HZC@M zm5Ikrz^Yp__&>hIB>Nu91?feF{V4D~qDyF#pD1GB+_Z}Y84Wj8WbJF!tL>5kADeyR z6l7R64l17n>|nV)`_IPw1O}@g`6%kYH}aot7JUq~mSHJKDxTn9f&1V8^2G*bOx|qt zzs970J;R6g2{pA+J6FXtNzE`$}$46ls||}SL!b_&0nW& z{0Bg^MKu>e{Fn8`UmJmf7WLNK8;m8w)p+N^{}rYE{Ri&va}S@-ID7Wt z95VIpU?e2u*On3zDzXw16e>>kW|lUlNJ#XtwlNj*?JBrGTdm~8qy*l-e%Sk)N;+0& zZiG>4LuHt-MmJ?#U1s^2>_ayqR1_jd*Fjmmh?n|lf2{Q`X9DzQ3C&6NMfp^V;^3vK@)Bg6ND(svL!?+#vc<&`7L%6U|I7}msnLd3H zyjdW!Jn1M$4qtc=zWn{HF9;f23);ezZ;}t)itz2&T;h4T#3;Xkop8CY}NJZJRrv+NMo0D z$0FvC$DVy1V93KwU&)#&Dk3q0G#U~VnFI+1q>#Z!1o_E7X=!A7q!A zxjy z&c(=`#m@QJ-27MHEoo^B#tg_1;pgNL`s@Dx*OmWB{I8O){!@~h zpX0wv{@0cNXGslbQzr>~TOiU!b|Ncg#EhJO4q4G#bFeF(?F?Dz3ophAhS;^Z_ z)H93(weQ`OEPEQ&U84ZR6!S#bpL371niG3I!aQ%?} z&ZeO7^IMYs_b(&}0Uivdf%ahkpBs>nJB0sT9tp}w?dMmB`*@!6pDakY7ozIFt5cUj zLe`*U|0?lM7WC2Cza)XI{1_xiq4;a}(0{Vf==c8BHjt%4i40W_{&s-j9q zxTX znhavCA-6LNA%sT);crl+oq+IncyLt4%vHJ4LwO|R#Dz*4bFdvl&?w*>>aVB?AW2A& z@EOFWfnk6=H5D{!WC#JGUr3l`hz+nM1vVC2We}$SCCD8MW%?|R*iY|4o2c~MLtyQo zOC$zRYD5)CRR>^O#BhTlDo%X737Y97>qnNaN63`i{Twtm8uMeZ7!qU(bfa5IJ5~)1 z)DnL6|6Sn}vdG+~-H(pu+!m4zeQ(BkMXq+trY-YIv}#j$W?ztdkLr2t)~wY`7;tLo zyC!Nod`H22_3lpxQSO`0XtT9sb5_3o4?AGkO!JCm7@!m}Zy+axDxR@-R||JMI6?=0 z7c23~7&2Q@EjyJxDX!z%oi^p)?|+YI{&r84k=Eg`i!RmodYSyYtJ}Osp0@j9a65a* z17hRSrab}9pGnLXu zWEgCrlDWe-C(1IDjY5dK8}BQZpg+P_5xqHnNufDMkZRx3z`vUCa#p#Pm@4MIpN?}M zcV7=%E2;62#Ko-hK(fbLGM$j7&HBZ9ChvA>lil}CEZ*6{S%-Ekl+JTxpf)Lbpj(u1 z1K%50=kvwTR2}Nxu^6PN8c`C|z;&`LBlkXzZCH*L3~<|_3?(&A(zRop*J_WBtC9)oXQf}fi9!rK{Jv)hYxUhCqDkYuwI%Wj-{ ztjsM-Qf@2t-`jEWtmtj_%Xms!vT!4a+~&<`%FzYZt_L9-OmN7Z1KpJcGe$10rUSsT9a-(4T|r8)O&bM&>|-8MD+4!~^t zxoZ^q)C>%4YutnbM+S@mnZQ@dHkt}<)n1XNLc$0iEpi-|r6W*WXKmWS>otDw34=+6 zjo$J?3QrZ#A3}1_fRJ>DHuSZhPwXFL>Dp9u1l(-qrk1K7qpX`4|C#sNPk!n)Tk)Gu zw42V)k1tnizjkxBfMZv`-+CwV_|5p` zlkKurpPk6Kvew(_>vhpP^yOIpMlv4_GmPz$x_Jz;0|G6*q2sMFrsMVfQK|=`Lc4c| zJ#7BzA48tgDSX0Jm+oP%iqy7mK8a+ryWSpMdUTJ!Mqt&DMK-F2DayK}CTR-H%(Xnm z={{&u%pb5}Nit?!Mieni@iD^|uJ&6+kFyyD9otI|54xGkpuYV_Rz)R*2@&J6jw1z4 zZu?{kgoBq{DRw0v{T}FXpJ>J7c3TP;arl6p6-7&yPc?eU)9iJI{eZ_j#gdjtSQqU} z9IOO73MR}}`UBb1l|O?LI9NLwR_R8^u}aVD&JBe1LmG?@ct{()-D;%+-V9 zYngY~dD>P5IoI+5H%jROC=tt>oqovo0z!v*Zw)+F6Z_(oMW(g{++L4qXkkfaU+LR; z?KQ05=M=m=-F@OYH2s`|C(5~~ti@ABw%!>pCxW^1eJpzu(2)V{p5>_6q!!e(S*H1w zg4@May@a|pivboj-kUie1Cp8X(p2-j6fDPM*s&p&G%A?LIUyend4*0V-_YzT6q$a$ z)Q!?>coWS}YZ0_Xe4Ne(r_Q)M+czBCIJ{}}9#hpJVkrH-f=M41uXf@eLS_gaJg`V9 zI(FNdG0n9o-gz_<|H8oPNz>tNh8>3 z;naBD`Y$9nUutTbCbIA3pQ9^3nu>LAx;dSZSY@-XL2H+ybF2oxnldYNPkt8U5bZuZ zLP_xS7_I!H&w`RL9;Pb)i->uPAr*A+nm%w|x)p`)^YQOMocliSkTo1sW67;ftbWRk z%ElCMlMooF`dZmCDv?5mW#EJmIpgnRql9k`tc)FPP0*k$) z#FU)TaSS6Lj2qu$DOJ6cU{+1<%j4&nzPBC}*VTm$_*z8zE~?*Do6SpaU*24n`Q9+e z6q0=LkbmbEk8APXuko;(DUc{*TPKm?e6aLomJwld&`mPcRz;le2_6By zN@H@P>o%c@GdLyS*wGgp*|*;CY{+_^&ACZH3)M&8tm-}g^9-5FpX&3HVcGWRVdFs? zrarC|iaL!EJv$c7)}o^l@73+qK6&xSoK@(L@BCQ7g}hyv9UGYR>&nY(c-}fatF3o8 z_Iyo0)6eR*^quv$15t6Xl?$YbG*g{1lyN=>mFZ;mhlN=uHon=K*K(_I=ACnS$sVQC zx(rRwG2S|zIIymk*AYrE*P!jU?88Sl)?n?Ig~rS?hu6Sc#8qmq_-?G7NK810%ApEGrWjGF3ocO*!DzF|rsWHM{xtA68^*LEZqxLd%)S7z-pd!w!U zKHwM%c=rv@VD~7OWnnS zdDq*S=}V)j`7Z|oO`7TvyJ68$J=57Cw~tzExjHXjP8Pl(NRkhI+Rb>dz1|m|5#a1I zU2X4`5a960;5co!1baZ+IZksu-FK|Dziikf__b>59*J*}=g4XY&kxMfn)c*He!`B{w6=qQVa(PK}A9r*M58^q#wa42)MXd(P?M{qgu_(4UJQiy8hpOr=_hQe;sbtN?A+r`tD)>Dc+(y}D z$o=|2<3n}gi<#|5xxSO-Mku$|EbyI`tbl{%VKc9_ERjE((j)o@N7hkHxLCBT&<6DHRzDiv=Vap| z%`df#(zbs4Ddcx`ews4}>*&^o(-5T)G51xw#U6>RY0t75t`%AjM&551mNpgMI`7|Y z8(i@}_c`3?xV~(Jh`9C=+H4Gvx5YnfXpAXxy;Ts+=_#>!f~8FUW3-oeeW%^J`aB~o z#Bfdl|7JWU#s0-_oW%>F)d@EelJl8`TS79YS_R|95Bohonfj&WIC1A*2ot>DxyPmF z7YIv18#P8#c8&ESJ+I_eg zOR6pDvH?{@@eiBK9`*G1o>~Pv{@7&?B6^V{(g}{D2|31`ttQvBZ}`{c(DTWC&cBm_ z2Rup-S^akPSm4skcXcMX`g2+KUtOoahz~WW;NiiJ2j5XqIXeQrF`d&bi%ofVaRYth z|M4byxyUK5IZ0DI1H3V{r~R^%GYmM>Z-B`4s97*ztf!SEGOAC>sc)IU3$>Z}(7uGh zSfB}QDiY#KY=&Xk8|J#*eN`EP6Q4$ld3+m^olp{5zXU{9)krEZtovH7KGT>EzIaXR3F$?cjNa7jB)n zArYaI*WRE8k&|}wg!j0PVbI?Nv%N7taq0T2M-S6w>F*sorR82*$s#pKqtSOTNxblO z?&4<&xCAFQ6r6fN+CS6}R|PYr3dr*54K;ABa;*Geq6+9;PLV<A2cA(V%eYw@iEK4^$`l#KyZHYSENQ!z>~1$IX~l-ysw1o>+mmlpESb>W@Kw zYC3j>d9TZop{toIj_&fyGANyjh9NC>(^6dpqww7vUP=}*s@ybEalgS&!nq$Mh0-| zW?kXOJ{CQen&rzX`;=deEib?+H&7~_MjV{$FC85FHrVq@U%&{mUtKL1;Hs!i>%1T0 zuN%y|DzSZzOh7=kMKFvxOMr40-e7b+ZyKztl%!=P=rQ9=G_#m83S7)?i2MkEYU8Y}r5_oe~+o zd&ulTc?F&7>#CJg0tZxw6;xFJ;)nT(erheIki{SV+|t-yaot8)QfxJZnYz-{cKktL zG4`@kzguL@kxvUqIJ@@vt?58mNbDTVj79WB4D>e-3fC&xas{h-DdFIQEU7SV4| zbw6THy?jjFvu~&TSDPpkTwHXB*yS*9voPf$y}4-qlyT@po3-nF3mh})alX{!aQTbF z5z7fm%L;4n4fWq#+X_{8#h+{{nyvR7>NOf?PsZO12^sW7lP3|-vV`j@^{|}FW7U^) zzxmnmbZ&aje7)rLlfJT~$GGwxMos0q>a{wWZ8!tdtOP+uL@F$NWy$W0oa+gif)9qW zge;`7{yGlZ&q!PK8Q(?sSsaG`pK>YDaAqgOipKSbB}i|o{&7My2Qp{@?FD8ij}3ct0EFA!nY zkp{HM857?Vy;Zv^6ilSOk-E94Iw)TBEjm)YyDs$^H(>OH?_hGAKX~8xECNfPVB0_L zVbxta=ZVO#+-?!-?^$UiO3qcQk3Q&uy(;=H;kHaU>0H=sh%A3t>&}sfBqX{p^;H!f zL56ep2ea3|cwn1vm4w+yx`)Lb`o}6!k8aE7qvCy1tmOh2>Nm&l~L(r^U^s8js4>_wYt1FStv3$nh8xRoPRQ zhP)$L<}U@F2GZLkJnC$^$5lph5uU%fv$EVWw|pmF<~uv3Y^*%EJ-J}yRXgm|zMY7J zSwRyxav16^WsK99XyGebFKx$G^-8Tj_zkkV(1ONnS&s%HccJmhv6d81RfoI2gkh!- z$}Op~^+d;UbLBBKjzYy6Ifu)-!wUUjvrYQTWf`6bB`ymo>#*F7s-ji-@UhNBGy5o=!9BD%3e{|Z?=1XOO&P#WMxl^Ri_&GlXxD&*E`AsBfFnjrI7_;kMBeK41*Dt> zIR`2`S|Cdzo)d1$@Wp#4o2h9wTisPx+W&bc-EHHAM z8yxjN{D>x#@oj7pq23I7p?++T2fa}90*>uI7XbkIcu6;e~$DG)5T0hPj^oCzOQ}W~YwV@o8 zft=u~bbs&+J-Mz=_OM-5WwYj%yUCHrnoy^LyObErTE)oHCpv9;#A0n8k0_IGE1tyd zz~!c>Hnp{yvpv6pQ8eI3&ox$>D0*RD(Lu`L1sUt3d;Zye>H<^zXD|tY=QHYAJ2pe7No^ z?TBd1rSsZW9d^O;q6t!#3=V^gJZM_>2_eKM&tKNOuK5u-xkwT77IK28(p?u6N)k*W z;Y~-la}E+#$ffO$Ul!O2~Wl9bq|tml^gfVAr|@NHhjzzq%e7IB}?@W zb!oJ67qr;pZ$~W9Q|g@=nfk;hLRW9DQ8IQ#`-GC*)js$>&udhdUzX%AdM5siqJA3p zSGr%}BYB2DFYR8{pk>e24{s?;K!GLEgb!bNa}-ldy!I0l4Ln@G-4Sgi5hje1Fre@! zUrR1Z;=~~TGsoGi^T%e`7jn&-E4e@teyiVdLNztt%qrYRd$R7w{|Gx&jdtcy!Berz z%_@DltlLwP-yavHsY_LvRzJP{!dNv*c1qZCNa1AbV=AjYlWIPt&P~B$(R_zfnZ5mV z>Y^t!fqOEE(XD^q->GwYny1Q7jiy+6!$KRi=2O~8eUnX&96wZ@K_JbV2O^7}Z?;U* zXAd6PC!O|a)qqC?`&n(1F?bpj9d`3D_ZO@#yJ>e`h`~Bk7lFcs!DwC{qUD?L@k%hG z(L>e8&Ke$wFKZHxhsJW>&L>I0YNXyD^+^JAxq{_0c--Rbg9@%@zqr_p@k}SBpI&=! z()M2I&gcXUX_8tBm01q7P-bi5PjKn=r7(3`<{0s&N>|yNtb_-`{_mm`+2ARkW44oD zXk5nADq=if<6cHVMMhJ90r1JwI=M1&n1@O_d}+Br zw63hT>p1`$2tfLX5JNTqh7MAA4tyvGB)|aDUu5ucpjMzFjVIAT5RePk2efpzt+NAH zCJj(3#}!F5L@qZFYfVe)4@&{{59{M;mFBo8_a)8}M%j8g*>H!kHM3lr} zrye5qQm7g=4no_GDdZ>uob5k~*r~M^GLNCgptx~`11^F+|FaFNKLYL43~eAof-T?&OA0#*vc{~6SS3~HN=`2bk-1nd*;T>&a2sP3L+CtNlI!Q5T03@B_ zt#~4Gxe;Cf$dUnmumB(vy^R(~LW2DB3>lUMRrNbjYJfqBe+ZlawEg4fdVx|mZ9(e- z>M8~m)FqTaL^LySriZ-p4V8A#ft z`E`QGwE`2Uaqm48`LAacUXY?XA~zg}B6tSQ4PW9xRq3nZgl-Nq&u>nBN8- zaQ41NAmou8`Hz5UR>piw=!);%WjcL6H?A{qEPS-o7LR^?xqG(IXI<88bZ5Xz{_gzq z1)z+50663R5Sp>6N{)Ui_z7h#bAh>G6who(q`9wpL?Iql_#X0K6ZYsC0&;t9=13st z1TnW>AFhSbWw4JZ^6FQ(Br}zL9lqYR_r)CWJsx^tHt(^H-UBWhHLtHbxP=7P!t#2tc;{?FnYBW<4<(_3Lr?OvqdDl(UDQ-t5N-qj(i^&yAroZLHym|a zw)f3^G%z9D7yZUudDWr`W3gytZc5wqI?Y+TSpLA3DS8dpJ}OLAm>{@JdXP6AkKcMT zuIJRXy4F{>P|EnegGhIDr{RU@NWq)V!qRmrG?PRyX+<wn%-U2o&U5?)_v~-`?{sZ!lSIt&lUuNx1qX!h0{Hg>#086pb2H~{ zz7B<$MU823rweqP(JQ8%mw31wFR$%+?QsL+khPfS2fiKLY%&H?Xo^2L{*D3;W7cxM zgvq&EJpx+ZREL=tvf3H_)Y>FfptZ&*ZC+&EaSwo%vWx4Zfxfc^Uwie50si$gvy?}Z z{63aX?BYv20*9%|_|Vkl;h!OLVe`Ndp%2jHX~vrg1Fw=P(~tUX%N@{!Xi3qrtXg@FuVZ?=l7<)2ZU2>Q_nK(jrqB50sfSGi9? zPEz>E!hGOMpR2L2uA3LNbX%kB^e3(kqzomMqRiG2&dDWCPS@!0pDY|hQ> z)a_^F267X!b$Vazy7?n{4RAOq;bVtjiOjp~&2~LC#`K@X%MbPQ1gy9hG{q(Z%x#SB ziy2^Zbp1sc3O$(zdy_2fiI98wMeE&F>)--g>owlE>JyuCe||91z2!#jy`W2vFZnm% zfxX{R$w_a=``d9uZEpX3@cm?->0SH#Jd(5DqISW@t7<*NE5)vsSD=3mxMZwNzWIQq z#4uM#7txUZcA6poYCXl8l@oXgKq+3752CmA7w5|#`lN{TO$z|HF#K+%qwc3wQA+E{OosyZ zm^wi)FK|_IUZnXr6@J!m&^RB#YnQGVw2n@M zjX^RDy!`9<-kb#9U#0n|x1@wRlkYdYk{SE#1>t$vZOrdjw1{o=Bb3oDx=fqeN#lr( z)x~|_6Y|`mSMF1T_Wa(L(!}@ig1`Ol^#c62uhe0ez4Uw>AC(dp0rTVXHy2u4we1>K zG2#x_QteabYbVF7v3wejkZPXS77o5CcsnO`!1niIDUrIYpu+7Rx~kuHO{u0kDh4dG8=XHt^^|=Tcwpsk(A-z{oE471A zxA&bjU1WZpp|4ZR`H=m|UOuX9RbJ-1!!G#QZBf~+d+B}N(3eF?;*O#k7vLmvKQTvX4?=Y zt3AEQeCgSMul*#rzWu_EU@8bdMytK)Y2# zVd>DP{DT<1qT_Glyxa31N)&%S&)jyEeTPl%T1hPGBVM%a3KP0dI<&w%Xzg?JU=}!p z7)y<2?liil_C%71{75&$^w+~fn9`rtc9MAoMTVc0%6!X5okQ61eMSe>-*7Q{x+viX zjE*d)kt8BKa;}JNOha=PIe2Gl%WYQ*Pz_xDoPh-k<1PS`mA$CAGCYEI)+$Kc(p5=e z%T#A^>2)eOBn*?@^`ZQ43wL>mSHN8lCUbPqj)U`a$`!_Yi2YWdVm%8Y$AT){J7d>W z5?d0vZ(?6PWtJKo%9Shf?)>1uO&ul-b~wtLT@t0^rb3zw6M2e1)O3sR&>di&!)#A5 z%4*DA9ZjVbTb~7PKsJv;BU#hq=lpbp`eoG_^lMxkw_#%(*Sz!ZdeWSxlZ|44)@+$_ zY_e@zm>!Cm9Z|ls^i_2@^c6LE?KAvqQBz2nD5K?kxCeNw2tU0g6ETUY;kC~EWBvHC zAg&B>bVJWCsh#fr`ll70I+V=Q!oNJsY=(&07Cm2$N4C&@0l4W=oKs4KJmT`X;WM=^ zj18{4tLyCo2P2CFgZ1m<(Lz)?iDd7qQ&-jMX*tO)?`L(>Rz<-Qz9m9VNRWq!(K=x) zKz*T;J>Ea(wcqStuGS-YKeS&8B|*@@hH2TZ4=FbOmzsP31w@WVMfJ^#Fo9O4 zvv2osm~jO$)`Res&>k@Af2u5qTWpy9=6^4W%x4`ubZ@`?p1m2@Y>&aTO7_1a=8|K! znyk$24qJaQ4+K1Ivo3KcU~+zP=0NcW4`RrsI}uiP_9kmyL)1p9_Utt7{E_ zTTqSuazQ-t(T{i(h-?pG@vvJCd+`Sww$ayRL{>AlwH&E7)ne`$PtjJQ5Xg|Wu-;Qf z!Wt+XH(3Ra^S6fY=xN|`=$Pnq6R$PepWCU$WTIhZ^6N4wo({!v_m4@y3LFdPOO5y3 zt~#LT%vwa&_0LN`mQ0?I+8aQy2vpiwpV?v=@lVuPPyDW8*{VQz8O|Ev)4@k0PDWoU zqrADqlTR0P<7qC&>IZ&;WY_#g)xfC<^-uk`X*`t_rTq{**Qmza0l|GCH;;0SuRcZ6 z5)Qid&8`kgRhQJz^pUL3j!KCgsO0Z()8cH~fJ+j|I!jIGyX8Ly<3E>zmZR+wfhT0s z$hFAo+3$0w@wEc2vWvhQ@RVyK<=q%CYAE4Z=(TCNp#9E&rraJ_nHi_+6xPva6iT+D zR1pl@L->0XQ08U4334KBdPrwe7sfdY($CtN@!CAmM$*<2KJUKY8qDHuB{^n!_dXf=?Sk|hZc`HkL zF%g44c%tBCUbz$=9h~dkIwSJM8A?L*RlAqKLek3=(}ZLiw{ySjsBy=*k=~}cyIO1Q zuDiAEu(h-!HZJmM8u~QB75F`%QsVZ^%Z0boj{4-;TJd(yHiqUR9}DFF780e1p@Y=m zs-!fQFR)ik)XucN9(MtM+1XOFNlamflNUd3_pMhSs!Gg!%h~+U}H1>faYjl7#Em8lUt9 z;@k`|rh&6muZMZ&qTx>ws1w2sn=ihMh4SSd$9-F@#ZB7A>%*R?or;y|aNdI-qG+!m zfN?USm&h1cod&YON7HwnWcJ9Y+}}(wF4A`2!W-A2?!V7yiA)olpOlamLT$ROv6&+C zy@TPdZoxnFK4u_eGH`y3p7Q7vwH;rI6EPlT_-RQ0(kDs(t!33+i&v`U8%_F_QvvHY zC!!C9v$?;u{{AXzrvEdz?>U-DI%30!rqRB!?fV8sbLie`XMw2|*S&@WCO0(Ys!yh^ zI=y4Sqm{7MMq~AH8QQvaFWK zuHe{=QnY(NcE(pHzv@*>S|+m{>g`QXIcbui55MzoAFq$SKTmkO*Q|@vx%sQOxAcB7 zPv_Mo`?h#Q7-So@5E-~L@Rf0h4zb1pPY45au!Jw}&(&~Iv6+G@S`Yvz5+Zqm$#xh$6d{)$!)s z#zCy=Eq8N_`wbncMw!2YN1V}>qh@8Kzcj|Nl7mvFo~oXcPVo`&jdsk#sMUEXS6iv$ ztGq2_x$@nOB*d91)_P7XS>us&i(xbGEZytyi*-IIssAWCeHixCgNr4JuoIL7GMBp} z2H@4Ia3Czmq7P5Dxoxv(dqi(#AL=Bfrn1ciX>-P2Jz@zXgs!)Fz%XNMLt%I&5) z-m_CMsogKthb1EWRCy>%#}rcOt&!h>i}LtZ;5U*pt(=v#)|gzu+!DzQ&z|dGp*fG5 zYIW`WxURyZ&?&A+fp;6|QeK}!qWHSrvA?y6852pGQy0WMJW;#=TXcGFH;o9lB&YTeET&iW4GD(*V&5LhNWvO^ZPkn!CjiiE%X6vy z(X&I_vi>s2UlUzZK*RBuboZ2m28m|Mwdlnt=h~7y8)p1yl{pUQwiN2XtE%3*<=`4O87a^0~Q0iCU?JCOjC?IJ0^+wC||)X)9JuLa$aCSPJAQh#-Z- z3d^mv(2x&8Q(x~|3C0Kx8NIc5ymp7mzLmG8J@&0)88~9ApZt10oO%Cqvu!Ycxs0*~ zztIvZM4M6t2YL0dH0Pd)SHXR=I=i{q#efe;FTCL5${Kt5L+8N2K+;iSKzBmjgu=;$ zt`?<~qrX5shPvG2*kV-Qik%RYOfqZrb!#T~_Dr@1ok$e-09Y^Ik?d$my>1ICvn?B9 zKRO!leVomR%M$J3r#}wYYhn#{%pg#{-CC*Y7bqg;S#iJG{Ti*26A?iXLjyT^q|(7> zfz_;i7N3SL+9%0l zTKXN>Ph9Bv)_o%D$TfV&iUC}^5^nILj@9kEu)RLZ;&<`kA;U0RxZr8lSWvGnbD-=jA_qsYxg zZg!lH`gj-k*cLT~m9O@i^R(Yph%7#TOA+ISc$w27RkMa!vgGX6Er^^CdM0Fwtn7)n zunJO5F>7%CAjIOuYJI3;NgsTjhm4$f$Dza(ioc5*%hs~ubd;n|Urkb1d{Zsos_(}0 zw7c3hpBB5+A4Y-*<+#8!VJu78ERymv{aPX&VMc!?J@2^9k`t@iUxS@KR%JOXBw`{)xkDgEgJ z$tZ*yrb&9njw5+zZL|@YRkFSeF>S@TpnfM0P38FU{3q5Eh9i&^L+=xf@<@NPY8GGv z{2m9vWG?qR%1A94k8!fTNp~5Rt&sTC(8Ww_`54S@`yf7mfk^(M=%W&_8)5-3V3;le zE6PeS;Q)ai)e&$Kg^Sf6b`Ru|pk?MG&?6rNjs8Ud3$zkkO^kA)N(80)!F4BtZzgPR zz=Yhuq{ab7Ml*>8mdl77>6FxDggIY66A|oL0ESIYQ8fk^Edg;tBBT`p+!8k*Va4~p zz!QL40a~bGofQO7EDVkT%;1c0M|3C&;H6`*b4e_a`wf`0`SENhW3a$gn*{uVAP}~u z1e$ULq4jD2X6gH&5i&~(Prizp6vyRKE2Bo5DqGX1q4}z2L!?3%%FHB`yx65 zQFw#qZ}of}q*Sp6xjOhXLI}gX2EYj#$WihXgD{YLzm3icksFN!>5hGk>2CwBnr5fT z&;Y}|GejaJs@DU%iwt^*fVgFc)V!ctX;3YPjsta*B}k;yZge3yGUx&lWDM{F4f*Kp zfBnR@&;b&1Sv)!Z9nlJ_0gWDuTNXgvjj-aU01i-S8IckB?}%9y7SNTNm3r*IE`wRj z1x&zdI*V=@ar2Y`VF3xDaqQ7HU>&G_GF{GYQnVrU`wbV>>@on$ZpFuV^|`x8Gt zga)8M_$*M<KdvpS5)c~s%rXw|wB63*}2vjVVog#)5$aE?W`%Ay50ts0b ze8~Rlh^i`I$vgr*vO%aN&7~FIZIIAr6C1_>2s8k+ny5pDHU$x|8-|fu3=rrh0`}QY z#c-hg0J^VZY{SY)4FZRvNVWOF zoPf-(3R+r3t`v|Ep0x;tsv&S_w=tzE7`7_VF>c+k_=REw4sC|YiUO^Q3F^{&^eowe z3M6P5Z7LBs6!2!o4Cvrt$bCLUV!jd@8y}Xnd*lY zgu;S>bOK1x@Kxh>skH%kK#ml=q&w;t+*c>Y_$c=6lR8>HIzH_uVDudTlw_(`_-xh; z*pxrpq{MuCsVr;?LPmLy0Axtfcbgm2a~;#53`JD7#H~jM9h2w$Nd>05I&3mKo1(#yeA{rTI#0&7U=^q%5Q+J zUF7c36+dk`utz5Qf_P&iY;up4hgEAS*7sYL`Bxt~kE*^1UcUu_$SELdl>tEsDiIGo z1$3&veWE~g<+VYG6B4;|?{l+3xY#jI!o8xWK3q0$dsyl|nP+&=o(lrAmBb%BqT(G? zyVjf2Z~K^RUtb&eT$pmsxr}spuv|@oxKf+jqs+UYTXOf`$l2XWtcs9AHBF-sKwSQI z6tZ&1K-Q4>)JLW+AF#LYd*>Cgs{d!fbGu~X5DWs%r{T4T?$i5(2|dkc^SXF#`Ks9% z*UA|}4PWk@l%k0_b4~qG2+`W+96O(4vhsuRz+9>lRzX>hzgA@^q8^dfo1!@Y$Gz$e z94$i#h`>|o0k2XK{5b$Sr^(cdW3 z69!`a$c_M)dUP%fB2$4i?5UP0Rs?sQg96xoHNU~W({L;;lyr?PehHqZ3TW%Ax68ti6TlsbtJjy z{L}Ec?ghZmH8~9T+Kh!&JUb^tydvhc##-!su6FAPb2@k`1InjIIPXBTD9S4L+nvdz z1>d_lnoQepH0t4d1l-trfL4}czxc?iFBd+6h*OuozYd&dUl+F`BI75!Dfsfrd1f6X zju4y^!2h3vDDGvFY=cCda|f3?>X zRq`r8;;5HXx!8LU5CGDZ9{su=)~5Qo*K3nU1ES>DAVj#JPf2k1(av@XxM6lOv}1Cy zh(b@XtYp0441x5~4s+jiGt45`OL6VA70UPNt3gD}{`cp70i+xzGVTWVXAqH*=ep82 z9J5LKakBt!86i+y$a-1*5(ao3AAz=-r#VVj?bgoRze;(&epqtv?#%bjd{oyJ_BwUVy z|Ahaftst*jP4lUM%RwMksV|7^y+_#i0@x|uj(`B**OihvKMx!gI&J|RWe=BFZ2@70 zsodkQ+N)0@HJn=OA9M)2y?sBoQGd(zN8;253`c7d_2e{ByvN4p(15UKjdeSNUpJ?! z%>6U#$r*JOS>5k$&icN{#s{ox#A7DuSQYT?Ckty9k>z9YOj5kw$=x)3J7xMkQge>H z-?FyKM!pWfzTYuh+9!-f?`Ao(@QX?t3#WiZKk(3Hf^fzvX$K%V{MpbJ@-b1=r!Fy5 zU;=f=Iu7&dHBfv0;-SDm^I$kvKw|V( zVVwTVE%h^za;%rxhk^ivM~xRb?bXiMVa}`W@y;aR=GT3C+4J3u`}(DFf)1?4A=Rc* z_HikJC`;`jwr>;P4WZwT*^YcwBKi7+2%P2gbEINQNpbu3BYOm^cG!4fbjdm$n3WVw zo+}@C{@|&=Xlr$`_my9bO;sNc%AY3ox$*EHNgoY786GIx%r?~4zP<2E0!y}x`A*Xz z$WijBWIUnD*~wABi)nS{>BzXft5mz{_PFL3lv`It1|MLfci1qzUD7)ytK>w!XP;Pk z*siH#YfAQWcU^U}{&K6xd*62Nh4wqJ8(0{1MfK|JgUxN+$7b1B;Tz88OqZI%z2Uxq zFIzq2t?Ayyq~h@g>;Efe?U-gW(rE|2S9)9BQmqBs+oEqA4scCj6(5|4J%XvLM^|1T zs46gncI;`#^+tWhjhd41a8Bt&1HKX;B@bE@U%j>IRXvE>4Q-ustV^7WBVb~wtr)9Z ziIXQh$y-+x7IWNt1sCzMVq}#peoqN7i>eiWHH|FiAWRot+M5b%=0(L(0 z^rJUDt2Y#5;m}nk(;&K(V`6;z`4}@{E7b&6F;N})$J2B_HMeTU*^P@0`{vnYXa?I` zV1_?liu}2ISIbfo&L<1IoFJOnAIg4Vk*`xg4?>aW^!c9 zuidQpicf3}D{<^X!JJMjzX)fF@UATvawK$TB9E2eBh zpR||ufd(74Hy0DqOio(KMcl5lj-53dUoqLXv=UpccPl!{SnvpZI*L)HOFS1Y*cgHd z))_bboh&t&yqk7wBfPt|#F!@vUVL8_!&v|U@H#-TRuj$;%xbsnA9uRKd zCg=x-X5x$yj}z*`iQbEeUKyfH1cbG5S7LKt&tmg(i!ykZl2|~Px5Kdl1gP;bw#%Jg zu$kyTSr11%1?7y2csV`oH+|ya6%t=cx;wC?e$W(*Rq*99I-l^QJN_gt@mxy?$uC;- zaWs<5q)7~Znq?V_TZ#FZaF(d_>u$M$$~=*tgP6R!AywwWvsIJz_n<;DCF+eY%-Sdz&XkE^g%Uk@)zHSel3-Ow^8F<`zp zpZqyK{<`v}*2THfKA|Wx_jFe6OXJ;$U*LHuY%|(aeoaQ-VctjMKF|9f zmdt zx|m!RVt?pq`jUB)rQdjeb351U?eG9$>h8%RIYb7Oo_kx$t#4m#Fwv-FAJY8Z@T_lI z@EjcZT~JJkYKu)=FKV@3eK_^QC{2s{)`ONcO!ISF&04R~`cEg5OcsUqUnAnw0S`f>IjI?|^seuSO_sh)q*2R=|QUjlwo#8*7O zyd5`RVx#iYKGF!njLhUe?>JS4evFdB6=T9CoF{4F9w?b4Va7x4^>cx_1}MuBa+kz4 zMl$U!T4-Cp5T*@-61R1l>+*MU*v8JEzrN9vzQI=5?+`j3sNSqNHayDI&NmaeSo?2 zoz|54WG?(r%Zc5S>-_Zlq17XGqW5|8;fe;|FWdcg_=w2d02$0ra>kudenctMF7@i= zGLi~(o#Qbek-eAxOf0?OEY{w9%&-8+ADAf$jX(dE;LfsYbfp$h6ci}}$bOCLlF-#H zX!Os<+2_mHIqLIupat{CC(l)77l%b!GJBuFt(ONsq*tD;Nl8QyhBiEZU?56(otbRO zB-{u-I*tL_#xOwP(?wf7i*kSjI-Uz>Jh@P3=T}xM^#9emVxUi4bGWMAu|dqdvb8ad zMqD*Jt!axweERG8j}htBY;7e$4FC#EPdan-^a3@O>2xFfdb_KHXd_Xzgjir~9Wx(! z7M->7EQj0;RQqTg4PothPH$vEa2<-5()0Bjhc+Iz$hVw1arI5H0ZrG&^5dEBj~8ZJ zfs_s`FXx;0m3N-UP@{NH-}T8Oh8#JzO#tjIzn9U6D7G4Sp4*a*dX%cUq{9~E-Bi9y z9GeK;HZXH)!_IGrj2~xT47A@TX8X~xTuX_P`BiVqs8jQ5A|^4Mf}*)T)*@K$hKW`#SnUR!AFDr9QA|62wqP2^$$~xX9Wux z(AZxK)BUm2rDLJRlnif5y2|`n+qo1_nC@-PdMxiG3`02NVC%kni(yzfCo^2jq8`4x z@MM4W#WvMcE<}2(S3{%OrSoaIS7BMHiXKd8aUVGODY8Px8f_3lrfTbBjWj-{8h|p< zGSmKC5!P+gb8cJY|Eqp+I`sS-X+yEgeq33&iww+S8Ph&x;0HD{CPU+|;k_t5jAg;e zCuieN+hv4^Yl1wlxL;Ho^nr;$WV{+p2;XqqHx|!iB}(9b{3$75ZFPD!!fR5-3!1)o z#K~Dz;EquQ4g;CUX`GIW{sA)iwc=QN0Iu(ars1`l;PO@EmbwtcH}OJ=pXv9#GBTsx zpeVY#>vsYUn-nv~hHm;t!DJFsyPeB&5hOlcEl{&M^X>l{q3d$c7)Y_$DhUl5dI5BKUuLi{NbO6rg$nGvRlPMBzvxHEFHS^qL852E6`iO+`S(Rp|hdnM0&{rA2#eJ2{D= zmofocao%)`o%ej!trh%phPS?3HB`u(>JRwM1Hx1lmCN`=N6#`w4mhj)F?>)PeggZh z%`hN)XqeyUX3xYAAKeSZGPvw-*HZX&A{EboSkU>7!*|*D8-FUn^bLoBkn%Uk01c1z z?zQ@WD-!Db{QOf=rQQz&xxpAdspF)hbgqAXJs${pqmp(ObAcjb-IA#$0< z&o%*lx+iO_VV_NQTPY*QC06Wx!{s*$1u@PB=bYMy4)>~DSY6u10Qd22wPwwxEK4(e zREAqfx;yC*aq_papz^07jcxwd22zrgidDPmTw))dO!~;f*DUh*tW+2hjO|A^H9tL% zi1AC2urL^B!{b<6xoX4Au^J2?>vB=LUtN$&Qt)`WWmrpzTX6L7i>TivUXW!F5T?{+c zhs5nvyZ)#D>9+JNc#~($18eo9;|wc;7t=0@AiQt8-ni$@&Q+bJ4QgiHMvSyC*+Lf2 zXeg_mmSRK?|0ZWBq{r`ENqGi%aZ1|fEFrdmJ2|!>eBZO8*rL4o`;kJdEermm$QSPE z2m2{kXL%L9q8W)H>iBW-IUnv{ic4G0pT8e&OilE6VLpn6%aWazexx4m)!%$cWHRQHa7rm`(7f-R6Gqv9u%o>b9@s87}K=i^ z#%w)n_h)arcLQU7MspN+X7lCJUOvASd3W(bd4vvV#FQS2IHNG}m7+zdJTqW3AkNzI ze}RAhx*SH77c^vGkz3s3d{G$LXp&WHRL`fkKwB6%7tAJc)2qKeiY+T0rba<=`(sT> zwM=Man3$MOFIgpMHcq%c=eT6R$}y&8{FS-i#M1?if$woT%4A(n**D5fQXCC-buPr< zz(r=M^HkV4SNR)1+?UIQj}qv@wNp~DKDwNcoS&R-+hYzU1d)-~4$^cRpUG$|YG0LdIG=`A-bB+0?Y;qfaX`Q5r!qh>C)eyp3S!DDD$vQ{qJYYeL?6JpdQ60ouJuly)(DlY*!XL9}+3XdCIUrNrOmx0XgF zqL@VG!h13=pvA698ExNBA@ZJZFA*@C!VRn1`gnTTWm`*(u5USX%)aV@f(Caajd~T0h3OZO&JK8x&jT!*dYZ%Nky~ zax?AKI3gDf%U|Vum@`7Hp29d@ys8q{fd9@Xk1z zH}+cFp%!S0*?P&mD|U+-Fa~alg2Hi>(WuItL)@$shr6w|6S z(4pEKiIWI>Nqt}2Gv(>QSG!1`}VC-8{^0v51HTzjgn4x9s z#Jw=?>jw1N>Q|ADRAu~89lol+B>H_nV)7ZlG80u)Vzd-tqkQxb0Lk$vAb?dK`kath zEH?HuT^m>M$;TKwxMq?cCe1YO0}P!H?bcbJvlbAHcjHchcM1(+yr3jSlAsEl;qQBh zQQ`=!(gOg*xFid+Ad=_~g0&;v+zfd;5U7UROf?py9|VDpvNNfOA0nec;R^H!LWl|< zK^GDXd5Zn%g#Z#jl#)e;kx+;zafR3iky~Pf@syd6MiNkiqKAPi{>e4R9se-koVUnt zlIH|Id1US0@oY6cNW|8+H$EA_x8(EybcLHnX#h?e&{9Q3dihy^?_W`pE)t5N0RtEf z3Oh2Eiy(&-JXPOC$O+v*MwrqRz6UOS7y{|v|L8PN4#5S%NH(wUMczRc(E&tzv1ubY zz<=7sk$wl@gNq{Q$naYx{ThIX^wLeJrD51R08BhExFZ8#7#Iu`I)hHoMFL8y z>xp(j2!kj66?UfrBng9oADL^6vXW?dRjO?iYKDy zX`mdkGsgfe$)(Yk3Igi10Be?v<3>X1|EeG(nn8uQoOjZZg2%K-G#Ls7#nXY}afnG2 zK;QpWgFOeZP0!1kXf|ZANT_!Mfa98S)leifhP(n32OT-kEnGrI-yxCU0AWJ;3jhDh zD6F|e5IC<4yxDz_!$HS*0~^dTtJFV0H#O@!1M6E5t^6O5+K)iK!2iKv+w+y6BnN(p zn=^A=i_x@JJIib;cW$`y!PgE$3;Ki-$yxd-KEW}#oi zPS;)vyKU)=9vl4k?kW06R+QtJL#{8qh1I{Vt`4~he*Lc4OwRJEao9Ys<%=}dWv5?jlQOq|QvZX11L@r_K)hnab+FN7pmuKI?NvYb^^xUbI6t3l zenam12up*U^+KCt|2Ggo;4$l3bPlW)cA>;Vv2>VchSNW=ISdgGRi{HVJP;Vbq2_71 z;;~jwtP-#HO1eJzJ^ZS;V_~@(1Rrq{nC#Zy`mXr^0N`=!T^dJWj~yavbM*g=If&3s zfU!bH=F%7dM&(DjVi7*7^~7(pD)h5W?)Mh+;QKCsnLsjyiQG~ z^N#NSa02{cUP!YM?rRrTbJdgA>dpQ$FG1)e5yf)jdp2IgcMt1wsw@hz70g4>4fM_@ zbya;lN=e&QzkeWttq+!r`w?la{lGvG8~JrfUM5+@8?ho`qkoFG!Uy47%*SL@?n|)4 z>r=n{ZN6~BPV12nx-od2QermH-u#2}AAbMqzS+;f&(rVEaFK-O*-t}#bGE3OAM)Vd z&CZxTC$G&}rN;!t-Q{`o{9VBl%Z=*2WZt0}mke!xK+< zpNV}RwX1!HC)R?DvnQrC*Ef3kFTQoe|9!sozsp4PzEzIDuy@nY(@nu{>y_U- ze$y5CVcv(wGOh#Xq+jqQE|`zN3qjYB-e+o^of7(o@JzWm^sScs@hf=q8{hw|A+U69 zF!(fJ+hwT+^SnVVLIgg4|(`&!oTF< zZJdUj+P~PtKWX^2cn7%O)y&nEr5m07V`hNF`@^4blqY+;k0u-^!?6kL0l)b(>gjat zi@Sx5`<4KNGw%nYymlm&_0K|2*6mkP9si_{TP5xWk{V$RAZ0yQWNKv~|6RRc#en90 zc$8}S$9MHNExzH8KGSDH`-2rpQ%R2v{m%T&^TS8mXqkN;tmH6Uj??S3{5#{iF~KnX zcG;v_8K~XC3pHsgmduhnU9f!;?tAs2xHI=<1a;PK4_naqQxEa0HFMq<6J2|I@kx$0 zmBoI7|ClUfFfZW!(}Z3B#G!CDOAM~GT)?gH?VPe*0)%f<*CVxH^h@X4bZjR)p?MwN z-MYiJH_J%;{cod}6M~#h>KpiF>x<2;!C&F!q&5>FF@5Ofop;H%%q0lDHKHP*5!ZVR zysyr32xNe+@V4TzbnEQae_!_rfmOA?=fU@veeDTGBr9nz2dOQKfzLci(_?JX-7s7G zMrYhQlUSSka=Yy1>y+uC{D%8MI91X;j zaexLUvxUt<)$Es0u9}&;^^Pr$Z|R^qfOzT&e}-QRzMU#Ujm!lzv)_2`F~gRXQVQ_vRV}UpW1&Rw|eo>v{e5X;}=0-JHjg~D}~%{Am^Dp zGJIE0@$IXBCS4>s*ktq(5o3Z)#$QvJmILfXtXVLBA!Wk<+E7@6Eep?c4Mw7WM&y4Q z@;@W;|JymDqGGbp_CY1Zj|U&<1Hxby1#R+NV6(0S*3thlb;2A}1Bm9A#aO9h1w2HG z_AInK+60?)PcVaiJtxV4LYBd*j+4feKrY!0Ww4ij*#yN8cmPgcqh!p@f6-&XBF=qa zV}Kgaf#gYj5_^oa2^e6TFGaKkqrDH->u2VNgqVOomI9XhE|c=j!Bb!estB1tsj!FO zMjl-|#vdQRKfYA&bqj;mQiIkW#iFqUz=Xi#%kUPUxdToL$Xdy{o23yQ;OdjRihu2j zbAg%34SFyRLpX!0;pVD+$hIOigb2_8;Ry;X%EONzWWXWm4N^_Q3p1v%4m5PRK&Bg% z8mRzoR3@SL;9v&+@sQKI6p&mqfZ|TxehBl^zI&2t$}i`*?uL;KbnL?H{k)JTh-al~2cuj7G~b_~ZjyfgU;? zP%B`oi3i+tgTm^CkLickK-{_%c!s*d$a$5&R@zVZlv@vHXy$3jCPW3j2LbHSV4ull z*8^nPSwwcKzfGPFiUEcHQjjqs(3Y%dBHzjqzCM(!2E`bjrs4l z*SmhijxTj|?|-`5Urjv#ysnMe*PjL7l!ANlc|3LuT-yuutKvbeT?d9gx*pgTa^}4< z)+I^wyLqVm&DwBkYqG2#4McDjS{srT1wp?s2at>sz(Ihq);p+1kx~7NwV!FS-)=$t zz>5z|@jrumPEEf8_MqgR$@k2;(2NYU5zI%)v`9tnds2x~1YA`!F7uBic-%Hdy3Rou zu?n8h5`hHysS@qp)TU#P*r^yT@?fl1GgWTQak*PtWLGR1k^YNUtrW@(1ZxJT6c8E^ z6-C(mf+Jwg2!X>7ZZJUUwVwehnEGeNuj1PfEPwt#ug%GYTr&DnTU$`DM@oA+DM~Yt z5+9VG#a@1zdA(S`Grt(2G!TlDSMcNGk?0kHu0bG|v?nGU&{Q2KcnCun;BF|RCFXK9 z0Bu~HigvRDViM7D+D1Z~uhUS0oJO9>3m4$FV~OR9TeX#hp%^s!UcYGaM=8Mp(ke>4 z$u_F$FkQk-d%19GhN>BUWgwV7o*zaf;vV@-iIatD%7c5=a^MS#vCnk{p6OtpV(nUb zsAAFA;BGzP^3KR-kT@c^-^Nd_hM3p$xwt zYRu4ht1!r%(I3Fb{0y=fmhquWWu)tMTVj*xeW!Nt9`LK-)sMRm{lh5!*1J<6k z*gOOML;gfqu>9q2m+|hm2yE&%M;}a1wuFRueNTUH&CSn8pw)j8=8q<|sK0GKOFn;1 zuM3&=IW_cebO{37KJf{V>fkL9e0c8%v_H&>!GcA*NGGWS-I?D9q_HRJ6#_B?j(K?gtLw{C{9X`A?D!VYOnJ`+{ci5hO&O9{tpd(yVq7Y^Jc10OSp{o$v@8{F zj3+!&;v^%KvYKtE-zsj!Td`McwBxb<#kf9Ikw;!>JCq*So_tonIWS4Q1swUT-^gOl2D9B**7fyZ0ptsQ&T2w0};LJTcE5F<)SoZ|wTY>M2^t zo1}M(Yk)t?_Pf?20pwqa-I6I3pctYEtm#7|(SZRuAc@=Pe(^#Ik|r>}dbe67wq|c{ zJnZzmex3%Dc9zKcH2{9?uPhac)8aV}e8&|2rBYbHEP&@YfksEoxxt)$MX0dk?IO)B zw-4#5yNGEEbl9=duKw|tP085^vd1lyLFU)OoB0f9$`k$qt4PTi9I=J65&*sTvgYNR zsjkknq$8C>us2ek*v7wcH3YOlizNHX53#9b{&$nlo<5Q~qH#*Q%607Ff2D1^rB?5y zm_+Y$vC?;L6l5TvA^yCQ-$be%?RqfIeVe2zxLO4>z(pT1H*iNF$lDZz+L93KKI{hc zUG~Ikb5!wDcQWePM>F5cxvs`wS4+Gie2Iz7Es^0k z+wgEYl1};sNTKnGt%Y(LynI<>=Kgy3o>-QsSFDJZ4_rt|q2_LRz*758>A98`#ZxT} zLBk~aR{9_*tAvF4I{p1v7*cMh>S2BH9@zq`kuonbqJQr!E6L%(^ebZK0IWFf6#aN% zd2}FhA!jGvNVV7*f&WULCyz2VK%mw@vR%muskJY^Iu?G0%wU2;!O!keM}{5yF+iT0 z*sWxAF?^)pd7ze^@F~JJ;4Rud;vF8V4p8}#e=rNlYDwsc0-1NIeqaU}_?c(qhjP`YkC-JkU_pDV16ocyD$0=`yp9W3u>8a&a52#$hK6zj=dC&@mpLEP(v(0A{nS8+syp_AGx%AtTUu<*^uiaf5 zD>0k2a~9H)Ay~$9)#!R95XAzFlJjlUQ3o0j#Kq28=>PWYgKaPf7p;-OX{h5jd^~Li z9RWn%`z+6A7fPohP#ezFlmB}T zn;9e1z(Ue#Q=Lp`kinSq-Uo`;g)ZC9-gjx|ddQ8a&uwJ(k|p*Q?nrgvlC^j(MRmT5 zQ59iR4xO*x4E~ef3M>1aA$)Y;<sNnQ3`KV8b(QSeTN-@rF#GJSNX`3lN3qOCk9O1>iIJNy`f&v$t#rIA;rmW8FO#Y`0Br10BjMyscx?Bvq3%DkOi+ z36L$5ZrZ}9`tw>XOlv~UAXwI;BaRm+;v^160(hYbc(%~JRi_AsLg_nE!c8+O?>0q1 z7OuXRvI*uP$?p=|j^i0%%v7_dvGA=U(Nb0JRYJ>~@e7N53{5bx{*H0@#{Lio9NG;NSw%8keGf?JR@2WzCCPC1lR zLZEt^B>^q>rxLx9mHqTQMF(>DkfE{o4<)EiR~E2Ib;@b)E=V=diHg4XeG!CDL(Kw# z^b*tEyqk6mmZ*a&z1MFbrl=&;X`DF>xwLP3N4a7zVE7n<<%SgEkMh3JCv}K)sNpUm zy}tk@Ixqye%%To{I44%McN5|ldjduXw{T_2N* zy;4Fh1#bV;d&&LEF;3(VZ%QgWZ$rC;-^!kc;Dd#}S!kdFS&h!b)yjEhIO)!4uq00M}JHQ2l8Rxt1|?h0YuF~eV4KuZ3=3HAyjk+qdfRNyvrX}@`V@dlt{4hg{6*iYU zkYubj1!1BQ$-zwD5+jvh^Bs!`n6sSFnfEicA@||u8x0^$7XA6}Bm@`27eIXSYVA5x z-p1TGEK~mT7**TFHtlHIOQy_4l=B0%yWt_p)sya>lz!@oEAJb!?-n)OJl$(P!wlex zd^O)MAM?;lE2lHnTNq%x-Gt0QO$$)CrJU38{-AN?=%AqG*%r=?eeg0!Lkz5&QqUR9 zM85Zzx+X&@P9N|fcbe`L#H>cj>lH(nFT8*m{eANJ_2ZUy%O0R!LH9`Xira4W_IPWO z_h9dRTw9RY6v`vn)A_tj#Kcy><{B>D+GWMW`x|w%b}Z&4kKo2Vcohj9PEpNv%TJ3E zx&(hIU!6XKJgN=<vKcm*Ao73-FOw+rlG7=q5b`w@3` z;DU3_??%?tC?vz9Gpx7AcXW=<^h?>mcBSSx<&8Qm8B4lmXE9mVf zNY8DolLTpiB?E$6cGQPVst;yDeBqix8G6s<97-gUV2^qg=n~k0!HmJLk6sBVDB7{*9+CDw8%y1A=pn9Kj5Yd zUWY@9JBEa|kD$>Ofe|Muy1si!FIjc#Eq_sCJ3(eeh%?4ucI&kP^h>yZOG1|qxA}^} z_p}lS7G8`4g^N&&05Hg3OsK1ln|c;5kIn|t0|ki~5KbIBduau;4u&X!li??VmEb&Q ze&?1W@u0-vzC+iI3pQwf14huHUxn1JcqbU$8+c$#Qz$3yu>L;Z>n^uuQL6MXY>z*h zyXJ1OW9`5R0UBC82m<4uJv#6f2W@`s%t~}16xC|cQVC@D1_$*uYM?}xY{MT!xop1n zNrFH%<}->KEwZ+G9V11neE$}~S~f~r!mG@?KdJL2m$TT~(Xr0=S5{U$f^!K@QwpOV z88bhH(KT{i&G1)4dPtcQuh)W+$<17Fkz>90P2zc%qb(575jmWJ6BxSw{`pq+cK7Qg zYDf-SQNzR63*TS9Rfzf4`}hnEW}utqcA;y~)3h41XIv+PLUCBQJ-!jt6|K(B)ES<^Q6BlwXy5?NW2he_5qM;* z{*@sB13879M!^9Fzy*`p zLkADSFb*ec2qvJ>D&~6%YnE+^e^=0GhXJ3z`B<@UC#=)8V6-w57X%+~UfWqu8rdurp%dxUjY+_;R zG249-;VX=sMm&;V8u~Cv+8S}&%9OKY6CduXYaMx4^*%^3XRFyFzW9+mvlDGbmdaqG z)-Vv1BN^!Ox(=^8&_dmeC4M=9=f~((ab1VD_d*skRtc6Cb;G)-Q%5LmMj%`**POnD zh2BO12!b0k<`H5L!I$Pbc)hK2Ugn!JqT`|HByyGO#qIcHtz4L=cglxu(-x#4J@J8F z3&`jgU%k6rR@JJLt`k$30>s-dNd9DC@Ufu;etj2mK=t)^a9X(e!_SRf7Rb5eFH2!B z=B!^AP}504k2sQZWw1bqD%U#7JE0M^JxlYKedHs#25dgp(IM;3F@upkN+AD2z)C1n zqv1sGPn!^KSt{QqtcAiZ7QXwAO-+)f7m^7}G==-5b8Q<-IO|}Ap#L-PFz2n>Yg(r_c44Lk18Gjmqwg|B`od2gzHRfyoi7}65O;9kN~Ga8t~E2DC6Q+rbY zKT8p1tLE%JWN5zsNCj`0PRzT2?3XNJ27;v>OmpBiZt&L1zm+fOSMgSm$jmA>vv*N` z*BE8Q@##}!GU9zlJE~@Gq11oHHYjawUjbYnL2S+Bt@ zhvFMgmysy3?e9)`_)_ZvCWuNO%hy}g=AYu-#*BDpUNXLcl)W6}54QsN<0QqqL4Qt9 zHlz;?8=c0FvDPPi?!C}fTteYko<`|>M#mJ|E1@B8?DZQ?&g*WVGuFd;^Gu?` zv3gKm7(+uHw){it+Sz_c*1g}lVyg%E9aGt?Rj3A;5H=h16B8r%5&nukq{OpLfHa2n zhYuN~sYU}Cn*D*zyQl(xNg8w+mXXwyuREJNx&{G<2jkYD!V!Dfbytm|^w}1*4gzA; zn9&21R-1_!Sjv0MVb}g9?URmUadlM7oUtc?9JkN5b?yEWM5iRZciaG zBkDTF{$@Wj>Hs9`#b$>)2p%upw}{(es|_Yo1%3xC=ITe3&TaQfJ094E+`tRV9cRTi zSw{!qpM+M(T)lzsF6x8LzFilAzE(4fAPgn$?K7*pDxgJDDfyeZA)F6LYv|n=Nezb; z=TO_>UeKq1JM!4gK9p*gv=LHe4ihRVUVgf8x>9=4(MNu;T)v(@(8aG%9k~46qv&ht zR4%!3^})`Zkmn?JA?WK9Wm8iW*LWW?W}c-|B|8Lk&G8@x-0{1}fR^YGM6~eGZ%K4ai&v z2N8bW!8!DE5bhILX81Tp!|yZfl`d*hUALXvBxJ}~kb9i>_1@eBUEf33lDfz?3^+CW~u1ByNqq79g$hoNZdU$>7%-@)Wl1GlP_o4?Uj zi3hjFZ0i}3VMf9@?jihhGyj!wCM0}|OIHv?YyB|c!C_rC?zl&x7By|@o7;pltem;@SfAFq2n(@{kXpor+KqHeiov1DHYQnelg`U{gp;u0D7|99*=N1l8!i;^ivE z1Vw-^b+-~7FgS)t%Q)>}_vR4!SrENU(N=*lM)H8GESm)U!B8xu?Y$F{{gZe0d6L~ z(>9E>n{Zsvx=}=*usp&Wl-$!9%@ly=Iq3TZt{o=j7ld*jObQsW^!&3sFUnszNH7A(HSI6^o$B?i=mPA+P zlh}!l5P+I3n$bwvSsXL(1ElHPrT@wJ`%w^XE>bQN$Va9S4Kst-jOFVd02*#Lh;Lv3 zlD^NPQ8u5Upoog@kz4^UcsGzmVF%Px|5qC$-Q_r|KnIu~s9x8d*PbneT7+^`%1h== zLJ65^0u(Wrb>g%P-A41D-T(sF0YHnombzmYRzC@EFb9|@Jbw+K{$-Ffn_s2nIE+Go z(~X`>r4D4)BE-lg3&J5M+yasa0iVp#JwMDFPgmN%1G>{96F|}U9e@c7*GR5GR{JdW zr7FKi$xc#g=g0&D-1rAT>*H~vu$ztA8@ccY#NDA+R<$%U!u%jh^zAFz6V3NBQ)A!c zB0i2h-;n}p{s()9{)0~e;M>A_6wd3igH+uZd@Kb~32Xutkdk-B#)SOo9xXsHy^DJ$K-r3r zd~(!p3;hok38)$5KCw@H1kW<$DTjPU0(~Zb0Pm9*XxtTJuSi2eh#LDvcI#ZVjH>WL zIuwIho@c$Nz~1)Wt{BJC3QyVrF(xE-HOukMfI}WFP>)xJW^)hg;%NOo3=y{&6e?_( zg%seJ2r37f*gAov%@|?+dCd@g^ZDi1?xcpq7~u?xCdFX)e6SI2225*mOQj@#){wh4dFCBqm?Mu^{55}!a9jV%xU)mBOjdcc*>SxC zfCBDqBwz7*c;-<+DM3!Pe@kSfk6`N{hWSnnBws?)iI`0I&zDu$-fEHk> zW#m?fzuDvAsNQ)t2a*q9(%!L8+H#Lk;m5o<*``?2Q`cQS1iRZWtP5g#(d;_R`BXwxiLLk z4&Efx&Jw(jAHQ*bzzggK!U`=u6wQFgFC9E(s0I>aOQOY!0*=AFkj#wm2s*26hZ3VR zgNHj%9VZv}u4X~oz5twbSx5DQ>(kjJbAY_0t^L%`AgQN`a6_@H)7nT}IRlGEuOpiXCgb_6FE9Kkw$zL-sRD1dUn| z-Xx}_g;`}dE>gyx9h~MYwP)2;9(X-V>5>g5SEjC???I|!S-PgD7}mO1bkpmPxB+}K zsB;WNIjPxNRXZ}Uj>!}xS$Y6cQY?u5=x9?i9WBro&^M?atuh@7jy=a=FK&XY@SYa= zT`R%SQ8E2tOqTO0An}0GYQRjh0vX8#?`P~Z(B@7@wYs%P5}bi_(#EzMr=9u`4V{7MG^$}TU~IG zXTCLgsXkq5`T#<}=6*w*gfHzrm9d9!Io%Niyyx8P_!fCi+_Gj9gJTsV>Pl#SSNn1U zxdzy=&!V0T4D)m4!y6PHhzp&FRt8hLgg7HAJEO~Creq_PsQM!J?=`6IIG;JzNG4#1 zBzp}1a5j9$^fS&sR58^1$Bh5?jw(vAlL>R(c&W>$A9VR0QW&0wVI*xNxB@{abP!hd zlCi!&bTTrrd=9;?eTr`aWG2b;|jw%IQ+0fd+s+-kQ4fpX5wQkZ}_ zWV9UCtoN;#j4YJU8?PYK5=ff#PNEhQ&{0_`BjtN6JIX&6-kDYPPBqAj{unDSM7C4r z)-*NJ=y!WZi)(2-A%6~;%=7RY<+%Ei9)ery9_iezb#Et$7N5?yz5FI29mLFGyK?@A zyx>mxCMNhkov0r(X3JgPLO|l!zSB#<$5@Teh=aW_|H+>#?b4|QRYZFajVK7`ciqjA zaiZVvLo)g{96m301(d+bDR&cq7VduOJt^w;(@CJm>Y+o>%Fnpd{~XWqaKu%4q-49$%QurxfeTVjA4eJj=sRgX>!&-7C`z^ z0M<9*y65!~d#|-cjPh*{ZXlHC%FcdrLL4~w9o+ACUf&rHJ`A)Ze`&FTLf9pVSb$(j zb^DoAUlu=Dk5mJ=CMj8Pi-Ga8X@_qosFNSuLp>e1pNR!m?R)shdXsO@fP>fq(q%*i^A`n zQvkF@Q{TKip{$$>oYVhgM`iU5j|2L{%GjwDS0J%GQwL!ulvt51?H}NEeUf~eHKl{1 ziN$6>KrI8R(0o4(2sNZgNw{VZ)Ie4}re02tj}o+^LaP99K*B4=?OrQsFUXCGR$sN2 zcb)&#VsTGA;Q=Nyl^#y7?D^hM52=ZpIY4gi0i!D=J?ioBfVKp`rkaCYL&uz1doO!q zk#760HZ;u}$FV(75G7E6v`Xdm1YI%^kBC})eP3Gso&Q0B^3Bf?fWT8&UjmckIJI-T zXjD~!b#A@o3m_j56Md{!+%8j@iNPF#FQROPy??mqIr9M{oETn7Z5L`9C~N-v)ocH1 zl!ck=D_Z**92o)pMU^BN6O_ovf%azcW<-u}Z8T3?@YP)uNYpM~YMdgat-Nm9lCRw3 zrI!(lXz4#gOZ6SaQQUx?3R=G4QgI*;i-9p=Xvo6S=Fik~dCdoo6J~;O7o5%aZ!~;n zmBT!pzB%>wSQLiehj%%S(@T*PxE5zs_p&uDn7+HbqiloW2yoE|gQ7eBjI$s0k6pD5Wn)f@(oG~FGJA@ z05k&m_(BY4M(2oDk9MqND)$MxqNX7Qie`ogQ8TT(o%tedC9x=wovKPob_NsFwpg)G z1;QCg#6KPD_B-Z)g^oDZFMo50$?Qgn-&+s%jP@iB3J6wc?nA^(UVtM;KAG@04ngoW z=x_OOV!G(yV;d-;Nh2;wcHwmEEy0Yl+ZG5&82AoVxHs_73h}hQ3^XE#Y6tztGOMM< z&H`XaEbRnTGNBp`sU|G4uw}N+ZLXsT+))xZr|J=QoB2#?(g4+`%I%fPa{_rqSO9lC zbyH*+u@Hf*?_c_0gi!wQ#&)H5`ayszL2`vKkaLxgFc6}B>%FNVjuJ>}ua9RSNG6Z? zf&`qMNkh{3!%}PUiDq*bi;kDF`bj%mC-DKk_G{Ut(7ixaKB3&dg}}UFbHMGHA3ITh z1XRGD1DkoA9(~?S`U$Wb3kF}?JND0CS~WnUiQtG&sU4+eQ0ZF-|G;ii31o_>5`+1M zu%SCi;|^eH9u$w!uou*fMN1haw_3f!C4$F8tE?G75C_9>BT=gcgBuZQ!_549cCpQ*X{N~p=TuGclbO0Pp| zc%|TepoNDG#8}7YbP6cTA})*b$)RYUYR9puZ#0Zg1B~+kMCDvH+uU~77`A-S*;9cR z%Yi;^rqg_3F>K0$f0vM11~e4yR*fwhxt%VqsrHy!Jh$X$kmjVxDJ~YhhtogpIXB|~ zMglF(;?P8k3l&o%2G)*7WN9%c`4(NVklf_t?X|A{>!gzAl-y6j8HQ#}Ur(!iE{-s1 zOmb$QB~+~&NSY=v6|WW_Q$^1Ut_L%s#Wu#mX1Ft_jSSHQTUtGSRpEQtMTj2A3)pJS zkUHv2@Nk}8e;)MsPDH~zv$hD4iSnY_foC>;A|8AB5!)g42e$PO>)*mIB`r=xNdu`I z8}Wd?;17$2V1*9;z_u~lH=vk%;UN2w&2HUVIH_G)$G@`xH1PJ#$e&M7n}Q4PAwiB9AwdKM8%v@N$B3r=B3SI z=)I|id1xdu<1{GiCGc!EykgdBLx~QDs6utXn^}Bn4}@Nx8;j0+dcS<*_W@gZu@Yq* z^YN0AHYbWE4x8UKFV3Q=)da2%H-XP;sFwilVt+6UJjv@e-}#~)yYi&gV!d@1v-7dn zz#ob&p;}pL!MD>c1~kmo5_Xl7mH{`(>aS<3&bLTr@smiUJ^7dJ!{y&~^YGJD>TC_; zpZQK_5*1qjV^5HidnigcXN6Lk%NW_ zZNvi3B+Rj)=#&R4iQIIRU(}(oUiN_tXXgRlC}pI-d5Zb99ygO>?C59pw#MVqupJG7 z1a0)~iv@#OCHfg<;<`FpL}}5Gp0Om`P$tOL+~7|cBkvp9SOc{ey?r2^v_;@NntuFm zvycbzni#BJIQrvyD^*FC{qK`J1}-{fw2?OrA^owF9@X}0>_dAn-bL;_n~ zQxof!$Y*_&z*jms!|_FpT-Ls3Xy;CNJ5&uAF1w@PGcQi%N_&7GHY zm~HT-(<5g}TUQOmg6r;cpg|RVxwp6YI#Fkn$bog|jgH)wkiKsrhn9fH;g3-AANb9{ zfG#6*^VNLhvL^XD{Yh#cAgqSw7#8!mrHOkW@y^rNb6J+Hz^_jwv|4%ZZ9o1^n1XY+{JUwljrYJa}RnCPt_W3rK}EJl^~gf zIUoI{zL``s4SJi<$=z;K;NSDEnX|*5We_;SrxlbTrMODsMC z*Zp_;!9H-3vi=G22P4J*K2xiFx&Iaum` zNS-A=%~R|~+F8ClQNUGckjmLenLKQ48-Wv*?yVzoJs80;>oC8Ga{Ks{H$?E$e6sE1 z_S`<_G!3mYm8X8{kSXju(kwgI>o0;m{kuBA?n6VkC|$N&)!(IRzRh7?qIp06Mkn3C zHkyMpp7ToQe6IOj>yK+v5m8ibNhVH-_qYZqGj1nakM7JW>8m!3cFf!OCp^L-o;q?5 zrIQ!zxWvYt7OrLr2qLfwQ1;|4^88uhB11#T1CUU)G`a%4id;_$9aDakdWDZzBx;X* zbgj|?Uj!0s5mZoHKFSwK7my_vanA?vrxc0dE43lI+}xAQ8`_GT#F&8;kzR1#O)X1t zexzZN)M_&tCStg!^F4oNuntRC*QJ{Yd#oASAMCI9rmwxycAWE=^cpbwzg!CJO#lAj z#&%wt?OSvJWKFSpkaTpkS@ap{9%3ismQ#vTXl#8KeQ0c+RlGEWE}#vnF#&ms1PlD3 z8Szk}+EA$c|I>)bdvW*`K_o03=`Ah6Gj~OLOCO@>A--5P9_0S(H{}6ty$49(MmnWE zE7~78Z$3V;COMj?MIPjzu9Ft8eK^T8BV*Z5u1xt;-2Y$RK^8?G7 zz^gZchja|CvsK1ILB$_aQUBi$4S_*pDP>Q9vPuJoE02LLD-y?vl>vV+8YnA+H0r=p zTZcVy2iX$>a;Ik5Hl$Be8Vr<8PT7P882Bsz9p$c{a1`ZRBfX(tu6y#3e%8N!IkaFv zuv#pJkfZ+BPX+159^TS6BK{XMLc;{RU;C1F5*}oH4>_3p(SX*ipk?6PUYFoDD0bSV zCP!uQtK4oFy3m zX+gRbkWOi&JEWz%rMpYIL>m6N^?9E0{RiLacl3_22g8B7_F8kzd9S&y`--LtM&w~( zUmTqt`hOGnrEC^Gv~^kS3=$mn%Y|=J=pF!cfPkUH^T)SE5Z1HgU72Ao|K(Go4f*b^Czf54y@(m(MQg8kx?VfgbH-HyDwX)=lU4z<7?x@P6AXZ)zBCq@ z8bM7vC@$;Y_>J3j#sEcavp74Rs|s|{agy2l*&4{l<2)mGvwu)w-W!pkZ3Vyx<-O?( zR_(W;fAIf(>x4L&GE{Kr1F+Pkff#hxe1hmVoPLVJ2lu7c`00SqOtb3S5mwGL5iYo? z!ts;A=!2ZDLMA*NX}1MO_<64yd9$;gO(t1DzKBVZYNb~cdWZmU6pK?8&DCoIqZpHa z(@PrOUjhyr0|Ep?KVb_mxz6f4t@*=;ggPVA5|O_C_#G49(B~$P5rlhIcOo(L12}~R zXfn_*nqvS0sSSz5bw-HX1F&Tv#2g`D^E%LI9^k0C;ry3P>p&9|3o^Dudm{)a!JnHT z_P7&`&XznfwqD?O=RiL(1YWtA0aB5AwjB_64UC`(Qc2+TNeg=s3WcXVZUFSDyt|EV z22lGqr2%8*&Byc(29e0CK6hn;z+Zg52;%g|@4EjA{q^tYAAc5XcGl4xn^!&_$drG_ zqLz!sql`|CMxHsNz#l%2{V=P14nbM>pkpsb`T+DG z(Li^@ts=DV@n_z0x|pL&Z7=Lh-)$Zs+oY6)r`mfnWvR<{Q-27{DxUn1w>6I}37bkJ zPYYLG=gFh3aHlVird+gO8fEUeF8B@L;r$lYT&UGuBH2j%+Y=^}ffmIHKnfPA7{5P) z(AY=Yk*^%@LkR8rt%D)D2OL@BuK>8_YXH)`KzW{J&e+}=^VskILP{R)Uv;HTpzJ+}w3 zo*lr9GBW`kUeAI)%VZLpzc7bMiNVnJp~2u%G@MB+SpL-*pq=3s{`^9uDohbKt0(}H zDxZ1V5;W=Ja8i#!*zrSzcCL5x+zN0`KMNEyMMBw4ha`!OT)->zBBMp~Suf$P+)3R5 zxRIZ?Y=%cYV3<+Rj^+jKo402cVZ6{}QR%$`$!Ayb69d^+z`gaAHjna>F$D>g_!hB)*)DAUqqmf9wUgetQ6u&^XuN@Hz*!89sC{IhLDgd*p(px)$`!b#W&L7~NnExIZQ*amk2d^K7Qy|(T9C(j$ zcR07Q47Tzl_11HE_?a4b(i(1X1u6|mnxk7AE3w2>vuSTr51Yc@N*#^iUyQy^3?Ovq zr+iQX^nP~~j39F~ zGSiwMT@*gL(bJQ|nlZF7@(tZAH6@AB1C?jS2JtY+Kc|XRyErT9e+3H0K0m5j2MX1Q zG*!_Ub90prT$w=b<__~6gSUE1XkW;M7ef9t?HOGN5 zSkkI)F*ixAWLP7utCSc<0MuQs<5Qp#Sj1e+Df<%?vJUb`l53p0R%kv9S5GwVmR&M3 zC{v12+@g_VX=JPkvCvo(!=RG#3#3Bb!M3iA;x{ zP8DMtvSV+ZWu>3Tb-4TZbP+2b%`H++)*7zFtKW zugs;;YMt+$oK3yzuY?CLU_!{3v&CJlBVSn+`auiV>qGpJa`l=w!-t_@p3kXji@P^u z=yl*JF~g_!k;+~?OicL;PwX@ZRhjqLy$5O7(mn`U!VPi&$UXjXxALjKLn)m!lJ7B> zbp+8{5!Jx^;mca!85Xo5*5OI=o!Jzu{xtf?6&o+LFbhe{NYMKCoe1nk`z+`6UKE#q$pgZo%v>p_n=*?V}uYnX@7 zsqEteAfNSjf5wt$N0Bcw@3$Cu+3Bnu^m(7+;r5b4Tz9SoDCn0)VVe+T+F9>dH zc?yTrObd}VBB!C-tV^G@KRae8e+LfL{#jqR3;hOfwMkG=O=gIoL*{xxo_t)WZtZz6 z2lMp%VU)dckfLURfL;>5=%*c<-193$mc{hv(|x$v;8aomU^%~M%_^T|!*uF(R~KoV z@YC&N_pH7r(?h8n+nLp5YpnBV32EY@_pI+8<3Rcd-yM1gJevTzF^Ij7YXupFijH6( z4z`rhu>RKXxM#m^=;|m5TWIaohWb`w7{5vI9Ih8J4j~|lT;}9H)^T${>5{dlHS-tK zsYm%fhK~Qd=F&KvoSN`{{WXt%Iq$x6;1G(qQ_u4c?=Z}FQlau&CDyqbG%`&}W$Rw| zreEviY;fZ#JW=v9Zt*%?a;DE(I;1|%s0f0Sg;RJ()j8a3(p6v6 zimc`p?7?%j=uyLQs$kN=yN^G%uK`*XD#(K^F)RI0n)zQZ06m`X zMs!cjyN}N(_6JKU5HsBOQ;a6IZi7uW*~R~986GxbPXYZ8!rj0@1gae4gnch4WWy%t z_FjFcJYn(|`TlJ&@7Fuxy*|+U)&d|hp^*Z8Hp@ms5?B5H*SDzQ$`zn@w$R|eN&)^$ z*+jnASZ}2YYnqBo0HdM<1lEiBydzu)n_jk)j`N$yjQ{){r-ozMuay*J;JuPh^K&1oX^@D zFz6ndB6|_#{V-Uqg30Ak+0771+bH>aaIXCajMbkg`4{|+B|i~971*v#9pBM&0SSYQ zz-Y`e@r;D$DZKG|MMXR{)HXU-PD>?!PDNElJVX8wR}Ug>Qh_}D8`}4CmBy0Z+UHas zPqU@O{9(wsQlsLPu7C!%qTBQI&$WW605|RE|x^_Q)`xH$ql14ml9v z2hVE+cLL{w6siC{D7e)mUVkIfh(~Tjs9>8QR!1gMpXNiMncj0>tTFGkBzEMn>R^n^ z;ICz4_T$|5%;}^Pud;sD0kt{B!G%!;xji9GmJw9M+i+9yP+StH3xb=wOvN;8Or>Dp zQ-#k=@QHByhYMrfoxa$Yh)FMnv` zOMt6Ld1ZMC(_Mu+vI#IVH*EhoTI69EfJkbs440r|`-8U8lzf-E2$AJ(OR^TjUV>Hw z8ZLcL_4DJ}IX991+e8}O`jekg+EI;SY*r82?rzt^OjzDli#yCkMw3P<2!@=SO_kk2 zZCjE{Vqfu#yTX3gp?x^G^_{xn z$ixK1NyRLBWUKwI;vA@OWfReU`{gE?5{)u?9Zhj$%{JPOfXsIVXn84c!pp0zHt!mH zwz@jAtZT6@ePcn4jNDWX6>!ja$EK~C^qffj$KXadi#>i5vKj8AmTMs%j(3ibp+(gc zj)s=#6ntrNDct0bvzO% z_`Dd`&YO7OE|S7#YT*u)Q2%Av2mM>}1kmzgB2jFC2snCRc5C%4!%`QSm)YwXO^OXJa$~03zAQ*EaS6uofe?8f93CrNOf7 zvPQg+)MYHU-9k{`xunux;(CD`VqAxldxxL1Ie1&{GL87p;7I|ri0@BnQHEGn)n~N4x%OI}cv6Hd_5yb#BQ*a$ zclQ8IE#dgl_~73gKlrHvx$NNdPU?S<1{l)ZRDks~-G!D>{ud9DCI^>4mk{{( zBJBYbw|7ih_m}SmIT%?m0GCwe=xks7y(~z*ZGtcZn|X z1wa4qg#K@Y{%;QbU1ir8@(Q4~@#Sx8W4ScmF{M^4C?p8ye|q-KrtC~FfW4C03~JU zN^bp&Uupzroxj6oRF69A=dJ1Z+-A+K5{kLqXEfQtKaP!IgZcZEd~5XcA#SU6T{YHD zV!RgKU=Vl3=l)oznTf{oc$w?*RaC4965#OWWns<1LTV0eNPFSfj_lK-kj%PLIz#M| zJDcV<5r}RQ8~cgr2>Gu`ci2Gq|Cm2vxau*+_cdfMVQ!V(paih zy?7fIa43FB!!<)uoUlJSALy3K=cQ7SutjzmWkTl9iCM@8#RZ(2td_sNr?k4osA8&> zSpyh%GlJc5u`*-RGSaYxj=X$!x^s^ zy@U)u?orLVZ+_W^^e!_|JuiXzQ#fGlDhS=5iq8WU@D}hGsV!)b_Y-OAj$;Vh-xO_- z6?>>cp+CIqBMaLPR;JdsXzTGxd3}{kSn$c|hBbrJzZj_y65|IrjC{brJYp@|UL2tK z=0J{?1U5DUm{LUKi?%giUgOw5T~_!G0Lm)K@#an&*vbUmS$%@A82=Ir^VhPS_wtgc zCrZJdK;`k|=~3~`mzU2`5;!vL{o27pa7@KS%PQ)_LG?yO7U54MqFqry-3taw3t z;4gKjkVTW}{QCixZO8|xc%5JGE)Qs6o}(aKO`h|i5l%%=?!ByY4pt(YpP}{?swpGw ztm(6pn)#eJ{04miq0ZHD*{oFM57D{B2GJ-CatJ2|XoTM0>}GEN6h}`0xJp~0O)m_| zOzH^tBBNXzaQhkn+zXszVC5rL2jE}_VbmB-L3hCEriPH2X}8WD_vWg;=>b|?Z*>w2 z@mLR*-&&~T6zcU-29Qc0XO~_BM2E4m5EA!O4Xo4#0jQmr(ED1+`XkSKJ)+J>$UDAb zc1w9JCu*kYVuO=mGgCcoodf=f>&h=|PA+;s=#q$G<9pMw+0VretK7>O^>^j8omIFQ zTAdybC2waaY`iLhmw1iz`4e^1mZ;wVBHBJsipR=x&v&CB#(zuzoZ-0Nhx3zWo}iC1 z@^cx04(0$TkAn>L)CPTq&;UI8y*3)}n%}2{@;PK7h_e~+N@3*Yo22uHfb(@rPD6~i zmANMG`0w%!GpAmE=q0f12d_!{ioM;k@}#N~TR$PRPY~~w`UAj@)nJ?uCBg(EGPZVk zCzO`yu4?bL3p{+2oZl1})#&n~#)YEb= zCo@y-PUemGZOadC`w*}5_4)-kx1~`$m5+j2>ew3Zs zuqZ+iFyJ}Swv4?ey8BWzReN`tYbQ*6AUGXf5HVh4zj`?5tRW~W2Uk6z(lj0 z#5~Gn)R`na@RO#XADKmN`d;bng}LrGG30n0t(29)Qmk4pi_oW75olO?<}64yBoYZN z0zP{{pex877;b$U15S0YkN5y14IGH$CZ%a>dGt|1&j9y&0R^t{$ln)>w+Wrh^AXr> zMFU44RkC~xK#UZEq$wU**&}24Vf|Ts{8P_5ZQyxiPE>US^{E83F(!{K?3RUOI-wJ$ z$xcL6GEFN8Eb7%X9L=W|*h*Pj7eBwJUb1C~R>Mts$9xuFCB{^1OUQTk^SFAC9@Vqa z`?8DAY26OX>~@ND|CU&zi2nksj!$B$OVgfKpPxCFwPd2?38Y5Cv9Lmb;>IMSAt37n z*Vp8-_kusdw08_rRiY=-_?)Xtca@&3V0YnSE zR^HRpB3_PRjSQR-);NHFnLt1AyGu>L zg;#>#uQmoIPdHj*_s=@6&iYbZ9avL;rrY|Fb4`W%wf0#)^M>z0^Zf=4_qya%+H08` zj5d924ge!05kT^6wkx0ry}utd07&~OzED>(k#QGzVlcRiinKGT5P~|BgwO>parJx6 z)bGYt7)ZbBX4Ic;zbBel$Qfgq+ZHUt>h2eB=kza@Yp~D{%@D5H0)*4H&iD+yFQ@aQ zqOQr~d;eG#&605cV1^#O_VETC$#c*tO&KG;^#(GN-*Nd)PgAMFGS|D>vRH}-<@LEs zE)!qbGp7Ymb8nM0JiBo=&3+3QroPZgrt_#%TJHTsdw)2T7y5zWrggrZnwx$R^moXw zg*Gm}Xph@#0qVlYTh`-c_Zily=RinMsIH;ig8&z9I*}{0<8OTxzH9*4vc3G~lj0q^ z(~+V4Bq-c#JD=fULtr2Lw&BQ_onA|vG#xN+P8}eX6hq5K`u(c8xHE;e7_+=m!G?=mSrP4p->dVq|Di+NT7pC~@N?g2=$ z*h|>9e!eeZac(J@Aqj~Abmx!kNEn0w-$kxzAKrg>UJ zmw?6!;W>kjZPRw5t^%3gBQSbg6sLMmPC_<}@l(^ZFA|6M#55-eRXhYGM#%eiPJa^~ zL0Ox2jl&Zp{l=%QJfR5!zP1xs3xriR2j6FyuZ0@1IPVb&?AuK1NM`iGg1oS0TgP2( zgfxdWbgc4D@hQF>DPGGZb`EMplKXtetD)S4PBd%3q8hO=To3iH7icrgt>}Y)iJ&1 z&n~vj(R_)$zNhDoqkOwtwX)4eLLkQu&Zp?ndqTd;9Z1;qMAp}jM_YBRi}HpS`AI_D zF*=G>V&(Uz>CbpPVGoXNA#`cBNG&yBsY{S1`Lh4RegegObE6 z2!Eht+AXRT{bmWCE{K5Shdr=}U<|_GIzGp?*CAX5ZRszQ)eJ_2ckPn%<$l$!+lyje z*)4g`e~-K&w8ORtj9QhUNq?2h2qk69P;RUoRNEV(A}Lz52RNHH&P;Xhq{Lz4?R?9b zVD}(dxK&16b;563+!pJ=USx%F)%a?&ZVeh)M2A#ipyDPTQz^w*0HeG0F{LY|!SU5B z>|3VUHyI6P-`?nE-1`JjS?^A+M(?S*@Q!_=r7dU@rp!6?c_TqwJ$CLCz+9 ze*&@IWjzEdODt{-*kFqGL)ZS?v3^(YHg8@A=Gd%~- zJQ#tcd!E0a0Du<2L!Eau9IOivWQI`qqL!*|@Lw`;wXkH_dH|Zk*Bd?YHPNYc3xhDY!Nfqq$Coyx#))=* zpIl%TkcFiL*1uIfXz$m4V{-5MU>TcZSar&5@YCpvbEqa8%o*@aMPZsdu6;$U=(dxt zEc0Bn#m}isu~N|LrSmxpfz!$x)IT2d-2_6)eJFmex@Gd&pTw4SO6$d3k0*3U2ph`z zFs}|P`HEY5zJ}sgRYH}+;d}cftl*tCB$>Eu06GN^u3jqq7}E!r9+F*PnD84}`7BG^ zFhw)z)U~&D#_auN2jLQEJ!;!cz#&tb7m^`N7D|VwsoKb#f=Ly!i#ZN2_38_A3`U0L zQ}VM*>!cvxeNybgHH67ui=qy5#-YKxF&c1c?j)Grw4uAM2=s!xNl`Ahtf*s@IWtA9 zEQ1;)Z??S3iM`!JZ`9nkWpRI1Y9x=gDDXtOHFmU+4jI`qnhu3xrHl9I5#e%Iq3YZd}_o zi9YZ^6Wd3|XN!9o$2$EkU@A?vnZ=o;v{N2>m*o>uhfpJF*kE2%7?!5kHM&L625WQl z1j3nE-nFE(aXnp#Mc;TD)Kdmi?tN5h$F`z#1i%?m;Cy{$G>7h-Z_kTCD22`!ws zw@>@Y@;t%YJe2j0!#%X5d8cJ^x?1WK;}>axfdBc~)*;m3kgeHOzWd`Q^WbNq$emN+ zDyX+84GNQgcthUkwIzTw0A}uV`6wof)W$9>{NkF?%Nx*2wq44k+B>hVsj}5+z6^cm zy=#?0bN|yF_R|uBE5y}N8pbhSQPlCZIes;)ll@eQVk_> zrW;UcyUS}nA*F_~o-0v!aVw;qLzl!N^WbE+_dmpMR=m!~<>!ddjC_>1LFmyF2nkyz z*YRT6g!Z`@L*p=ghjo?78N6k&D(Y(y5OYOs%~f>#OVnGzN+R^)x73Nolq5STFH61d zoBZ7K`{r4{`fXLlIxCAU<1e(tT|@4PHgK1%O37b7XpenNLb-BQ%}B&HL|S^yDi=`+ zh?dDGgnC@Eh{@Tg^$3Kl5(cHh*cKTZ9~jr;td@~&`IZt>L^`3FOCJGc1Z|3^*T-gL zhM2;t6b=ce@%=DFl$tN<5TzWTyG zCYYJ>hO81(`28uOn=aO`shW7p5oRNTqc za8r9n%MSQjl2GL5501;yLiG_|jsn8(DxfLv zo`|hCY(|Fb<9<^TO1nxrb`Tb?we|1(@I2FwkQD~Y#yjDguw6<|Wk4L$Ee=$o{O~ao zHlX;9f0`~Z}nuHzzZ zb1;VosVns?9O^!!qSit?f1D-JfbnPRMIU32y?si`DNuBr+BvoElaN`c779sJK z`EdF8&29Aw$YIU)YPG<9E;ienTLgQpz+xldE{{<|1mY}gVMnSv=(jm)3ic8XsB0?Aj_&iBV-bSLoHin4smiG3N^Fo6aFz?~Z$EFk2 z5BZgDLQcun$hK^i`c6)%&7*E~8NI@S1&-vzyqK5gxqLpdb_^>Lw!`SzLI#yFYl&Q2 zPnTJADf*Ey7$=dc;}ozI7^l7x6WrFA=Xp1;tW$1Bt?pa2sFzh~FpAPkn6SL>y}B?m1JIeBYwb~&g?CPq~9w^S99sy#XyD7AwOtv%g;^cb9knbB3%#A&Z2~=4YBHwVf%FxyyJOOTOee z_=CWUCR}}BfQ?W#)h1-@eqPYW82?dJ=WZ9xSIr4QF^Lo5Kw>%hHB^DBuu<}O9jdN3 zt3>S9%k%TlFD_n!u@zMUk0?UlNO@Qj6R6F#Tkv^4e_HyoG?v?|EEp)Uh zgH=$vj<>@cEboVAy>&TKyfq^G$?eo&_&B$;i@Mu3iE_eO%TUbC{hB_Ps{TrYsi-lN zRzF0ny7al?wXk6Xm$4fT+|5BI>XxTvx>*d85Xuc`i>k&Wr?!&qUy8Z~x>C#(Rt^6791n7DAFl*!hBJk%aY4Dhc-;u zH6pF5HN}J85e#%8xr&gsAmWlWdf+pKydTSWsc4XYlh2tSe#UK_AC@W zq-{zRSpID{3-LE_*S>&D-nWM7Ej$bEO0LxjK3vbKzum#K65ekg<|E$V@DWbtg3HcJ ztn0UZl^bXjkxH){;U@<3(Ap6T=brV=x9T3IcW1d;si52&r;*#&Z3&=KofjBZkL=h$ zC^abcl#SYt90R+`I;_&ocL$H#1KVcDU#iENT0Vv|6L_n)%9#Y$jdM!^W%uQxje*)b z4_MljE1W`g(?AO znkneVQ%cE0W{2Ldf`y$jC4dU1Lj4oWf}O~5`jC|o^@)Zdw|IHX8W9x^IRgF*suq85 z*sBNaYH7!EzrlJ5XoGw^f@5!575on`C4ksyA2i+(mufu^1BdJ#oqv;B{~?O6_xyH2 zI&mDBKNl*+Oyd;Y71&|8*jq<6TZawUy}34`qA3kNOH*f+9}b>jU8d3%iI4llZzrnC z5y`c)FgKuAc#U@dLS__LpWQuo8(W7ugx6|c;`|f{L(*+8VEYwFQ}TyugN;J+a+TUG z=a{-Gcr<-{k2oIAc36wrsBs5FDLKnkEfGWZY;%);_xQjoS(TVsWy{L$A!;7!tW#JwS~g^ ziG1T+PC1yC-vKWo_Ib9S3MNiD{H52=&7XK*jOscE?1!c!O@@@8CY4Hmi4F8v45gsi z%xa&(*u)Y73(Z{tL-(y1sZMs69f#IjK}`z(FeKVV+~Gfim#kl!>BJe=k=H_Gk!h#k z#$zkekNM@_eE@ch+g)D!;du=Qp+7q}{PLb75V#^m37&2a*7uz%MG+Q%zidL!^@1=B z+>|qz1ZFjo_$qs(>hPT|%KYpJFqq4qT?R+IUw6K^b&Q88F=2!9KHQ1I@^##9-`_qL zCrIQfrL<46C*oMVSCYGxMJ&`~;jJ5K44Hnpn3*WbHfuj=l2|mhR5!UScG!pdVa_Rb z!QSzn@G}B!Gnu)?o77sNn{dk+k-@8*W6q;gCP>IL=Qb|*#rY6xtG6VD7b(7<1KKGSO}1&MJDcuuBnCk_p5V+ zuR|tIpSe~r4Qq+PcH!VtL}1w%p;xV!!)BDcyE2YH__*|AEin5f%G*YC#%?yw&s6B= za40=l%g4JgjznF2vtK{c{e8`cowz>RgvoWRfbvGF`AFyU?6J0?tK9y<(b20{4-G&6 zjo%!s;#Jz0zOqh3xm%Mqn8H;&-Ad`k4)c%`;?}rUSFFrwU`Nl)F1M5D?{0cKnW#Qo zyZC)>!x)n%4YhS&kMP$=e*1H^o|3i*(bjEb(0due(;_XD3S=P7P~);MEHMz7S06aa z`Kl$yQIoX;_Ahy2BI}PvJxFzVODtZ6TKDsbc{REcWq6}Mse*0yH>H(@SP@4p-Sf*{CZewuXmX#Wl%V2R zwB4Ck*`Fu(1envQ;Ro$aVu-F5p0c&kHT)Dq9zbF!jsCFStTLH&_j0<^%0i(S1K1OO zSJKb>Q($ONG+VwQ&su`bL8Bg&I_AUn*7N#O!@2Di*-EUdq z*1L*ph~S=XS>q@R+3HOaO5u>!pAiF?m3YP8u_>qBF-F6zx47Sv$C$He=fHQuE*Al*5Ft{+jW&zu zkbhFzh0CRebsGpu=(XG@pEG``K=TgZO0#WeBOa`5UZValdxQ}-fB0VAP2g;#I$pgY zI>KG7DZ;OKbxbSkuoa2M6x5wUyE<*r^1~LLb?cWgIVY&3{hu&1cR?tSjR=AAv=~XN zlel(yS0{wzw*^=x`T&CGa=zSmrU4Nx%%5R>9%b~ERNEO|BgyXIOAS>Y$X^qtv%{xj47ze6V7QLOt|~R^~oVcjYh11RUYp8QQ;W0R0>3i!?g>!@pJ&?6hJtlM) zbY~v~J(w3Gmj*q!jwW>g(EIRmK|7*~vQ38Ab|`I2B&3yBKzw7A;_LGRhb=ewnHSnA zmLP)rMKiFIn4I9O3xo}$X1xgYs9qjV$I^7^%z>4Q~DVm_}S>&;!` zerdNOYv-rujEeR@sk9<|LEj^N*jWOPFA;P<1|jGrb>Q}1PB`=6U7RwI9hG-eU@Lk2 zG@Xf2@ZhQ0yrzus5IMsJY@B%nwfEcxZzM#K%>%xO5UnFTp24Z@Bx5RD*Y@3z5HeEo zWKMh=Po&k#_-G&`+cSx$!)&qGF^0%eOrN7BZQdQtkfqZ^d8M-EhLX07Qsth0J9OZkbv;44mCb%|U=E!|dQ?vS z8PCw(nP;<^GO5=~hL;yYT;OMo&$IK_P%I1+-{t95`DKxnn$Su@s|0GMz z6EXk|Pvw;x_qreaZcSeI9hx7B!U_!p&58PAmdYP;cMrN%K~sKb`SfBIk%?<&!mPgk z%i4)$1)IZSN16vIJ+xxVr3CWovgGW; zsY4W-0vXCCx&FiHvg^Z4 z4B%AAL%}hke1=f_P+l_tM<_D?dszBrevg*Q&v;7j2q#4Fr~5br+>GUaQ6o^_TT#R@ zQ)aQ7Acc;IVn-vUu<`cstFc6bUvz!=sj4K;n}n%B07d1L^Jq2%rOT|YBsiVqj}I)H zGXYKQCZ$^9FVln2f3OTyoyB$^M91apVvQWmf(G}dhJ^MI)`ZT1Dz#xg24TH)DcQsW zKbv__tf?_l50;#wwEd$MIsD0_dgP+HHf8zYd%^`{{$^Ag#GyJVAoD#3bTxVw#YAA; zDFSUyqx2ecJB;{Cs{TEeWpJnYOn9N(%T(dG0-?6>SM*^H)RaE#oy7)%DOlHi_l+Ae z=pb~4;p-Cb=-nM}uL$%u#4c0eDEwj(WeKg$EPGNy&`h??jP=-qxoXM50KLFy@T- zrEax!xIwyF9pt=YwnKy;bY}to)EyLIK9J_eIw--ux8FUP@uZ`muv+p~|Ahby3x`W+ zpvJ5DG8wHa#U-{Y!Od4nXn;Jloz>3{Ar58@!d9BxaNTZpQ5w+R!~6-blC(l|~p9@_wUNGVy)CEcrj z>Hk0e&pTSE`)6nRV52~}k}>}&yX9ktT7}1c6b@C60~BU94#}rRPKK`z#ler*!zR_2 zABoLYM+9Wx^<5^Bs6XgNxU=UGAZy4a2ueVkL+t%5vKaqT)nX~0l}z=Y)oizn?GEPd z2@jNNw+$#d`z8td)cDBU=xDGchhZtxGW5Po;M3Do_DRsgOx#{mBnUyl&vQmGDXu zd!l)Fxq*qnKJ_ie{3f=dz$tt<7~wS5UZY$jl>#Y8~-+}Qa|t_(ZyKe~AgZ48lFpTnIpD^d!t2*Yd?s8iU66T!8ryc&MZNZ>Yicq4bmj6CAY{1r8n9S*D zeu0mu7A8UuHw73N)I6(h&0ZHH5wYdXss&VeyT+mYNJAf(=t!~tF_E!B=(NAt1(zBu z7pLJmg9*}$cf9@}&6k&G#q%lfyi_OS<;zpaMXSCe^QdLFJg(wwjFc$I4L`@I44_qv ztxn5vphtd#s{)iFm`Z>T6|VR3Cc&a%v}H!4OlN@Kri_lEY)L-xcu70)*IY;1qeZXW zjne{W3hTNlFeOg|`Nw1S+_hp%ECm@}0YJdb?vy3~DFmR1kt@-WORlSBJQ~v_K*EY$ zdlH^#PNGOj4VYR*7T?#G9o|-?o5cjbKJV|}yPP>7_x?zV$=6H# zmk#WIj`glIIp-a{Z)Rop;0qAH{PBLEV7ntUBvccrWRgvalx|tmfKE_DZtDTPcgXc= zbtUE5g4*TDQ;C6jp}Xg|0l7~TBIKsZ$`hv*oV1wd-{L)FC3fhJ>fe>tobc^tI3KrO9XpTJ3@+OM>DlGHrTeJP>g94=X52TaCzs;- zhPKA33f)B+j(>FSzn7gt6}|bFDV1mtbrlC%_j^_M2U~G@iYFv0#b$_>w%6u0&byk` zX7fn%6^X|f=i376#p)&Wt~Q1^8%jq*c8lh7pT54lkw@R*N23yp;TReJLums`Uhb2sc+zJ@ek`U&5UCR_-2UF*F(XwK%KiBJqaN@s%o^<~oe6UMW~FqDVe*gU58-xmz&fdkFd6&+ z&MMnn1yAYxZqfAW%x)8(T=rImFvZ+`RLZ_`ZAFM#_BUM;nw_kWrZ?&HyT*spYdTE~ zC04`}jJ5sBB|ZMBP;B5<6ld|j0|QPwRrEJ#>)(^syXoxrZ)laW{N(Tj=({rzZ^T2R zVu2_*vJh93XL(M#@9Jok<+Z19_U5(o!HE22HmU^TM<$YT2U*&qYwS1jH)8VNJ}Tyr*Q=V}F~_F(n+QQ^k(D+4fFO`?o& zfVNG0gF?s{jEslG=Q}t>P;Grl!+$c&@x=5k1Rasi^8azX7vzxRU5XP=oi7PDt$rcY zXZf(=V>87TLG)Uo4%qoir>iRRNdoCP&P=P-%pu=-$9-^<&&rRdSh_gsptK?2Li|@U z0#S{Kkx9W#nWi@ntsDAIKa{H;&|6KPKNgze3z^fK3yxC z%uGrkt(dq(#-V?u81)jj)*BaMpz-e`!U(y^m!zz79)FgDapmDquV!l>a>OiAKgxf%0b50)#`-!%_n|$*0s;~X0>nMZ(OrE zg|YKm{xc|m{PZ7w0cKC|=wkWv=+>i|)a`siLdvhqC`6octt+)}ccp4lzH)h%FL>j5 z-M?+&uif^?9ht1{L-||W!2bu&z95lEaG-ZYPzg6E{Mkbe;m*c9FX?6M@OgR z`9kTu!qu5p(x0OR*=<>z^dDLjHZ$c(MU$3;hQ1HmYQAfaJ1Q0Sb~Gah|GmM_5X}<( zSYWZAv$-uv%vV~)agYp4=r30KFF8454P>%p#*s2ZxLpIO&MS#|Lz+R@niW2I-&Ga3 z3I4Ynq}$&@w_++e2(ZfW4c@z59Y>$xFls4z5>#ka1*G||G^+eQ%%PxB=#RSlGh~1H zE{UbyF6okxyST6Up1{fA26<2>NrBjkmqaWW@qZV_f^hvt#F(`&iZp3kSP-81-;c^yzY5s+Ml6B&t5Yiu!)R~oM7j*69Ct?#rPpZr z5}!BD(t4ZzR|8&Q;6WK8(MCK&C&*|sit(zmuxm1Z?HN>=K(Cu0q30o~ZZ)@NJ<31t zfd4;({QeivtvE{B>^j_k4Y&Utiv-{j=g%lo|2IRVnZPEt$hD16=YJQ14z@lX&tl~N zH^Fmp!8UrTX_biSzc;}T9&Doo{rUl3^xx|V2k$Y~%kLxq$veTX+hM?wNkq)B{&y8c zl0c+Wd(0#CZ-W8W*%}lG5I`rZ{@*D2PeIf{I#;RRAokpUKc{{G7lOx_|6j=vv4*^q zmEDbu|8Lp51IL>|m`}FOFX^qwa7ol?C=n0Cibi$X2kHYuqp`Nsf#^Yiq zApj0ZI$JC_S9+YnX{B*jc5i|zuTqu+1av?aZk<5yBV^(FNv?f*A$`DO8_Q&9o6sW5 zOJ&NdiC8q4z#KosTt)#G-ws46W?<1KpnUl3{R&bG@Sujd%kJH4fu>|eJ$7lpdb(J) zti;+ugTC>-sqF6zXnHDp4tN=`b4*2G)NayCWY*0{4T-mByxgCOxxL)a9kP6mcpVJt ziQzHq-QPk$fB~5CE$7E~7p4Zq0*)IJbGR2<16bUv?^CoEY@Ea?`GD*B!wlD-P*{++ z!tyU~1sb~2`RmnaGo~j}6}Akkq##|{vw*>|oZ>xs`S0FpZ#Iqi5Y1>n(LBr@v4JQO z0mI$s;QOy{J$s{^0Ru#n>Yq5eKG8ZyT>NG%TdC~eTHvOC%hK5^Lxu)TN1la7#7hQ= zV;@Q7Iq;36QA`A67X#BU4P1CB4S90tfzz)yn^BbKAfCbL*BUwB{lWX&V{HCS7rOhC zPL(|)cf%MpIDnbQ8f)2CDqT(y#qjhs#8*=a5lGBdyMFR}a` zhTPt8rhHpo-NaOnQ&wGvo0L&fEjQ|p<7*0YW*pf5K_otg+>8Lur3rXJV7*YXTlgLW ztmYe@C?3JUsrT<#HP0|H9DJL?oUIU;$%V0eI9UW-wWC5n;niM7BF8J#xq-O zeb~DtWZDFwO)>Dgb~th@GLDh|`C$tQk%^x#BBSnA3HE!S8n3ZbBw#m_Zpyxs5g``# z(KGPfl-Dk_$asN@e}Vd?B5BdyYH+?#g509RLp6WY!EJFqfbNw&({#F01suniTDh$i zoo0DjJgtV@D7{l6c7G=Hubs1J&nTB1qG?xARH=OtY(u*VWFu3iCJ9%<1Te z2{d-pFE$1Z7HU?p+ys$Bh@9i*(r4>jhmsF!C=kuy{B7}+B*SqNK%N6vepMP!b*NOC z^d#@UDZxHMX3^39&aIs&fH-G};e)3n6-kt^PTBWsoBz%p8*F@-+kU?&QmXoaE%4;U zr~8^6#yIoi_r8piwj_`@MmIK12uD(Sq*NzIus5@<%m>V)j%<6K{rt`@%)+T$x)f5i z3$SS79b{D%H0f#Lxok9&awMUz1izJ^X#UAA(eS3o%;(#j+MrW3l9|o;QoV}zZZduJ z7r(Z$g0S*>(S{Tb$uic#9q-ERdd1hTjq}V|vWbjzATdF{ZX%L8HYdT{hQ@=<5TmjU z|DHMPO%wr3=KJ{#2l3U4wqqTukCUje!iGXL!g{b9R)1385BMfABh$5Rgjr8KX3vm& z$eFSD2Zo{zG@N0Hcfa_!hO3C7;O~XW*u&kJT@_ESEwU0Zmelz^MGY}7UeQ>gEQ`o(JyNzBe99YuH7$Fut*uZML(vOo38aH8Cfu<^e7 zEi}*6E#Z6b1GP6y&@#Y~+o|>b{Oz)xU_`Zcjl~QG^!PVx>yM*pe50Ymz$uI-`AF;& zk<$lSm_^@?B? zaB#C1!}w|jrmOvx6x%k=0O8w+rI=NBIK{vTEF(c%nA*D5d8y-vSB0X9-?jU1{{mBw zk>^x4j&(9;At5m)J_p^}z1C&e>`Z=F6+Xw~uMA6Xi>0BeiQIX~B4y1d)9+e`DEVJ= zUY_FEhuq0@Mk7UR&}^&KaDa%bdVaVDZb7xYs)XD%+bjV~m;i6%R^JcbyKH`T;v%`0 zF-Nh6B;3zhYh2%U*D8D^rjZB{=4#1Ef54Ye`)%x zp|^5!8Lxv&?2^q@drj4KBE7cY5*b=+Ro7!z2?GLV--xl|10LoBSeH2W4`*$31sbD0 zw+Ewa&&6pvsEk?@EC*$hG%}pYT1oKB_Gr%Jt2v2Xe+8w975}A965gus!E- zdjo!bXTI|5V~-4=QdE9?5%vwxExnQ-44|U7wZ2*waht$}hZA`K1feA0Q=kcg z-M4=ASJ{kKSw9Kr`yf3TNiVMB2p9tv6awIzp|dy*DeIj{91(*=E*aER-0#NcSv};A zIerMhQR1gRjZw-M0802@q-pfl{jeWh;1Lr$g)`fj4r%@LXz(Sci>K_wVDg^cvBdwP zOo!d4Yc)GU_6p9mb~jh&7L$wqcR>amXem@Y9tPmj76Jeazs^~#(}eb(<8pYf<1PQ$ zg?XgdqUhfM)<-0ymct75*zu4Z6`W=JT%+V(pX+;OR|u5!(ercO9xKXUCAvnt%>@QS z88m9iUlURRKnEH)aS|~`EH-2z+Hzya^#wjV%`$&+QXd%t;JE`qq~cuzfO8y;s)#Cg z8lqvpgJ2D?-GjsA)P)sg@1h~lRW`cq_9|5f1Lh%n3f-O@_`ad+GftXz^EbU+EaaU# zrowBxIeeM3mb)SfKY^)Xz4MG!CN0CsqEgeZcg;gp%g+tra|(uJMw0K=tq`RG2Wakc z-BGWW&&yw5W)_hl0zbFP92RpQ6G*(|136syRK{B=sTR`={zzwx6Xx|7vD%gN`rUk* zZhO2no=AC$4Wmg{SNxW6PeDf{L(B0X$g+`WXZmC4TLFnsU&*8GgwR1R$B|V{?~xGC z>HSKlLWdc6-&t5ZmhY5$8``kqD578dH^9aQF{ zu*s)t8RE2}ROLGMFv{gGOkcj}2@JwP*o%SpAB6WYsLAU)Rs=FxWinet3TuwO3JnLe zqxe?`rg;5Q1P?W2>t_&|9Xcd6<6zU8-l3N)4rRe$n9Md!IRSs2Y?8nmMpnTlN@zZ@ zzJ%mX6|ex^VyJ|jVMWBpIl;~to1fJ9cZ2D7zN<)$|KvPFh z|B7>=#^JNcjl85VsFfSyvGYen`+vd44f}VM5VcZ+!`4fd%IN$SDy2yUjZ_IK zQ_MshHFtl@rnXP00q0;Oxwh|ITrF(py%!8^5kd}wJK4OCe=qB53}}6uG^JTn@(imy z!+iCI7yNKwNpTA~Mpi}oS(4eL@);Z`y^LFL_wzzh`PCu6TZox4d6$ultvQ!T3K)Z% z)(w|zvS_m12?HMk53W>W-TsLpsci@cZ=M9u!zTlpW-}PXIsC1lI@&? z^<;#AA1+&?3EhHDMklxKORV)Ee8H@e;CF4lIfd3%VC<^dk#wlqr8D`P1-pAId5?} z+!Nbs+h4K5_~|TLznp$n;^(^pI860*q>6y98c4V_4-o>XiZ4z!whi|hV#QFDWYAHY zH1Cxnb4`4=tdhtm<11JEM*D^1%Qrm+H1nv_XrFB4H`F^FS;2Dyh4b$z^y`%AW5f(T zJ zn2&7xVA6Cv_gk0&XkR6^J)8$b?u(v6II2Dd(m-a=v*MRSH*(SJnas5Ng*2yVy0l`0 z%KW2l;?A=NbfSgEPqVZ*LS)-?#(?u7|7-Ng$?hD_`RG-vCts#%iFT$e$S&94`m0H^ z#m3Jp%^L!NCgz&AHM)!8lNiNz6J>9cmrb2xXqbgZgY$%`!+U9G<^BR1w4cICC+8>E z;V)#TP&*n%ro>O_*%iE_QY0$qm1Mra=eatau=XNX}-y=S)8bJ{%E-~)j(JG>ZrzeT4x zqd_%Vg(Wx1y$R0I<1e0NHwQ-q79?g4=2&G0^sL6}E;UWe?4>JwP2KSFJ<^nRIn+Bg zXi_A`sa)2QntG)f%^I-nBLD7q`zR<)oOm)wv}v8zl8jBL!i9 zj#XX);;?=~%C}uxzXOTVvYaW1JW<>P=k^Wfh)FCLILmTcz`bxBGoN5dIG3X3qCJ(*i*m>JB;tc zFdiF)rpM`b1bVoTb<92e><+^sFMAt<*K@1SJ!HQzRgf`%5DG7T@ue6(UmaH8A2YT+ zuJ&tteD3?MUXAWV5K_jv4|nbLV`kO4K_3>1)cqg?sg^bG>@H-o4=cd1M^{*@@QtQd zlv7?)ua#1Y^orNi`7u8q8dF%?f2uHkd0bg-AH!GRPMO4UH6Vd#& zm^C|{nOjh~1wGSEzWLy-R235bilo~}rexQ^(fa8OHhIhIgKkQ#{m+Q^z-R7Ss#SpQ zjy{^;G=k8)N(KrL7n^A&8$r?Z^Mit;QD2cy)7CWVkZn3|fcnsl z03w}XDGrC38uzq7Z-Pqj9w5{k#SlWT_zCh@(Y06sr$k(1D&LW+toz2~lnY{Naq|Lw zHQz{2xx{+LjQV1q4dh@6C{brICSd_hhEfHStk_Z0z{>yAV(2i!s6nKiU#YQZYHP#n zGa~&TAJV?@OCQsQa8-hN-yBr6B&qQtP&gSz?}@s1RqL$^^RQz(p*8!g26qf+Kb9b)i-x73%uort>tQ`B7YuV0MV z;9_S8(I-}`YmO|(or`rNnti=R9Kxn>nad&lx#$@&DZ>>KDE7EctHTgV`vZQVb&%ZP zCr#O`%cX6n*-8PmF(SSd-Tr2~0aR+(OUiKWlmIR6q43mu9Em6B*#6#;8$lI|`}hu% z!Zed>r+^tSE5vwXv>=9%QF5|B^_&W>i-JxuCqzYcsRiI79U%#L{hXPMTeHUAf<4rmV-2c&YqL%t z$xg(4x~Qj$;rCcg`;P-$*Q(}pb3<^n(r=okRa;djdL!pXa{_z^-?86%Dz@=u-X~^Q z5{s|4%D|}60%@)G`Mo~B>PR+&2g!^P*A?6+9?dX6<0Y7s$4=V25+T6G8Ep234-&Cy ze|;N)%6=YU?Psg4)t9-yqy&NRlME3-uEgeQJ`}jE69_byZkfVKmZyo5_3*^lq6>UW zV-OWz2SNFfHgeD!rXQ=Jtx?Z!+n2fvJ*Zz%^(Ob?(0a-iB9VMlvmd z7RMf>6R#hMB$O$ISKJ=d@6CDvscGgjZ6t8VF5bXo{xVY zla%ovek7#hn(@GrMyeI9o%ntTtnrQe@z&Q2Yx|VS(9_j)zc&V)nP7^(T`If{BV?SE z>AXzQk3nYy+nwx2Y+?Hn+LAt!$TFr+8K$}%a`u9Iz3jf^f@C3yJbPB_@Z#S+<$l!w zZGiiV=o)XHt}^d{@uU8}=%|nU=I0jT>5#VkWv_*C^x55aHKl+F?*m#9=l2g}lt@VP zR1?u&`@aKeaKMbQsfif)JDuYlz?6}UgJ#xoi0CNW0?$a z!hqf1ucP`4n%O@HCR(xGUKR!g^onwZxJX}EbpUR<*)xKJfzGuE(QQ+|wnl7nIHqQr zh>BaU0423U=a+kM&a`{Agp-e{)ylESH#_|Ra~Y|;=d@jnK0Nhv_u4G|4@R-H9fmPF zT=JR?K<4e+vw(JX_Zs&QFS+;Jm-JD!=feQiRL-}M-ED?|L#lT0pcUH3Xl z400qgJolOmo1!N7_(s2xLi2~U{#$djhQXyV#r08#x!^rk6*bN=EFe;*-`mV(Q< zor_<)LLf|+vR1*smKJAOjp#_0Kq`N1toajonNo<5gY42!j@HZw(w>(?0+_$abSsR(9vZwBf zd2T3AWx-fUxl6p~s@?;{Gh8Xhr$&b&Dr<@=V*Yk32%H}7;yUOYYY@qR4 ztzmoP4mvUG!aV@uP1=_>YSd)cOv`@5lFUiP>ije=RdBH=>zms5eDL4x`RCjTt+Ks4 zG6(qCo?9~$W|9eaRe3kH)P?n#jmP$GU?3Wa^p|j5Equ2@?el<~%QcCR zL8R^Rp?i!M;^J_odW$aI_?Z-Lz13-Ej&37=OP%*$U%B!vGLW<#do)x0kCDO;L@Qxr z~5+`tHiqMO60S-~b2;2=Y7;Wm?^cH83_-2+b_bNOA_dM=S)H>IO(`N!MZ2JCz6!d{Ubw4b1Bu2IQFDXXJ4_bu(_a= zxrpN@@f4y^>X;qddw*sen+?PV2`_5pP zVMcYMMX$*#7L2GV(Zj9#-+XeUwbTWLp~cF{IYfj5PS&%^QX~yYMM-(RmOL+udojLG zsKH5|`}<|7d2S$#1nWl)rd7u4-po>)(llLz+nL19NFO>Lvr#hctIp>swcRJp@EDeY zux$S}`B(aizfWFl0T}rqEk7L59$H`e9`!0)K=bQm&`CL%kcmD;Q`E*=NghA~<(l4Y zsp8$VtGAa`2`G9#3aOp(l&&J;6zBxfWh2^9yq~Bm8rLbIC;XDUanC$u1=(n4F$ToY z3XR?;+anIY2a^TX*DG|%hBM?F^VE`~BOdSvc;j7iM3S*me|<94aL&>u?{Y^Aa2&f{!GmHI9S0=PH@&r9WW z33p_E1s0W!S6|WUsImxX?Rhd7k5gd!N0bAvex0((=O;5Xw2WKm7GopJ-{cEJOg;nZ z>C&TKFe+M~|8Nir92A>47H=k>PhWE;lYsID0{@Vq^G0)AylbO9$UPGXf}UVCn2VIH z+>P^n!k}L=Zz7e+`xh(x`sIj7JtDi6^MeqwB_k5K-Yezzb|`3`XJ9|U4V(s_c6gk14IFlHeR7aq_; z9JB4AgQM(=5xdhLAFL*Q{;YgM2-{k$zWy%mG>Bf~8Txaf-+YmX))b$z=rhCs&fd$n z9VF_u-}x7#W=Wy8G<#<M-LMboRriJxterUP@+Sjc?-!l-^VEGm9td@+{vZ8;E%|m>ol@L!4Z%|GI~X?SVkPs%Tuz2_3x zDHa#U9RD(CjzK5eialH6F`q8rw%`lNwcFo+qcX*P^IOKsHvf|NiY>!wo^_#w<9-T) z>B*@`qx=qK>$Jsl$sV3i0Hi$V;LRkhPyPy0Z#A=`!@57qEf^~cf7rI=bQAFWa4N0( zPtx(W@8V!t>HD$a#8-NWA2%_luh*<5EG6x5soO0KpW4tbc>DR6zR-FV%9fcBfPX-M zP3k^8=K;HrUY`EiB%v0;(Zs$lV7eES?r}p7^_~7;wVQWYciAX)__q#kpm&}{z_H&eFoZ|iizKc%MKv&uY^`$yQaXE())uP;?u@!a^yaxMA9NH&X+am8$t zA%r2Jn66KTzdPZ24R}XImTQpPqkzYz!&J^yWAHR2z4U8VcS70+xwcP?0Zg#WE0|2T^4aYJ|Q2iUJcGQU%L`x|1>Pm^~K4Ah7R{rN%UU zGOq!<8XB(L;l+;e_u;ZXxJ$2%690FKUv3+gNjnU7-OMI&35zY zp?I_giH7Y%Rh(Ab!uJD$5b$NDY|QoXDb-r1(?Sd8;U!b+R^F7HZS^wJEnM*l`RN4i zBps0aG!VNOcgG_77KR{IU3X=7h%R?ZleqRef40*XXokMUW=B@-=rRxE79$V=1>Z%A z9<7a=doOx1);K&5Cy&E8KiX1b(1tu+QlM&lT+6LgIaBNO>fE-7C~shwT|ttCc+fK% zByMZ_B>17#ef-~b@^z9P=|j2Kbw~WdD@*nLDtE@qdvx9aaXYpPD-tRsFKFWygy?WA zEfPjeypXfX3J}(euG2ZLZb6&zFqg*0<<}2cB6E@?4GFkLf?oIb;Mo|PhH(KOkeX4J zCtN>YPviNIO5V-@Eo4{Jba0<6hWOn*IQ|?>156_>l{VZ*-uvvs^eow;_E)dhn=esl zm3B+XKGx>QWcIWGOB-Qr8`c3Ey2KsRvDg8(E;1}V!xob=`BYR~N7&6nsPCZ!kpy88 z5Pik%z9*#w2$W(2@5XKXZ`^V|Rk+|}JaWb)MA z$cnW;@~2edtn4EDq+df2&a;}Snkl{9gP0BFUri<9A5O45*?5(qmC8UhtfdK@7yVuf zlXd`^kd4DM=o8JybVW(dXwAOf66q0EeD`Vux>uwKYb5tJFO0K<@Yr3DBMu7(u3DN zA1Xw@u2Bo39qzXUu4|x?vh$DNkA)GvufkJ1%I>H@juzzg^ShCox&)(=sl@un2>2I@ ze89-RkM-n=#MzrI3LbqjXpq-s-=)Xg!NJm&1@!HWvd6L}nMuYIKl)$OH}0g({tUgp zlG@o}p+x;|E8 zV6Ht$ptdNlnF;fI$|K1}?N^eSsk5xyP%h?=7?x*t5nzbpQyQ*?GBU2{yoUcUbe0Ek zFqJl+9_ij0ZQCyRMWX=D9}ZR|^S4H`!y!`3C0Mq)cytI6Q<7856rXn@&Jm`}HE(P2 z-?$v5RX;zb%Fje$1*wwaUOlGtG$-E}&~XYnxjsnq5I}XT+Kz$!*IX@}oR9T4*OFet zJCz@zQ?b-qw7}#7n$JmXU@^|Bx8bq~D?%K49Y}*4@`vbmBt48Av&;)LWlz?s2#RHWt%F3-{9MAM;gA&k^#3?_{z>)*r6@m{{7U7rfn)UlB2l4!E4wt%vDPhg=4{6K8fC*{m|D~c##@Eo)!~}R z&F8KTTFhyJ;V+Z=+H%5jtS!S{NBoYJ6}n5@TW{=b0d`L=p}&5R4zGI$DB?MFw+_ea znIW+MtSV5V+IfR$K>x)&L8`3BoO<+-;#n9fdCT!0jsgb26y+qipYv~95CUsqV6#O> z%<{?Gnv5ZUZ29SYw_*?YjZ1^MB#1m%SGBcN?2H;HPB(7&_#KtBZ5T_4C!aIz%Ex2s zsSMD7JzOz`#u=Bq0BtRN!WjQsLE{R!9cu$J|m(cqE c|3Wu1HGk-PR)jke?*V@*&ovdw + digestAlg="http://www.w3.org/2001/04/xmlenc#sha256"/> ``` The following attributes are used by the Rackspace identity provider. Add the following options to the @@ -274,6 +282,81 @@ used by the SAML2 identity provider. ``` + === "Keycloak" + + The following attributes are an example of attributes exported through keycloak. + The attributes name provided by keycloak depend on the backend configuration inside keycloak and how + other identity provider are imported and mapped. Add the following options to the + `attribute-map.xml` to have it compatible with the provided `saml-mapping.json` which will be used + within keystone. + + ``` xml + + + + + + + ``` + + The most simplest form of user mapping with Keycloak is to pass-through attributes and map to existing + Keystone groups inside a domain. Those groups would inherit OpenStack roles such as *admin* or *member* + + + !!! note + By default Keycloak exports group names with a path prefix such as **/** and this behavior can be turned off + with creating a mapping rule for a attribute such **group** and configure the option as shown below. + In addition to disabling **Full Group Path** option, enable the **Single Group Attribute** option. + + Select your realm > Configure > User Federation > Provider (such as LDAP) > Mappers > Add Mapper + + ![Attribute mapping](assets/images/keycloak-group-mapping.png){align=center : style="max-width:296px"} + + + When using the Keycloak the *shibboleth2.xml* file must be updated to include signing for all requests and responses. + + The *shibboleth2.xml* file must be updated to include the following options, commonly the URLs marked with `example.com`. + In this example the username attribute + + ``` xml + + ``` + + The `MetadataProvider` should be made dynamic, where *openstack* designates the created realm inside keycloak. + + ``` xml + + + + ``` + + Additionally the `Handler` for the `MetadataGenerator` must be updated to include signing: + + ``` xml + + ``` + + !!! note Configure Keystone SP inside keycloak + Keystone SP must be introduced to keycloak as Client and entity ID among redirect URLs must be configured + to enable the SAML protocol via *Client type* SAML + + ![Allowing entities](assets/images/keycloak-client-config.png){align=center} + #### Generate the SAML Keys For **N** numbers of years. The keys are used to sign the SAML requests and responses. @@ -323,7 +406,8 @@ kubectl -n openstack create secret generic keystone-shibd-etc \ --from-file=/etc/genestack/keystone-sp/shibboleth/shibd.logger \ --from-file=/etc/genestack/keystone-sp/shibboleth/sp-cert.pem \ --from-file=/etc/genestack/keystone-sp/shibboleth/sp-key.pem \ - --from-file=/etc/genestack/keystone-sp/shibboleth/sslError.html + --from-file=/etc/genestack/keystone-sp/shibboleth/sslError.html \ + --from-file=/etc/genestack/keystone-sp/shibboleth/sso_callback_template.html ``` ### Create the SAML identity provider @@ -343,9 +427,49 @@ You're also welcome to generate your own mapping to suit your needs; however, if !!! abstract "Example keystone `saml-mapping.json` file" - ``` json - --8<-- "etc/keystone/saml-mapping.json" - ``` + === "Rackspace" + + ``` json + --8<-- "etc/keystone/saml-mapping.json" + ``` + + === "Keycloak" + + ``` json + [ + { + "local": [ + { + "user": { + "name": "{0}", + "email": "{1}", + "domain": { + "name": "KEYSTONE DOMAIN" + } + }, + "group": { + "name": "{2}", + "domain": { + "name": "KEYSTONE DOMAIN" + } + } + } + ], + "remote": [ + { + "type": "REMOTE_USERNAME" + }, + { + "type": "REMOTE_EMAIL" + }, + { + "type": "REMOTE_GROUP" + } + ] + } + ] + ``` + The example mapping **JSON** file can be found within the genestack repository at `etc/keystone/saml-mapping.json`. diff --git a/etc/keystone-sp/shibboleth/attrChecker.html b/etc/keystone-sp/shibboleth/attrChecker.html new file mode 100644 index 000000000..a3ddf6ef5 --- /dev/null +++ b/etc/keystone-sp/shibboleth/attrChecker.html @@ -0,0 +1,57 @@ + + + + + + + + Insufficient Information + + + + + +Logo + +

We're sorry, but you cannot access this service at this time.

+ + +

This service requires information about you that your identity provider +() +did not release. To gain access to this service, your identity provider +must release the required information.

+ + +

+

 +

+ + + + +

Your session was already invalidated before your information could + be examined for completeness.

+ + +

+You were trying to access the following URL: +

+

+ + +

For more information about this service, including what user information is +required for access, please visit our +information page.

+ + + + diff --git a/etc/keystone-sp/shibboleth/attribute-map.xml b/etc/keystone-sp/shibboleth/attribute-map.xml new file mode 100644 index 000000000..53da93e44 --- /dev/null +++ b/etc/keystone-sp/shibboleth/attribute-map.xml @@ -0,0 +1,173 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + --> + diff --git a/etc/keystone-sp/shibboleth/attribute-policy.xml b/etc/keystone-sp/shibboleth/attribute-policy.xml new file mode 100644 index 000000000..64274a936 --- /dev/null +++ b/etc/keystone-sp/shibboleth/attribute-policy.xml @@ -0,0 +1,77 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/etc/keystone-sp/shibboleth/bindingTemplate.html b/etc/keystone-sp/shibboleth/bindingTemplate.html new file mode 100644 index 000000000..59a924b67 --- /dev/null +++ b/etc/keystone-sp/shibboleth/bindingTemplate.html @@ -0,0 +1,58 @@ + + + Shibboleth Authentication Request + + + +

Shibboleth Authentication Request

+ + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
+ + diff --git a/etc/keystone-sp/shibboleth/console.logger b/etc/keystone-sp/shibboleth/console.logger new file mode 100644 index 000000000..dedb731ce --- /dev/null +++ b/etc/keystone-sp/shibboleth/console.logger @@ -0,0 +1,33 @@ +log4j.rootCategory=WARN, console + +# fairly verbose for DEBUG, so generally leave at INFO +log4j.category.XMLTooling.XMLObject=INFO +log4j.category.XMLTooling.XMLObjectBuilder=INFO +log4j.category.XMLTooling.KeyInfoResolver=INFO +log4j.category.Shibboleth.IPRange=INFO +log4j.category.Shibboleth.PropertySet=INFO + +# raise for low-level tracing of SOAP client HTTP/SSL behavior +log4j.category.XMLTooling.libcurl=INFO + +# useful categories to tune independently: +# +# tracing of SAML messages and security policies +#log4j.category.OpenSAML.MessageDecoder=DEBUG +#log4j.category.OpenSAML.MessageEncoder=DEBUG +#log4j.category.OpenSAML.SecurityPolicyRule=DEBUG +# interprocess message remoting +#log4j.category.Shibboleth.Listener=DEBUG +# mapping of requests to applicationId +#log4j.category.Shibboleth.RequestMapper=DEBUG +# high level session cache operations +#log4j.category.Shibboleth.SessionCache=DEBUG +# persistent storage and caching +#log4j.category.XMLTooling.StorageService=DEBUG + +# define the appender + +log4j.appender.console=org.apache.log4j.ConsoleAppender +#log4j.appender.console.layout=org.apache.log4j.BasicLayout +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n diff --git a/etc/keystone-sp/shibboleth/discoveryTemplate.html b/etc/keystone-sp/shibboleth/discoveryTemplate.html new file mode 100644 index 000000000..244e1f51e --- /dev/null +++ b/etc/keystone-sp/shibboleth/discoveryTemplate.html @@ -0,0 +1,48 @@ + + + Request for Authentication + + +

Request for Authentication

+ +

This web site requires you to login before proceeding. Please identify + the domain name of your organization:

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +

The system was unable to determine how to proceed using the value you supplied.

+ + + diff --git a/etc/keystone-sp/shibboleth/example-metadata.xml b/etc/keystone-sp/shibboleth/example-metadata.xml new file mode 100644 index 000000000..1b99d15ed --- /dev/null +++ b/etc/keystone-sp/shibboleth/example-metadata.xml @@ -0,0 +1,172 @@ + + + + + + + + + + example.org + + + + Identities 'R' Us + https://idp.example.org/info/ + https://example.org/images/logo.png + https://example.org/images/favico.png + + + + + + + + + MIICkjCCAfugAwIBAgIJAK7VCxPsh8yrMA0GCSqGSIb3DQEBBAUAMDsxCzAJBgNV + BAYTAlVTMRIwEAYDVQQKEwlJbnRlcm5ldDIxGDAWBgNVBAMTD2lkcC5leGFtcGxl + Lm9yZzAeFw0wNTA2MjAxNTUwNDFaFw0zMjExMDUxNTUwNDFaMDsxCzAJBgNVBAYT + AlVTMRIwEAYDVQQKEwlJbnRlcm5ldDIxGDAWBgNVBAMTD2lkcC5leGFtcGxlLm9y + ZzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA2VnUvWYrNhtRUqIxAuFmV8YP + Jhr+OMKJpc/RaEs2C8mk5N5qO+ysClg2cVfkws3O4Lc15AiNdQ0s3ZijYwJK2EEg + 4vmoTl2RrjP1b3PK2h+VbUuYny9enHwDL+Z4bjP/8nmIKlhUSq4DTGXbwdQiWjCd + lQXvDtvHRwX/TaqtHbcCAwEAAaOBnTCBmjAdBgNVHQ4EFgQUlmI7WqzIDJzcfAyU + v2kmk3p9sbAwawYDVR0jBGQwYoAUlmI7WqzIDJzcfAyUv2kmk3p9sbChP6Q9MDsx + CzAJBgNVBAYTAlVTMRIwEAYDVQQKEwlJbnRlcm5ldDIxGDAWBgNVBAMTD2lkcC5l + eGFtcGxlLm9yZ4IJAK7VCxPsh8yrMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEE + BQADgYEAsatF5gh1ZBF1QuXxchKp2BKVOsK+23y+FqhuOuVi/PTMf+Li84Ih25Al + Jyy3OKc0oprM6tCJaiSooy32KTW6a1xhPm2MwuXzD33SPoKItue/ndp8Bhx/PO9U + w14fpgtAk2x8xD7cpHsZ073JHxEcjEetD8PTtrFdNu6GwIrv6Sk= + + + + + + + + + + + + + urn:mace:shibboleth:1.0:nameIdentifier + urn:oasis:names:tc:SAML:2.0:nameid-format:transient + + + + + + + + + + + + example.org + + + + + + + + MIICkjCCAfugAwIBAgIJAK7VCxPsh8yrMA0GCSqGSIb3DQEBBAUAMDsxCzAJBgNV + BAYTAlVTMRIwEAYDVQQKEwlJbnRlcm5ldDIxGDAWBgNVBAMTD2lkcC5leGFtcGxl + Lm9yZzAeFw0wNTA2MjAxNTUwNDFaFw0zMjExMDUxNTUwNDFaMDsxCzAJBgNVBAYT + AlVTMRIwEAYDVQQKEwlJbnRlcm5ldDIxGDAWBgNVBAMTD2lkcC5leGFtcGxlLm9y + ZzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA2VnUvWYrNhtRUqIxAuFmV8YP + Jhr+OMKJpc/RaEs2C8mk5N5qO+ysClg2cVfkws3O4Lc15AiNdQ0s3ZijYwJK2EEg + 4vmoTl2RrjP1b3PK2h+VbUuYny9enHwDL+Z4bjP/8nmIKlhUSq4DTGXbwdQiWjCd + lQXvDtvHRwX/TaqtHbcCAwEAAaOBnTCBmjAdBgNVHQ4EFgQUlmI7WqzIDJzcfAyU + v2kmk3p9sbAwawYDVR0jBGQwYoAUlmI7WqzIDJzcfAyUv2kmk3p9sbChP6Q9MDsx + CzAJBgNVBAYTAlVTMRIwEAYDVQQKEwlJbnRlcm5ldDIxGDAWBgNVBAMTD2lkcC5l + eGFtcGxlLm9yZ4IJAK7VCxPsh8yrMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEE + BQADgYEAsatF5gh1ZBF1QuXxchKp2BKVOsK+23y+FqhuOuVi/PTMf+Li84Ih25Al + Jyy3OKc0oprM6tCJaiSooy32KTW6a1xhPm2MwuXzD33SPoKItue/ndp8Bhx/PO9U + w14fpgtAk2x8xD7cpHsZ073JHxEcjEetD8PTtrFdNu6GwIrv6Sk= + + + + + + + + + + + urn:mace:shibboleth:1.0:nameIdentifier + urn:oasis:names:tc:SAML:2.0:nameid-format:transient + + + + + + Example Identity Provider + Identities 'R' Us + http://idp.example.org/ + + + Technical Support + support@idp.example.org + + + diff --git a/etc/keystone-sp/shibboleth/example-shibboleth2.xml b/etc/keystone-sp/shibboleth/example-shibboleth2.xml new file mode 100644 index 000000000..71ebebd5a --- /dev/null +++ b/etc/keystone-sp/shibboleth/example-shibboleth2.xml @@ -0,0 +1,299 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/etc/keystone-sp/shibboleth/globalLogout.html b/etc/keystone-sp/shibboleth/globalLogout.html new file mode 100644 index 000000000..86f205044 --- /dev/null +++ b/etc/keystone-sp/shibboleth/globalLogout.html @@ -0,0 +1,29 @@ + + + + + + + + Global Logout + + + + + +Logo + +

Global Logout

+ +

Status of Global Logout:

+ +

If the message above indicates success, you have been logged out of all +the applications and systems that support the logout mechanism.

+ +

Regardless of the outcome, it is strongly advised that you close your browser +to ensure that you complete the logout process.

+ + + diff --git a/etc/keystone-sp/shibboleth/localLogout.html b/etc/keystone-sp/shibboleth/localLogout.html new file mode 100644 index 000000000..75bd3e170 --- /dev/null +++ b/etc/keystone-sp/shibboleth/localLogout.html @@ -0,0 +1,27 @@ + + + + + + + + Local Logout + + + + + +Logo + +

Local Logout

+ +Status of Local Logout: + +

+ +You MUST close your browser to complete the logout process. + + + diff --git a/etc/keystone-sp/shibboleth/metadataError.html b/etc/keystone-sp/shibboleth/metadataError.html new file mode 100644 index 000000000..e0e6a1b05 --- /dev/null +++ b/etc/keystone-sp/shibboleth/metadataError.html @@ -0,0 +1,35 @@ + + + + + + + + Unknown Identity Provider + + + + + +Logo + +

Unknown or Unusable Identity Provider

+ +

The identity provider supplying your login credentials is not authorized +for use with this service or does not support the necessary capabilities.

+ +

To report this problem, please contact the site administrator at +. +

+ +

Please include the following error message in any email:

+

Identity provider lookup failed at ()

+ +

EntityID:

+ +

:

+ + + diff --git a/etc/keystone-sp/shibboleth/native.logger b/etc/keystone-sp/shibboleth/native.logger new file mode 100644 index 000000000..e9a43a571 --- /dev/null +++ b/etc/keystone-sp/shibboleth/native.logger @@ -0,0 +1,30 @@ +# set overall behavior +log4j.rootCategory=WARN, native_log + +# fairly verbose for DEBUG, so generally leave at WARN/INFO +log4j.category.XMLTooling.XMLObject=WARN +log4j.category.XMLTooling.XMLObjectBuilder=WARN +log4j.category.XMLTooling.KeyInfoResolver=WARN +log4j.category.Shibboleth.IPRange=WARN +log4j.category.Shibboleth.PropertySet=WARN + +# useful categories to tune independently: +# +# interprocess message remoting +#log4j.category.Shibboleth.Listener=DEBUG +# mapping of requests to applicationId +#log4j.category.Shibboleth.RequestMapper=DEBUG +# high level session cache operations +#log4j.category.Shibboleth.SessionCache=DEBUG + +# define the appender + +# Change to SyslogAppender for remote syslog, and set host/port +log4j.appender.native_log=org.apache.log4j.LocalSyslogAppender +#log4j.appender.native_log.syslogHost=localhost +#log4j.appender.native_log.portNumber=514 +log4j.appender.native_log.syslogName=shibboleth +# Facility is numeric, 16 is LOCAL0 +log4j.appender.native_log.facility=16 +log4j.appender.native_log.layout=org.apache.log4j.PatternLayout +log4j.appender.native_log.layout.ConversionPattern=%p %c %x: %m%n diff --git a/etc/keystone-sp/shibboleth/partialLogout.html b/etc/keystone-sp/shibboleth/partialLogout.html new file mode 100644 index 000000000..fe24a7c3d --- /dev/null +++ b/etc/keystone-sp/shibboleth/partialLogout.html @@ -0,0 +1,24 @@ + + + + + + + + Partial Logout + + + + + +Logo + +

Partial Logout

+ +

You remain logged into one or more applications accessed during your session. +To complete the logout process, please close/exit your browser completely.

+ + + diff --git a/etc/keystone-sp/shibboleth/postTemplate.html b/etc/keystone-sp/shibboleth/postTemplate.html new file mode 100644 index 000000000..d8c4728d0 --- /dev/null +++ b/etc/keystone-sp/shibboleth/postTemplate.html @@ -0,0 +1,37 @@ + + + Login Completed + + + +

Login Completed

+ + + +
+ + + + +
+ + diff --git a/etc/keystone-sp/shibboleth/protocols.xml b/etc/keystone-sp/shibboleth/protocols.xml new file mode 100644 index 000000000..648bcbc32 --- /dev/null +++ b/etc/keystone-sp/shibboleth/protocols.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/etc/keystone-sp/shibboleth/security-policy.xml b/etc/keystone-sp/shibboleth/security-policy.xml new file mode 100644 index 000000000..f8eaacda9 --- /dev/null +++ b/etc/keystone-sp/shibboleth/security-policy.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/etc/keystone-sp/shibboleth/sessionError.html b/etc/keystone-sp/shibboleth/sessionError.html new file mode 100644 index 000000000..7ccf17be7 --- /dev/null +++ b/etc/keystone-sp/shibboleth/sessionError.html @@ -0,0 +1,45 @@ + + + + + + + + <shibmlp errorType/> + + + + + +Logo + +

+ +

The system encountered an error at

+ +

To report this problem, please contact the site administrator at +. +

+ +

Please include the following message in any email:

+

at ()

+ +

+ + +

Error from identity provider:

+
+ Status:
+ + Sub-Status:
+ + + Message:
+ +
+ + + + diff --git a/etc/keystone-sp/shibboleth/shibboleth2.xml b/etc/keystone-sp/shibboleth/shibboleth2.xml new file mode 100644 index 000000000..c10a36c1b --- /dev/null +++ b/etc/keystone-sp/shibboleth/shibboleth2.xml @@ -0,0 +1,73 @@ + + + + + + + + + memcached.openstack.svc.cluster.local:11211 + + + + + memcached.openstack.svc.cluster.local:11211 + + + + + + + + + + + SAML2 + + + SAML2 Local + + + + + + + + + + + + + + + + + + + + diff --git a/etc/keystone-sp/shibboleth/shibd.logger b/etc/keystone-sp/shibboleth/shibd.logger new file mode 100644 index 000000000..39950c591 --- /dev/null +++ b/etc/keystone-sp/shibboleth/shibd.logger @@ -0,0 +1,73 @@ +# set overall behavior +log4j.rootCategory=INFO, shibd_log, warn_log + +# fairly verbose for DEBUG, so generally leave at INFO +log4j.category.XMLTooling.XMLObject=INFO +log4j.category.XMLTooling.XMLObjectBuilder=INFO +log4j.category.XMLTooling.KeyInfoResolver=INFO +log4j.category.Shibboleth.IPRange=INFO +log4j.category.Shibboleth.PropertySet=INFO + +# raise for low-level tracing of SOAP client HTTP/SSL behavior +log4j.category.XMLTooling.libcurl=INFO + +# useful categories to tune independently: +# +# tracing of SAML messages and security policies +#log4j.category.OpenSAML.MessageDecoder=DEBUG +#log4j.category.OpenSAML.MessageEncoder=DEBUG +#log4j.category.OpenSAML.SecurityPolicyRule=DEBUG +#log4j.category.XMLTooling.SOAPClient=DEBUG +# interprocess message remoting +#log4j.category.Shibboleth.Listener=DEBUG +# mapping of requests to applicationId +#log4j.category.Shibboleth.RequestMapper=DEBUG +# high level session cache operations +#log4j.category.Shibboleth.SessionCache=DEBUG +# persistent storage and caching +#log4j.category.XMLTooling.StorageService=DEBUG + +# logs XML being signed or verified if set to DEBUG +log4j.category.XMLTooling.Signature.Debugger=INFO, sig_log +log4j.additivity.XMLTooling.Signature.Debugger=false +log4j.ownAppenders.XMLTooling.Signature.Debugger=true + +# the tran log blocks the "default" appender(s) at runtime +# Level should be left at INFO for this category +log4j.category.Shibboleth-TRANSACTION=INFO, tran_log +log4j.additivity.Shibboleth-TRANSACTION=false +log4j.ownAppenders.Shibboleth-TRANSACTION=true + +# uncomment to suppress particular event types +#log4j.category.Shibboleth-TRANSACTION.AuthnRequest=WARN +#log4j.category.Shibboleth-TRANSACTION.Login=WARN +#log4j.category.Shibboleth-TRANSACTION.Logout=WARN + +# define the appenders + +log4j.appender.shibd_log=org.apache.log4j.RollingFileAppender +log4j.appender.shibd_log.fileName=/var/log/shibboleth/shibd.log +log4j.appender.shibd_log.maxFileSize=1000000 +log4j.appender.shibd_log.maxBackupIndex=10 +log4j.appender.shibd_log.layout=org.apache.log4j.PatternLayout +log4j.appender.shibd_log.layout.ConversionPattern=%d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n + +log4j.appender.warn_log=org.apache.log4j.RollingFileAppender +log4j.appender.warn_log.fileName=/var/log/shibboleth/shibd_warn.log +log4j.appender.warn_log.maxFileSize=1000000 +log4j.appender.warn_log.maxBackupIndex=10 +log4j.appender.warn_log.layout=org.apache.log4j.PatternLayout +log4j.appender.warn_log.layout.ConversionPattern=%d{%Y-%m-%d %H:%M:%S} %p %c %x: %m%n +log4j.appender.warn_log.threshold=WARN + +log4j.appender.tran_log=org.apache.log4j.RollingFileAppender +log4j.appender.tran_log.fileName=/var/log/shibboleth/transaction.log +log4j.appender.tran_log.maxFileSize=1000000 +log4j.appender.tran_log.maxBackupIndex=20 +log4j.appender.tran_log.layout=org.apache.log4j.PatternLayout +log4j.appender.tran_log.layout.ConversionPattern=%d{%Y-%m-%d %H:%M:%S}|%c|%m%n + +log4j.appender.sig_log=org.apache.log4j.FileAppender +log4j.appender.sig_log.fileName=/var/log/shibboleth/signature.log +log4j.appender.sig_log.layout=org.apache.log4j.PatternLayout +log4j.appender.sig_log.layout.ConversionPattern=%m diff --git a/etc/keystone-sp/shibboleth/sslError.html b/etc/keystone-sp/shibboleth/sslError.html new file mode 100644 index 000000000..367366a63 --- /dev/null +++ b/etc/keystone-sp/shibboleth/sslError.html @@ -0,0 +1,33 @@ + + + + + + + + POST Failed + + + + + +Logo + +

POST Failed

+ +

+You have attemped to submit information without the protection +of TLS to this site.
+

+ +

+For the protection of your submission and the integrity of the site, +this is not permitted. Please try accessing the server with a +URL starting with https:// and report this problem +to +

+ + + diff --git a/etc/keystone-sp/shibboleth/sso_callback_template.html b/etc/keystone-sp/shibboleth/sso_callback_template.html new file mode 100644 index 000000000..3364d69e5 --- /dev/null +++ b/etc/keystone-sp/shibboleth/sso_callback_template.html @@ -0,0 +1,22 @@ + + + + Keystone WebSSO redirect + + +
+ Please wait... +
+ + +
+ + + From ea4dc45fe31787f9f49e549bd2f3576e450d2a6e Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Wed, 3 Sep 2025 09:42:41 -0500 Subject: [PATCH 95/97] chore: add warning when secret file exists (#1165) Signed-off-by: Kevin Carter --- bin/create-secrets.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bin/create-secrets.sh b/bin/create-secrets.sh index e0c73c2fc..eda025b04 100755 --- a/bin/create-secrets.sh +++ b/bin/create-secrets.sh @@ -102,6 +102,14 @@ ironic_rabbitmq_password=$(generate_password 32) OUTPUT_FILE="/etc/genestack/kubesecrets.yaml" +if [[ -f ${OUTPUT_FILE} ]]; then + echo "Error: ${OUTPUT_FILE} already exists. Please remove it before running this script." + echo " This will replace an existing file and will lead to mass rotation, which is" + echo " likely not what you want to do. If you really want to break your system, please" + echo " make sure you know what you're doing." + exit 99 +fi + cat < $OUTPUT_FILE --- apiVersion: v1 From 888a9fa6a6fe16a920cf6865c1ced796b8044b81 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Wed, 3 Sep 2025 09:43:12 -0500 Subject: [PATCH 96/97] feat: add PCI-DSS compliance handling for Keystone (#1164) Signed-off-by: Kevin Carter --- base-helm-configs/keystone/keystone-helm-overrides.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/base-helm-configs/keystone/keystone-helm-overrides.yaml b/base-helm-configs/keystone/keystone-helm-overrides.yaml index 3215f7d75..4abc23dbc 100644 --- a/base-helm-configs/keystone/keystone-helm-overrides.yaml +++ b/base-helm-configs/keystone/keystone-helm-overrides.yaml @@ -99,6 +99,11 @@ conf: rackspace: role_attribute: os_flex role_attribute_enforcement: false + # NOTE(cloudnull): See https://docs.openstack.org/keystone/latest/admin/configuration.html#security-compliance-and-pci-dss for more + security_compliance: + lockout_failure_attempts: 6 + lockout_duration: 1800 + disable_user_account_days_inactive: 90 logging: logger_root: handlers: From 7c7a347365fb97a3a2eb0578a5950b20f5b669a0 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Thu, 4 Sep 2025 10:44:56 -0500 Subject: [PATCH 97/97] feat: add VPNaaS and FWaaS (#1138) This change will add VPNaaS and FWaaS to our default neutron configuration. This will ensure that we have full OVN functionality for our end-users and give our consumers access to advanced features within the platform. Signed-off-by: Kevin Carter --- .../neutron/neutron-helm-overrides.yaml | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/base-helm-configs/neutron/neutron-helm-overrides.yaml b/base-helm-configs/neutron/neutron-helm-overrides.yaml index 38489d2a0..69956bf77 100644 --- a/base-helm-configs/neutron/neutron-helm-overrides.yaml +++ b/base-helm-configs/neutron/neutron-helm-overrides.yaml @@ -53,6 +53,8 @@ dependencies: jobs: null ovn_metadata: pod: [] + ovn_vpn_agent: + pod: [] ovs_agent: jobs: null rpc_server: @@ -167,8 +169,21 @@ conf: router_scheduler_driver: neutron.scheduler.l3_agent_scheduler.AZLeastRoutersScheduler rpc_state_report_workers: 2 rpc_workers: 2 - service_plugins: "ovn-router,qos,metering,trunk,segments" + # NOTE(cloudnull): in 2025.1 we can add firewall_v2 + service_plugins: "ovn-router,ovn-vpnaas,qos,metering,trunk,segments" + service_providers: + service_provider: + type: multistring + values: + - "VPN:strongswan:neutron_vpnaas.services.vpn.service_drivers.ovn_ipsec.IPsecOvnVPNDriver:default" + # - "FIREWALL_V2:fwaas_db:neutron_fwaas.services.firewall.service_drivers.ovn.firewall_l3_driver.OVNFwaasDriver:default" + fwaas: + agent_version: v2 + driver: neutron_fwaas.services.firewall.service_drivers.ovn.firewall_l3_driver.OVNFwaasDriver + # NOTE(cloudnull): in 2025.1 we can enable this + enabled: False agent: + extensions: vpnaas availability_zone: az1 database: connection_debug: 0 @@ -225,11 +240,17 @@ conf: metadata_workers: 2 ovs: ovsdb_connection: "tcp:127.0.0.1:6640" + ovn_vpn_agent: + ovs: + ovsdb_connection: "tcp:127.0.0.1:6640" plugins: ml2_conf: agent: availability_zone: az1 + # NOTE(cloudnull): in 2025.1 we can add fwaas_v2 extensions: "fip_qos,gateway_ip_qos" + fwaas: + firewall_l2_driver: noop ml2: extension_drivers: "port_security,qos" mechanism_drivers: ovn @@ -322,6 +343,7 @@ manifests: daemonset_metadata_agent: false daemonset_ovn_metadata_agent: true daemonset_ovs_agent: false + daemonset_ovn_vpn_agent: true ingress_server: false job_db_init: false job_rabbit_init: false
+Please visit + +the support page +this support page + +for further instructions. +