chore: Azure Migration

pipelines/deploy_dv.yaml

@@ -0,0 +1,91 @@
+trigger: none
+
+parameters:
+  - name: Image
+    displayName: Build Container Image
+    type: string
+    default: "prod"
+    values:
+      # - dev
+      - prod
+
+variables:
+  registry: "hcsccrrc.azurecr.io"
+  repository: "nsp/sdse-spib-hopic-dv"
+  tag: "$(Build.SourceVersion)"
+  appName: was-spib-sdse-hopic-dv
+  subscription: SPIB-SDSE-HOPIC-CICDLZSP-DT
+  resourceGroup: rg-spib-sdse-hopic-dv
+  envars: >
+      DB_HOST=pgsql-spib-sdse-hopic-dv.postgres.database.azure.com
+      DB_PORT=5432
+      DB_SSLMODE=require
+      SECRET_KEY='@Microsoft.KeyVault(SecretUri=https://kvspibsdsehopicdv.vault.azure.net/secrets/hopic-django-secret-key/)'
+      WEBSITES_PORT=8000
+      LATEST_COMMIT_SHA=$(Build.SourceVersion)
+      ALLOWED_HOSTS=was-spib-sdse-hopic-dv.azurewebsites.net
+      CSRF_TRUSTED_ORIGINS=https://was-spib-sdse-hopic-dv.azurewebsites.net
+      DB_NAME=hopicdb_migration
+      DB_PASSWORD='@Microsoft.KeyVault(SecretUri=https://kvspibsdsehopicdv.vault.azure.net/secrets/hopicapp-pgsql-password/)'
+      DB_USER=hopicapp
+      ENV=dev
+      AZCOPY_AUTO_LOGIN_TYPE=MSI
+
+
+pool:
+  name: spib-sdse-hopic-agents-dv
+
+jobs:
+  - job: Deploy_DV
+    steps:
+      - script: |
+          sudo apt-get update
+          sudo apt-get install unzip
+        displayName: "Install Unzip"
+
+      - script: |
+          sudo apt install -y docker.io
+          sudo apt install docker-buildx
+          sudo systemctl start docker
+          sudo usermod -aG docker $(id -un)
+          sudo chmod 666 /var/run/docker.sock
+        displayName: "Install and Configure Docker"
+
+      - script: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+        displayName: "Install AZ CLI"
+
+      - task: AzureCLI@2
+        displayName: "Login to ACR"
+        inputs:
+          azureSubscription: $(subscription)
+          scriptType: bash
+          scriptLocation: inlineScript
+          inlineScript: |
+            az --version
+            az acr login --name hcsccrrc   
+
+      - ${{ if eq(parameters.Image, 'prod')}}:
+        - script: |
+            docker build -t $(registry)/$(repository):$(tag) -f server/Dockerfile.prod .
+            docker push $(registry)/$(repository):$(tag)
+          displayName: "Build and Push $(repository) Image"
+
+      - task: AzureWebAppContainer@1
+        displayName: "Install $(repository) into $(appName)"
+        inputs:
+          azureSubscription: "$(subscription)"
+          appName: "$(appName)"
+          deployToSlotOrASE: true
+          resourceGroupName: "$(resourceGroup)"
+          containers: "$(registry)/$(repository):$(tag)"
+          containerCommand: "gunicorn --bind 0.0.0.0:8000 server.wsgi --timeout 1000"
+
+      - task: AzureCLI@2
+        displayName: "AppSettings for $(appName)"
+        inputs:
+          azureSubscription: $(subscription)
+          scriptType: bash
+          scriptLocation: inlineScript
+          inlineScript: |
+            az webapp config appsettings set -g $(resourceGroup) -n $(appName) --settings ${{ variables.envars }}

server/Dockerfile.prod

@@ -1,60 +1,103 @@
-######################
-# DEPENDENCY BUILDER #
-######################
-# NOTE: builder layer must mach python and distribution versions of distroless runtime layer!
-FROM python:3.11-bookworm as build_env
-
-# MUST keep these envs in sync with the Dockerfile.prod "FINAL" layer AND with Dockerfile.dev-management
-ENV HOME=/cpho
-ENV APP_HOME=$HOME/web
-ENV PYTHON_DEPS=$HOME/python_deps
+# Builds a Prod Image expecting to write to ACR and run the image in an app service container
 
-RUN mkdir "${HOME}" && \
-    mkdir "${APP_HOME}" && \
-    mkdir "${PYTHON_DEPS}"
+###########
+# BUILDER #
+###########
+FROM python:3.11-slim-bookworm as builder
 
-# Update pip
-RUN pip install --upgrade pip
+# set work directory
+WORKDIR /usr/src/app
 
-COPY ./requirements.txt .
-COPY ./requirements_dev.txt .
-COPY ./requirements_formatting.txt .
+# set environment variables
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
 
-ARG DEPENDENCY_SET="prod"
+# Install build dependencies
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    build-essential \
+    libpq-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
-RUN [ "${DEPENDENCY_SET}" = "prod" ] \
-    && pip install --no-cache-dir --target "${PYTHON_DEPS}" -r requirements.txt \
-    || :
+RUN pip install --upgrade pip wheel setuptools
 
-RUN [ "${DEPENDENCY_SET}" = "test" ] \
-    && pip install --no-cache-dir --target "${PYTHON_DEPS}" -r requirements.txt -r requirements_dev.txt -r requirements_formatting.txt \
-    || :
+# requirements and wheels
+COPY server/requirements.txt .
+RUN pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/wheels -r requirements.txt
 
 #########
 # FINAL #
 #########
-FROM gcr.io/distroless/python3-debian12
+# pull official base image
+FROM python:3.11-slim-bookworm
+
+# Make sure setuptools is always up to date
+RUN pip install --upgrade pip setuptools
 
-# MUST keep these envs in sync with the Dockerfile.prod "DEPENDENCY BUILDER" layer AND with Dockerfile.dev-management
-ENV HOME=/cpho
+# Environment Variables
+ENV APP_NAME=hopicapp
+ENV APP_USER=${APP_NAME}user
+ENV HOME=/${APP_NAME}
 ENV APP_HOME=$HOME/web
-ENV PYTHON_DEPS=$HOME/python_deps
+ENV VIRTUALENV=$HOME/env
+ENV WHEELDIR=$HOME/wheels
+ENV PATH=$VIRTUALENV/bin:$PATH
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+EXPOSE 8000
+
+# Install minimal runtime dependencies
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends \
+    gosu \
+    libpq5 \
+    netcat-traditional \
+    openssh-server \
+    postgresql-common \
+    && addgroup --system $APP_NAME \
+    && adduser --disabled-password --shell /bin/bash $APP_USER --system --ingroup $APP_NAME --home $HOME \
+    && mkdir -p $APP_HOME $VIRTUALENV $WHEELDIR \
+    && chown -R $APP_USER:$APP_NAME $HOME \
+    && echo "root:Docker!" | chpasswd \
+    && echo -e "HOPIC Container\n\nFor management commands run: \ncd $APP_HOME \npython manage\n\n" > /etc/motd \
+    && echo -e "cd $APP_HOME\n" >> /etc/profile
 
-# this is the ID of the distroless "nonroot" user, using ID instead of user name because the k8s runAsNonRoot security context
-# can't verify non-rootness when the docker file sets user by name
-# https://github.com/GoogleContainerTools/distroless/blob/9c5d2c431825d7aa21017551b2ec75c29c1f23c6/common/variables.bzl#L18
-ENV NONROOT_USER_ID=65532
+WORKDIR $APP_HOME
 
-ENV PATH="${PYTHON_DEPS}/bin:${PATH}"
-ENV PYTHONPATH="${PYTHON_DEPS}:${PYTHONPATH}"
+# Copy wheels from builder stage
+COPY --chown=$APP_USER:$APP_NAME --from=builder /usr/src/app/wheels $WHEELDIR
 
-COPY --chown=$NONROOT_USER_ID:$NONROOT_USER_ID --from=build_env $HOME $HOME
-COPY --chown=$NONROOT_USER_ID:$NONROOT_USER_ID . $APP_HOME
 
-WORKDIR $APP_HOME
 
-USER $NONROOT_USER_ID
+# Create and configure virtual environment
+RUN python -m venv $VIRTUALENV && \
+    $VIRTUALENV/bin/pip install --upgrade pip && \
+    $VIRTUALENV/bin/pip install --no-cache-dir $WHEELDIR/* && \
+    $VIRTUALENV/bin/pip install --force-reinstall setuptools && \
+    rm -rf $WHEELDIR
+
+# Copy project files
+COPY --chown=$APP_USER:$APP_NAME server/ $APP_HOME
+
+# copy sshd_config file
+COPY server/sshd_config /etc/ssh/
+
+# copy version file
+# COPY version.txt $APP_HOME
+
+# Setup entrypoint and create staticfiles
+RUN chmod +x $APP_HOME/entrypoint.prod.sh && \
+    rm -f $APP_HOME/sshd_config && \
+    mkdir -p $APP_HOME/staticfiles && \
+    SECRET_KEY=t ALLOWED_HOSTS=* DB_NAME=d DB_USER=d DB_PASSWORD=d DB_HOST=d DB_PORT=1 python -m manage collectstatic --no-input
 
-EXPOSE 8080
+# Set entrypoint and default command
+# Wrapping the dynamic entrypoint call to ensure parameters are passed through correctly
+RUN echo '#!/bin/bash\n"${APP_HOME}/entrypoint.prod.sh" "$@"' > /entrypoint-wrapper.sh && \
+    chmod +x /entrypoint-wrapper.sh
 
-ENTRYPOINT [ "python", "./entrypoint.prod.py" ]
+ENTRYPOINT ["/entrypoint-wrapper.sh"]
+CMD ["gunicorn", "--bind", "0.0.0.0:8000", "server.wsgi"]

server/entrypoint.prod.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+echo "Starting SSH server..."
+service ssh start
+
+echo "Generating static files..."
+python manage.py collectstatic --no-input
+
+if [ ! -z "$DB_HOST" ] && [ ! -z "$DB_PORT" ]; then
+  echo "Waiting for postgres ($DB_HOST:$DB_PORT)..."
+
+  while ! nc -z $DB_HOST $DB_PORT; do
+    sleep 0.1
+  done
+  sleep 1
+
+  echo "PostgreSQL started"
+  echo "applying migrations..."
+  python manage.py migrate
+  echo "migrations applied"
+
+fi
+
+eval $(printenv | sed -n "s/^\([^=]\+\)=\(.*\)$/export \1=\2/p" | sed 's/"/\\\"/g' | sed '/=/s//="/' | sed 's/$/"/' >> /etc/profile)
+
+exec gosu ${APP_USER} "$@"

server/gunicorn.conf.py

@@ -3,8 +3,6 @@
 
 import structlog
 
-from server.open_telemetry_util import instrument_app_for_open_telemetry
-
 # See https://cloud.google.com/run/docs/tips/python#optimize_gunicorn
 
 PORT = os.getenv("PORT", "8080")
@@ -26,7 +24,7 @@ def post_fork(server, worker):
     # If NOT using BatchSpanProcessor (likely a bad idea, it's much more performant at run time) you can move instrumentation to wsgi.py and enable preload_app
     os.environ.setdefault("DJANGO_SETTINGS_MODULE", "server.settings")
 
-    worker.flush_telemetry_callback = instrument_app_for_open_telemetry()
+    # worker.flush_telemetry_callback = instrument_app_for_open_telemetry()
 
 
 def worker_exit(server, worker):

server/manage.py

@@ -5,15 +5,12 @@
 import sys
 
 from server.config_util import get_project_config
-from server.open_telemetry_util import instrument_app_for_open_telemetry
 
 
 def main():
     """Run administrative tasks."""
     os.environ.setdefault("DJANGO_SETTINGS_MODULE", "server.settings")
 
-    instrument_app_for_open_telemetry()
-
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc: