feat: extract Atlas as a reusable GCP production template

.cursor/mcp.json

@@ -0,0 +1,25 @@
+{
+  "mcpServers": {
+    "bigquery": {
+      "command": "npx",
+      "args": [
+        "-y",
+        "@modelcontextprotocol/server-bigquery"
+      ],
+      "env": {
+        "GOOGLE_CLOUD_PROJECT": "${env:ATLAS_GCP_PROJECT_ID}"
+      }
+    },
+    "dbt-atlas": {
+      "command": "${workspaceFolder}/.venv-dbt/bin/dbt",
+      "args": [
+        "--version"
+      ],
+      "env": {
+        "DBT_PROJECT_DIR": "${workspaceFolder}/dbt/atlas_dbt",
+        "DBT_PATH": "${workspaceFolder}/.venv-dbt/bin/dbt",
+        "DBT_TARGET": "bigquery"
+      }
+    }
+  }
+}

.env.example

@@ -0,0 +1,20 @@
+# Atlas production-template configuration
+ATLAS_PROJECT_NAME=atlas
+ATLAS_ENVIRONMENT=dev
+ATLAS_GCP_PROJECT_ID=example-gcp-project
+ATLAS_GCP_PROJECT_NUMBER=123456789012
+ATLAS_GCP_LOCATION=US
+ATLAS_GCP_REGION=us-central1
+ATLAS_DATASET_PREFIX=atlas
+ATLAS_RAW_DATASET=atlas_raw
+ATLAS_DBT_DATASET=atlas
+ATLAS_OPS_DATASET=atlas_ops
+ATLAS_GCS_BUCKET=atlas-raw-events-example-gcp-project
+ATLAS_RELEASE_BUCKET=atlas-releases-example-gcp-project
+ATLAS_SERVICE_ACCOUNT_PREFIX=atlas
+ATLAS_DAG_ID=atlas_batch_pipeline
+ATLAS_SCHEDULE=@daily
+ATLAS_NOTIFICATION_EMAIL=<operator-email>
+ATLAS_COST_CEILING_BYTES=1000000000
+DBT_TARGET=bigquery
+DBT_PATH=.venv/bin/dbt

.github/workflows/atlas-ci.yml

@@ -0,0 +1,170 @@
+name: atlas-ci
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: atlas-ci-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  PYTHON_VERSION: "3.12"
+
+jobs:
+  atlas-security-shell:
+    name: atlas-security-shell
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout
+        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: pip
+          cache-dependency-path: requirements-ci.txt
+      - name: Install validation toolchain
+        run: pip install -r requirements-ci.txt
+      - name: Dependency-file sanity
+        run: |
+          python - <<'PY'
+          from pathlib import Path
+
+          for name in (
+              "requirements.txt",
+              "requirements-ci.txt",
+              "airflow/requirements-airflow.txt",
+              "dbt/requirements-dbt.txt",
+          ):
+              content = Path(name).read_text(encoding="utf-8")
+              assert content.strip(), f"{name} is empty"
+          print("dependency manifests present and non-empty")
+          PY
+      - name: Security and shell gates
+        run: bash scripts/validate_ci.sh --mode static --group security-shell
+      - name: Upload gate results
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: security-shell-gate-results
+          path: logs/ci/validate-ci-results.json
+
+  atlas-python:
+    name: atlas-python
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Checkout
+        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: pip
+          cache-dependency-path: |
+            requirements.txt
+            requirements-ci.txt
+      - name: Install locked dependencies
+        run: pip install -r requirements.txt -r requirements-ci.txt
+      - name: Python gates
+        run: bash scripts/validate_ci.sh --mode static --group python
+      - name: Upload gate results
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: python-gate-results
+          path: logs/ci/validate-ci-results.json
+
+  atlas-dbt:
+    name: atlas-dbt
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Checkout
+        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: pip
+          cache-dependency-path: dbt/requirements-dbt.txt
+      - name: Install pinned dbt environment
+        run: pip install -r dbt/requirements-dbt.txt
+      - name: dbt static gates
+        run: bash scripts/validate_ci.sh --mode static --group dbt
+      - name: Upload dbt manifest
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: dbt-manifest
+          path: dbt/atlas_dbt/target/manifest.json
+          if-no-files-found: warn
+
+  atlas-airflow:
+    name: atlas-airflow
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - name: Checkout
+        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: pip
+          cache-dependency-path: |
+            airflow/requirements-airflow.txt
+            requirements.txt
+            requirements-ci.txt
+      - name: Install pinned Airflow with official constraints
+        run: |
+          pip install "apache-airflow==3.1.7" \
+            --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.1.7/constraints-3.12.txt"
+          pip install -r airflow/requirements-airflow.txt -r requirements.txt -r requirements-ci.txt
+      - name: pip check
+        run: pip check
+      - name: Airflow gates
+        run: bash scripts/validate_ci.sh --mode static --group airflow
+      - name: DAG tests
+        run: PYTHONPATH=src:dags python -m pytest tests/airflow -q
+      - name: Upload gate results
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: airflow-gate-results
+          path: logs/ci/validate-ci-results.json
+
+  atlas-ci-gate:
+    name: atlas-ci-gate
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    needs:
+      - atlas-security-shell
+      - atlas-python
+      - atlas-dbt
+      - atlas-airflow
+    if: always()
+    steps:
+      - name: Require every job to succeed
+        env:
+          NEEDS_JSON: ${{ toJSON(needs) }}
+        run: |
+          python3 - <<'PY'
+          import json
+          import os
+          import sys
+
+          needs = json.loads(os.environ["NEEDS_JSON"])
+          failed = [name for name, value in needs.items() if value["result"] != "success"]
+          if failed:
+              print("Failed or skipped required jobs:", ", ".join(failed))
+              sys.exit(1)
+          print("All required Atlas CI jobs succeeded")
+          PY

.github/workflows/atlas-deploy.yml

@@ -0,0 +1,105 @@
+name: atlas-deploy
+
+on:
+  workflow_dispatch:
+    inputs:
+      confirm:
+        description: 'Type "deploy-atlas-dev" to confirm'
+        required: true
+      target_sha:
+        description: "Commit SHA reachable from main; empty uses main HEAD"
+        required: false
+        default: ""
+      create_composer:
+        description: "Create the ephemeral Composer environment if missing"
+        type: boolean
+        default: false
+      leave_paused:
+        description: "Leave the DAG paused after smoke validation"
+        type: boolean
+        default: true
+
+permissions:
+  contents: read
+  id-token: write
+
+concurrency:
+  group: atlas-dev-deployment
+  cancel-in-progress: false
+
+env:
+  PYTHON_VERSION: "3.12"
+
+jobs:
+  atlas-deploy:
+    name: atlas-deploy
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    environment: atlas-dev
+    steps:
+      - name: Verify typed confirmation
+        run: |
+          if [ "${{ github.event.inputs.confirm }}" != "deploy-atlas-dev" ]; then
+            echo "Confirmation input does not match deploy-atlas-dev"
+            exit 1
+          fi
+      - name: Checkout main
+        uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd
+        with:
+          ref: main
+          fetch-depth: 0
+      - name: Resolve trusted target
+        id: target
+        run: |
+          target="${{ github.event.inputs.target_sha }}"
+          if [ -z "$target" ]; then
+            target="$(git rev-parse HEAD)"
+          fi
+          git cat-file -e "${target}^{commit}"
+          git merge-base --is-ancestor "$target" origin/main
+          git checkout "$target"
+          echo "sha=$target" >> "$GITHUB_OUTPUT"
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: pip
+          cache-dependency-path: requirements.txt
+      - name: Install and validate
+        run: |
+          pip install -r requirements.txt -r requirements-ci.txt
+          bash scripts/validate_ci.sh --mode static --group security-shell
+          bash scripts/validate_ci.sh --mode static --group python
+      - name: Require WIF repository variables
+        run: |
+          test -n "${{ vars.ATLAS_WIF_PROVIDER }}" || { echo "Set ATLAS_WIF_PROVIDER"; exit 1; }
+          test -n "${{ vars.ATLAS_DEPLOYER_SERVICE_ACCOUNT }}" || { echo "Set ATLAS_DEPLOYER_SERVICE_ACCOUNT"; exit 1; }
+      - name: Authenticate to GCP
+        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093
+        with:
+          workload_identity_provider: ${{ vars.ATLAS_WIF_PROVIDER }}
+          service_account: ${{ vars.ATLAS_DEPLOYER_SERVICE_ACCOUNT }}
+      - name: Set up gcloud
+        uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db
+      - name: Ensure Composer environment
+        if: ${{ github.event.inputs.create_composer == 'true' }}
+        run: ATLAS_APPROVE_COMPOSER_CREATE=true bash scripts/manage_atlas_composer.sh create
+      - name: Build and upload immutable release
+        run: bash scripts/build_deployment_bundle.sh --upload
+      - name: Deploy release
+        run: |
+          flags=""
+          if [ "${{ github.event.inputs.leave_paused }}" = "true" ]; then
+            flags="--leave-paused"
+          fi
+          ATLAS_APPROVE_DEPLOY=true bash scripts/deploy_atlas_release.sh \
+            --git-sha "${{ steps.target.outputs.sha }}" $flags
+      - name: Upload deployment evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: deployment-evidence
+          path: |
+            dist/release-manifest-*.json
+            /tmp/smoke-warehouse.json
+          if-no-files-found: warn