Dev

.cloudbuild/pipeline-docker.cloudbuild.yaml

@@ -0,0 +1,11 @@
+steps:
+- name: 'gcr.io/kaniko-project/executor:v1.3.0'
+  args:
+  - --destination=gcr.io/seqr-project/seqr-pipeline-runner:${COMMIT_SHA}
+  - --destination=gcr.io/seqr-project/seqr-pipeline-runner:${_CUSTOM_BRANCH_TAG}
+  - --dockerfile=loading_pipeline/deploy/Dockerfile
+  - --cache=true
+  - --cache-ttl=168h
+  - --build-arg=PIPELINE_RUNNER_APP_VERSION=${COMMIT_SHA}
+
+timeout: 1800s

.cloudbuild/vep-docker.cloudbuild.yaml

@@ -0,0 +1,12 @@
+# Run locally with:
+#
+# gcloud builds submit --quiet --substitutions='_REFERENCE_GENOME=GRCh38' --config .cloudbuild/vep-docker.cloudbuild.yaml loading_pipeline/deploy
+steps:
+- name: 'gcr.io/kaniko-project/executor:v1.3.0'
+  args:
+  - --destination=gcr.io/seqr-project/vep-docker-image:${_REFERENCE_GENOME}
+  - --dockerfile=Dockerfile.vep_${_REFERENCE_GENOME}
+  - --cache=true
+  - --cache-ttl=168h
+
+timeout: 1800s

.codacy.yaml

@@ -1,3 +1,4 @@
 ---
 exclude_paths:
-  - "deploy/postgres/initdb.sql"
+  - "deploy/postgres/initdb.sql"
+  - "loading_pipeline/deploy/Dockerfile"

.dockerignore

@@ -6,6 +6,7 @@ static/*
 deploy/*
 !deploy/docker/seqr/*
 !deploy/docker/vlm/*
+loading_pipeline/*test*
 .git
 .vscode
 .idea

.github/workflows/pipeline-dev-release.yml

@@ -0,0 +1,74 @@
+name: dev release
+on:
+  workflow_run:
+    workflows: ["Pipeline Unit Tests"]
+    types:
+      - completed
+    branches:
+      - dev
+
+permissions:
+  id-token: write
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.workflow_run.head_branch }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11' 
+
+      - name: Build package
+        run: |
+          cd loading_pipeline
+          rm -rf dist/
+          python -m pip install build
+          python -m build
+
+      - name: authenticate to google cloud
+        id: "auth"
+        uses: google-github-actions/auth@v0
+        with:
+          workload_identity_provider: "${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}"
+          service_account: "${{ secrets.RUN_SA_EMAIL }}"
+
+      - name: "setup gcloud sdk"
+        uses: google-github-actions/setup-gcloud@v0
+
+      - name: Copy files to release directory
+        run: |-
+          gcloud storage rm -r gs://seqr-pipeline-runner-builds/dev/latest/ || echo 'No latest release'
+          gcloud storage cp loading_pipeline/bin/* gs://seqr-pipeline-runner-builds/dev/latest/bin/
+          gcloud storage cp cd loading_pipeline/dist/*.whl gs://seqr-pipeline-runner-builds/dev/latest/pyscripts.zip
+          gcloud storage cp loading_pipeline/bin/* gs://seqr-pipeline-runner-builds/dev/${{ github.event.workflow_run.head_sha }}/bin/
+          gcloud storage cp cd loading_pipeline/dist/*.whl gs://seqr-pipeline-runner-builds/dev/${{ github.event.workflow_run.head_sha }}/pyscripts.zip
+  docker:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.workflow_run.head_branch }}
+
+      - name: authenticate to google cloud
+        id: "auth"
+        uses: google-github-actions/auth@v3
+        with:
+          workload_identity_provider: "${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}"
+          service_account: "${{ secrets.RUN_SA_EMAIL }}"
+
+      - name: "setup gcloud sdk"
+        uses: google-github-actions/setup-gcloud@v2
+
+      - name: Build and push images
+        run: |-
+          gcloud builds submit --quiet --substitutions="COMMIT_SHA=${{ github.event.workflow_run.head_sha }},_CUSTOM_BRANCH_TAG=dev" --config .cloudbuild/pipeline-docker.cloudbuild.yaml --gcs-log-dir=gs://seqr-github-actions-logs/logs .
+

.github/workflows/pipeline-prod-release.yml

@@ -0,0 +1,121 @@
+name: prod release
+on:
+  workflow_run:
+    workflows: ["Pipeline Unit Tests"]
+    types:
+      - completed
+    branches:
+      - master
+
+permissions:
+  id-token: write
+  contents: write
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.workflow_run.head_branch }}
+
+      - name: set up python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: build package
+        run: |
+          cd loading_pipeline
+          rm -rf dist/
+          python -m pip install build
+          python -m build
+
+      - name: authenticate to google cloud
+        id: "auth"
+        uses: google-github-actions/auth@v3
+        with:
+          workload_identity_provider: "${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}"
+          service_account: "${{ secrets.RUN_SA_EMAIL }}"
+
+      - name: "setup gcloud sdk"
+        uses: google-github-actions/setup-gcloud@v2
+
+      - name: Copy files to release directory
+        shell: bash
+        run: |-
+          gcloud storage rm -r gs://seqr-pipeline-runner-builds/prod/latest/ || echo 'No latest release'
+          gcloud storage cp loading_pipeline/bin/* gs://seqr-pipeline-runner-builds/prod/latest/bin/
+          gcloud storage cp loading_pipeline/dist/*.whl gs://seqr-pipeline-runner-builds/prod/latest/pyscripts.zip
+          gcloud storage cp loading_pipeline/bin/* gs://seqr-pipeline-runner-builds/prod/${{ github.event.workflow_run.head_sha }}/bin/
+          gcloud storage cp loading_pipeline/dist/*.whl gs://seqr-pipeline-runner-builds/prod/${{ github.event.workflow_run.head_sha }}/pyscripts.zip
+          gcloud storage cp -r loading_pipeline/var/vep/* gs://seqr-reference-data/vep/
+
+  docker:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.workflow_run.head_branch }}
+
+      - name: authenticate to google cloud
+        id: "auth"
+        uses: google-github-actions/auth@v3
+        with:
+          workload_identity_provider: "${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}"
+          service_account: "${{ secrets.RUN_SA_EMAIL }}"
+
+      - name: "setup gcloud sdk"
+        uses: google-github-actions/setup-gcloud@v2
+
+      - name: Build and push images
+        run: |-
+          gcloud builds submit --quiet --substitutions="COMMIT_SHA=${{ github.event.workflow_run.head_sha }},_CUSTOM_BRANCH_TAG=latest" --config .cloudbuild/pipeline-docker.cloudbuild.yaml --gcs-log-dir=gs://seqr-github-actions-logs/logs .
+
+  helm_update:
+    runs-on: ubuntu-latest
+    needs: docker
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: broadinstitute/seqr-helm
+          ref: main
+          persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
+          fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
+
+      - name: Get latest pipeline-runner version
+        uses: mikefarah/yq@v4.22.1
+        id: current
+        with:
+          cmd: >
+            yq -r '.version' charts/pipeline-runner/Chart.yaml
+
+      - name: Bump version
+        id: bump
+        uses: cbrgm/semver-bump-action@main
+        with:
+          current-version: ${{ steps.current.outputs.result }}
+          bump-level: minor
+
+      - name: Update appVersion and version in pipeline-runner Chart file
+        uses: mikefarah/yq@v4.22.1
+        with:
+          cmd: >
+            yq -i '
+            .appVersion = "${{ github.event.workflow_run.head_sha }}" |
+            .version = "${{ steps.bump.outputs.new_version }}"
+            ' charts/pipeline-runner/Chart.yaml
+
+      - name: Commit and Push changes
+        uses: Andro999b/push@v1.3
+        with:
+          repository: broadinstitute/seqr-helm
+          branch: main
+          github_token: ${{ secrets.SEQR_VERSION_UPDATE_TOKEN }}
+          author_email: ${{ github.actor }}@users.noreply.github.com
+          author_name: tgg-automation
+          message: "Update pipeline-runner chart appVersion to ${{ github.event.workflow_run.head_sha }}"

.github/workflows/pipeline-unit-tests.yml

@@ -0,0 +1,91 @@
+name: Pipeline Unit Tests
+
+# Run the test suite on pushes (incl. merges) to master and dev
+# Run the test suite when a PR is opened, pushed to, or reopened
+on:
+  push:
+    branches:
+      - dev
+      - master
+    paths:
+      - 'loading_pipeline/**'
+      - '.github/workflows/pipeline-*-release.yaml'
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - 'loading_pipeline/**'
+      - '.github/workflows/*pipeline*.yaml'
+      - 'clickhouse_search/migrations/*.py'
+
+permissions:
+  contents: read
+
+jobs:
+  unit_tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.11']
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Create ClickHouse Config
+        run: |
+          cat <<EOF > test_config.xml
+          <clickhouse>
+            <user_files_path>/tmp</user_files_path>
+          </clickhouse>
+          EOF
+      - name: Start ClickHouse with mounted config
+        run: |
+          docker run -d --name clickhouse \
+            -v ${{ github.workspace }}/test_config.xml:/etc/clickhouse-server/config.d/config.xml \
+            -v /tmp:/tmp \
+            -p 9000:9000 \
+            -e CLICKHOUSE_USER=default \
+            -e CLICKHOUSE_PASSWORD=default_password \
+            -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 \
+            gcr.io/seqr-project/seqr-clickhouse:26.3.9
+      - name: Wait for ClickHouse
+        run: |
+          until docker exec clickhouse clickhouse-client --query "SELECT 1"; do
+            echo "Waiting for ClickHouse to start..."
+            sleep 2
+          done
+      - name: Set up ClickHouse Settings
+        run: |
+          docker exec clickhouse clickhouse-client --query "CREATE SETTINGS PROFILE clickhouse_settings SETTINGS flatten_nested=0, join_use_nulls=1, stop_refreshable_materialized_views_on_startup=1, allow_materialized_view_with_bad_select=1, enable_join_runtime_filters=0 TO default"
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      # Install uv
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          cache-dependency-glob: |
+            **/uv.lock
+            **/pyproject.toml
+      - name: Sync dependencies (prod + dataproc)
+        run: uv sync --locked  --project loading_pipeline
+      - name: Check Ruff Format
+        run: |
+          cd loading_pipeline
+          uv run ruff format --check . --diff
+      - name: Check Ruff
+        run: |
+          cd loading_pipeline
+          uv run ruff check --output-format github
+      - name: Run ShellCheck
+        uses: ludeeus/action-shellcheck@master
+        with:
+          severity: error
+          scandir: './loading_pipeline/bin'
+      - name: Unit Tests
+        run: |
+          export CLICKHOUSE_OPTIMIZE_TABLE_WAIT_S=1
+          export PYSPARK_SUBMIT_ARGS='--driver-memory 8G pyspark-shell'
+          export CLICKHOUSE_DATABASE=test
+          uv run --project loading_pipeline pytest --cov=loading_pipeline --cov-report=term-missing loading_pipeline
+          uv run --project loading_pipeline coverage report --omit '*test*' --fail-under=85

.github/workflows/unit-tests.yml

@@ -8,11 +8,15 @@ on:
       - master
       - dev
     paths-ignore:
+      - 'loading_pipeline/**'
+      - '.github/workflows/*pipeline*.yaml'
       - 'vlm/**'
       - '.github/workflows/*vlm*.yaml'
   pull_request:
     types: [opened, synchronize, reopened]
     paths-ignore:
+      - 'loading_pipeline/**'
+      - '.github/workflows/*pipeline*.yaml'
       - 'vlm/**'
       - '.github/workflows/*vlm*.yaml'
 

.github/workflows/vlm-unit-tests.yaml

@@ -113,7 +113,7 @@ jobs:
           export CLICKHOUSE_WRITER_PASSWORD=clickhouse_test_password
           export CLICKHOUSE_SERVICE_HOSTNAME=localhost
           export POSTGRES_SERVICE_HOSTNAME=localhost
-          python3 -m pytest --setup-only --create-db --reuse-db --ds=settings
+          python3 -m pytest --setup-only --create-db --reuse-db --ds=settings vlm
       - name: Set up Clickhouse test user
         run: |
           docker exec clickhouse clickhouse-client --query "GRANT SELECT ON test_seqr.\`GRCh37/SNV_INDEL/key_lookup\` TO vlm_test_user"