Dev

.github/workflows/pipeline-dev-release.yml → .github/workflows/dev-pipeline-release.yaml

@@ -1,4 +1,4 @@
-name: dev release
+name: pipeline dev release
 on:
   workflow_run:
     workflows: ["Pipeline Unit Tests"]
@@ -27,8 +27,11 @@ jobs:
 
       - name: Build package
         run: |
-          cd loading_pipeline
-          rm -rf dist/
+          rm -rf pipeline_build
+          mkdir pipeline_build
+          cp -r loading_pipeline pipeline_build
+          mv pipeline_build/loading_pipeline/pyproject.toml pipeline_build
+          cd pipeline_build
           python -m pip install build
           python -m build
 

.github/workflows/pipeline-unit-tests.yml → .github/workflows/pipeline-unit-tests.yaml

@@ -9,7 +9,7 @@ on:
       - master
     paths:
       - 'loading_pipeline/**'
-      - '.github/workflows/pipeline-*-release.yaml'
+      - '.github/workflows/*-pipeline-release.yaml'
   pull_request:
     types: [opened, synchronize, reopened]
     paths:

.github/workflows/pipeline-prod-release.yml → .github/workflows/prod-pipeline-release.yaml

@@ -1,4 +1,4 @@
-name: prod release
+name: pipeline prod release
 on:
   workflow_run:
     workflows: ["Pipeline Unit Tests"]
@@ -28,8 +28,11 @@ jobs:
 
       - name: build package
         run: |
-          cd loading_pipeline
-          rm -rf dist/
+          rm -rf pipeline_build
+          mkdir pipeline_build
+          cp -r loading_pipeline pipeline_build
+          mv pipeline_build/loading_pipeline/pyproject.toml pipeline_build
+          cd pipeline_build
           python -m pip install build
           python -m build
 
@@ -48,9 +51,9 @@ jobs:
         run: |-
           gcloud storage rm -r gs://seqr-pipeline-runner-builds/prod/latest/ || echo 'No latest release'
           gcloud storage cp loading_pipeline/bin/* gs://seqr-pipeline-runner-builds/prod/latest/bin/
-          gcloud storage cp loading_pipeline/dist/*.whl gs://seqr-pipeline-runner-builds/prod/latest/pyscripts.zip
+          gcloud storage cp pipeline_build/dist/*.whl gs://seqr-pipeline-runner-builds/prod/latest/pyscripts.zip
           gcloud storage cp loading_pipeline/bin/* gs://seqr-pipeline-runner-builds/prod/${{ github.event.workflow_run.head_sha }}/bin/
-          gcloud storage cp loading_pipeline/dist/*.whl gs://seqr-pipeline-runner-builds/prod/${{ github.event.workflow_run.head_sha }}/pyscripts.zip
+          gcloud storage cp pipeline_build/dist/*.whl gs://seqr-pipeline-runner-builds/prod/${{ github.event.workflow_run.head_sha }}/pyscripts.zip
           gcloud storage cp -r loading_pipeline/var/vep/* gs://seqr-reference-data/vep/
 
   docker:

loading_pipeline/lib/tasks/dataproc/misc_test.py

@@ -31,8 +31,6 @@ def test_to_kebab_str_args(self, _: Mock):
                 'SNV_INDEL',
                 '--run-id',
                 'a_misc_run',
-                '--attempt-id',
-                '0',
                 '--sample-type',
                 'WGS',
                 '--callset-path',
@@ -47,5 +45,7 @@ def test_to_kebab_str_args(self, _: Mock):
                 '[]',
                 '--is-new-gcnv-joint-call',
                 'False',
+                '--attempt-id',
+                '0',
             ],
         )

loading_pipeline/lib/tasks/dataproc/run_pipeline_on_dataproc.py

@@ -7,10 +7,18 @@
     BaseRunJobOnDataprocTask,
 )
 from loading_pipeline.lib.tasks.run_pipeline import RunPipelineTask
+from loading_pipeline.lib.tasks.write_existing_variants_parquet import (
+    WriteExistingVariantsParquetTask,
+)
 
 
 @luigi.util.inherits(BaseLoadingRunParams)
 class RunPipelineOnDataprocTask(BaseRunJobOnDataprocTask):
+    attempt_id = luigi.IntParameter()
+
     @property
     def task(self) -> luigi.Task:
         return RunPipelineTask
+
+    def requires(self) -> [luigi.Task]:
+        return [self.clone(WriteExistingVariantsParquetTask), *super().requires()]

loading_pipeline/lib/tasks/dataproc/run_pipeline_on_dataproc_test.py

@@ -13,6 +13,9 @@
 from loading_pipeline.lib.test.mock_complete_task import MockCompleteTask
 
 
+@patch(
+    'loading_pipeline.lib.tasks.dataproc.run_pipeline_on_dataproc.WriteExistingVariantsParquetTask',
+)
 @patch(
     'loading_pipeline.lib.tasks.dataproc.base_run_job_on_dataproc.CreateDataprocClusterTask',
 )
@@ -26,6 +29,7 @@ def test_job_already_exists_failed(
         mock_logger: Mock,
         mock_job_controller_client: Mock,
         mock_create_dataproc_cluster: Mock,
+        mock_variants_parquet: Mock,
     ) -> None:
         mock_create_dataproc_cluster.return_value = MockCompleteTask()
         mock_client = mock_job_controller_client.return_value
@@ -63,6 +67,7 @@ def test_job_already_exists_success(
         self,
         mock_job_controller_client: Mock,
         mock_create_dataproc_cluster: Mock,
+        mock_variants_parquet: Mock,
     ) -> None:
         mock_create_dataproc_cluster.return_value = MockCompleteTask()
         mock_client = mock_job_controller_client.return_value
@@ -91,8 +96,10 @@ def test_job_failed(
         mock_logger: Mock,
         mock_job_controller_client: Mock,
         mock_create_dataproc_cluster: Mock,
+        mock_variants_parquet: Mock,
     ) -> None:
         mock_create_dataproc_cluster.return_value = MockCompleteTask()
+        mock_variants_parquet.return_value = MockCompleteTask()
         mock_client = mock_job_controller_client.return_value
         mock_client.get_job.side_effect = [
             google.api_core.exceptions.NotFound(
@@ -139,8 +146,10 @@ def test_job_success(
         mock_logger: Mock,
         mock_job_controller_client: Mock,
         mock_create_dataproc_cluster: Mock,
+        mock_variants_parquet: Mock,
     ) -> None:
         mock_create_dataproc_cluster.return_value = MockCompleteTask()
+        mock_variants_parquet.return_value = MockCompleteTask()
         mock_client = mock_job_controller_client.return_value
         mock_client.get_job.side_effect = [
             google.api_core.exceptions.NotFound(

loading_pipeline/pyproject.toml

@@ -19,7 +19,7 @@ dependencies = [
     "onnxconverter-common @ git+https://github.com/microsoft/onnxconverter-common.git@f7a8eb699caa6f6a78d3bdfbcaf5dfbd43569ebd",
 
     # Dataproc
-    "pip==23.2.1",
+    "pip==26.2",
     "google-cloud-dataproc==5.16.0",
     "google-cloud-bigquery==3.29.0",
     "google-api-python-client==2.159.0",