AzureMLOpsInProduction/.github/workflows/task_aml_mlops_e2e.yml at main · GrannyProgramming/AzureMLOpsInProduction · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
name: Azure ML Workflow
on:
  push:
    branches:
      - main

jobs:
  build:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        environment: [dev] #[dev, test, prod]
    steps:
      - name: Checkout Code
        uses: actions/checkout@v3

      - name: Set Up Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: 3.9

      - name: Install Dependencies
        run: python3 .github/utils/install_dependencies.py

      - name: Build Wheel And Install Helper Functions
        run: |
          cd .github/utils/py_wheels
          python3 setup.py bdist_wheel
          for whl in $(find dist -name "*.whl"); do python -m pip install $whl; done

      - name: Json Schema Validation
        run: |
          python3 .github/utils/json_schema_validator.py variables/${{ matrix.environment }}

      - name: Set Env Variables
        run: |
          python3 .github/utils/set_env_variables.py variables/${{ matrix.environment }}/parameters/parameters.json "['workspace_name=mlw-amcg0002-gwxp', 'resource_group=aml-dev-rg-001']"

      - name: Azure Login
        run: |
          echo ${{ secrets.ARM_CLIENT_ID }}
          az login --service-principal \
          --username "${{ secrets.ARM_CLIENT_ID }}" \
          --password "${{ secrets.ARM_CLIENT_SECRET }}" \
          --tenant "${{ secrets.ARM_TENANT_ID }}"

      # - name: Compile & Deploy Bicep Templates
      #   run: |
      #     bicep build ./infra/main.bicep
      #     python3 .github/utils/create_azure_resources.py

      # Create Cluster  - Done, having issues with creating the kubernetes compute and AKS compute. Create seperate script for this. @GrannyProgramming
      # - name: Create AML Instance/Clusters
      #   env:
      #     ENVIRONMENT: ${{ matrix.environment }}
      #   run: |
      #     python3 mlops/sdkv2/create_compute.py

      # This step is optional and relevant to the quickstart.
      # If data already exists elsewhere reference it in the data asset step and skip this one.
      # May run this as a standalone workflow, for now, fine as is @GrannyProgramming
      # - name: Create An Mltable
      #   env:
      #     ENVIRONMENT: ${{ matrix.environment }}
      #   run: |
      #     python3 dataEngineer/nyc_taxi/src/createMlTable.py dataEngineer/nyc_taxi/data_mltable

      # Create data asseets
      # @GrannyProgramming should I leave the unused file types in the json?
      # - name: Create Data Assets
      #   env:
      #     ENVIRONMENT: ${{ matrix.environment }}
      #   run: |
      #     python3 mlops/sdkv2/create_data_asset.py variables/dev/data_assests/data_assets.json

      # @grannyProgramming - should I force a environment update if the version is updated but the dependencies are the same?
      # @GrannyProgramming - Not sure if I can use the same logic for dockerfiles as I did for the conda file
      # - name: Create AML Environments
      #   run: |
      #     python3 mlops/sdkv2/create_environments.py variables/${{ matrix.environment }}/environments/environments.json

      - name: Create Components
        run: |
          python3 mlops/sdkv2/create_component.py variables/${{ matrix.environment }}/components/nyc_taxi/comp_multiple.yml

      - name: Create Pipelines
        run: |
          python3 mlops/sdkv2/create_pipeline.py variables/${{ matrix.environment }}/pipelines/nyc_taxi/pipelines.json


      ## Monitoring stuff, alerts not working yet, everything is gucchi
      # - name: Create action groups
      #   run:  python3 mlops/monitoring/azure_monitor/create_action_groups.py variables/${{ matrix.environment }}/action_groups/action_groups.json

      # - name: Create alert processing rules based on severity
      #   run:  mlops/monitoring/azure_monitor/alert_processing_rules.py variables/${{ matrix.environment }}/action_groups/action_groups.json

      # - name: Create AML Alerts
      #   run:  mlops/monitoring/azure_monitor/create_alerts.py variables/${{ matrix.environment }}/alerts/alerts.json