Update pipeline workflow (#66)

* Update build.yml

run tests in separate folder

* Add names and deploy key

* Add warning annotations for GitHub Actions on build errors

* Fix syntax error, comment out debug statement

* Update build.py

* update annotation message

Author: ehennestad

.github/workflows/build.yml

@@ -10,7 +10,10 @@ on:
 
 jobs:
   build:
+    name: Build classes
     runs-on: ubuntu-latest
+    outputs:
+      build_updated: ${{ steps.check_changes.outputs.changes_detected }}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -25,35 +28,122 @@ jobs:
           pip install -r requirements.txt
           python build.py
 
-      - name: Checkout staging branch
-        uses: actions/checkout@v3
+      - name: Checkout main branch
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          path: main
+
+      - name: Copy changes to main
+        run: |
+          chmod +x ./.github/workflows/scripts/apply-new-build.sh
+          ./.github/workflows/scripts/apply-new-build.sh main
+
+      - name: Check if there are changes
+        id: check_changes
+        working-directory: main
+        run: |
+          if [[ $(git status --porcelain) ]]; then
+            echo "changes_detected=true" >> $GITHUB_OUTPUT
+          else
+            echo "::notice title=No changes detected:: No changes in build output compared to main branch"
+            echo "changes_detected=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Upload build (target) as artifact
+        if: steps.check_changes.outputs.changes_detected == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: build
+          path: target
+          retention-days: 1
+
+  test-code:
+    name: Test with updated classes
+    runs-on: ubuntu-latest
+    needs: build
+    if: needs.build.outputs.build_updated == 'true'
+    steps:
+      - name: Checkout main branch
+        uses: actions/checkout@v4
+        with:
+          ref: main
+
+      - name: Download build (target) artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: build
+          path: target
+
+      - name: Inject build into code directory
+        run: |
+          rm -rf code/{types,mixedtypes,enumerations}/*
+          cp -R target/* code
+          cp -R code/internal/resources/{content_files,readme_files}/* code/
+          rm -rf target
+
+      - name: Set up MATLAB
+        uses: matlab-actions/setup-matlab@v2
         with:
-          ref: staging
-          path: staging
-          token: ${{ secrets.GITHUB_TOKEN }}
+          release: latest
+          cache: true
 
-      - name: Push to staging
+      - name: Install MatBox
+        uses: ehennestad/matbox-actions/install-matbox@v1
+
+      - name: Run tests
+        id: test_step
+        uses: ehennestad/matbox-actions/test-code@v1
+        with:
+          source_directory: code
+          tests_directory: tools/tests
+          tools_directory: tools
+      
+      - name: Restore MATLAB path
+        uses: matlab-actions/run-command@v2
+        if: always()
+        with:
+          command: "restoredefaultpath(); savepath()"
+
+  push-changes:
+    name: Push updates to main
+    runs-on: ubuntu-latest
+    needs: [build, test-code]
+    if: needs.test-code.result == 'success' && needs.build.outputs.build_updated == 'true'
+    steps:
+      - name: Checkout main branch
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          ssh-key: ${{ secrets.DEPLOY_KEY }}
+
+      - name: Download build (target) artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: build
+          path: target
+
+      - name: Inject build into code directory
         run: |
-          rm -rf staging/code/types/*
-          rm -rf staging/code/mixedtypes/*
-          rm -rf staging/code/enumerations/*
-          cp -R target/* staging/code
-          cp -R staging/code/internal/resources/content_files/* staging/code/
-          cp -R staging/code/internal/resources/readme_files/* staging/code/
-          cd staging
+          rm -rf code/{types,mixedtypes,enumerations}/*
+          cp -R target/* code
+          cp -R code/internal/resources/{content_files,readme_files}/* code/
+          rm -rf target
 
+      - name: Push to main
+        run: |
           git config --global user.email "openminds@ebrains.eu"
           git config --global user.name "openMINDS pipeline"
-
+      
           # Conditionally set commit message based on the event
           if [ "${{ github.event_name }}" == "push" ]; then
-            commit_message="Build triggered by update to pipeline"
+            commit_message="Build triggered by update to build pipeline"
           elif [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
             commit_message="Build triggered by workflow dispatch"
           else
             commit_message="Unknown event"
           fi
-
+      
           # Only proceed with commit and push if changes are detected
           if [[ $(git add . --dry-run | wc -l) -gt 0 ]]; then
             git add .

.github/workflows/scripts/apply-new-build.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT=${1:?Please provide the root folder as the first argument}
+
+# Safety: prevent running with empty string or system root
+if [[ "$ROOT" == "" || "$ROOT" == "/" ]]; then
+    echo "Error: invalid root folder: '$ROOT'" >&2
+    exit 1
+fi
+
+rm -rf "$ROOT"/code/{types,mixedtypes,enumerations}/*
+cp -R target/* "$ROOT"/code/
+cp -R "$ROOT"/code/internal/resources/{content_files,readme_files}/* "$ROOT"/code/

build.py

@@ -32,7 +32,12 @@
         try:
             MATLABSchemaBuilder(schema_file_path, schema_root_path, class_name_map, jinja_templates).build()
         except Exception as e:
-            print(f"Error while building schema {schema_file_path}: {e}")
+            #print(f"Error while building schema {schema_file_path}: {e}")
+            # get relative path from schema_root_path to schema_file_path
+            relative_path = os.path.relpath(schema_file_path, schema_root_path)
+            schemaName = os.path.basename(schema_file_path)
+            schemaName = schemaName.replace(".schema.omi.json", "")
+            print(f"::warning file={relative_path},title=Error while building class for schema '{schemaName}' ({schema_version})::{e}")
 
     save_resource_files(schema_version, schemas_file_paths)
 

pipeline/translator.py

@@ -258,6 +258,7 @@ def _extract_template_variables(self):
 
         self._template_variables = {
             "class_name": class_name,
+            "full_class_name": _generate_class_name(schema[SCHEMA_PROPERTY_TYPE], self._class_name_map),
             "base_class": base_class,
             "has_controlled_instance": has_controlled_instance,
             "openminds_type": _expand_type_namespace( schema[SCHEMA_PROPERTY_TYPE] ),
@@ -338,6 +339,9 @@ def _generate_class_name(iri, class_name_map):
     # Ensure first letter of type_name is capitalized
     type_name = type_name[0].upper() + type_name[1:]
 
+    if type_name not in class_name_map:
+        raise KeyError(f"Class name '{type_name}' (IRI: {iri}) was not found in the map of all class names.")
+
     return class_name_map[type_name]
 
     #for i in range(len(parts) - 1):