test: add testme.json format validation in the pre-commit + improve the error messages of test.py + make it faster to start.

Author: svalat

.pre-commit-config.yaml

@@ -57,3 +57,23 @@ repos:
     hooks:
       - id: cpplint
         args: [--counting=detailed, --exclude=test/*]
+
+  - repo: https://github.com/python-jsonschema/check-jsonschema
+    rev: 0.38.0
+    hooks:
+    - id: check-jsonschema
+      files: ^test/.*/testme.json$
+      args: ["--schemafile", "pytools/testme.schema.json"]
+
+  - repo: local
+    hooks:
+    - id: pytest
+      name: testme.json validation
+      entry: ./test.py --validate-testme-jsons
+      language: python
+      files: ^test/.*/testme.json$
+      pass_filenames: false
+      additional_dependencies:
+        - matplotlib >= 3.5.0
+        - pytest >= 6.0
+        - jsonschema >= 4.26.0

pytools/idfx_test.py

@@ -141,6 +141,12 @@ def __init__(self, current_test_file, name=""):
             action="store_true",
         )
 
+        parser.add_argument(
+            "--validate-testme-jsons",
+            help="Only validate the testme.json files.",
+            action="store_true",
+        )
+
         parser.add_argument(
             "--help-pytest",
             help="Display the options you can transmit directly to pytest in addition to the specific to idefix tests.",

pytools/idfx_test_run.py

@@ -56,11 +56,28 @@ def _loadTestmeSchema(self) -> dict:
 
     def _validateNaming(self, namings: str, autoExtracted: str, file: str):
         names = namings.split(",")
-        for n in autoExtracted.split(","):
-            if not n in names:
-                raise Exception(
-                    f"Naming parameter list not match the auto-detectection, some are missinge. You gave : '{names}', detected '{autoExtracted}' into {file}"
-                )
+        names_sorted = namings.split(",")
+        autoNames = autoExtracted.split(",")
+        autoNames_sorted = autoExtracted.split(",")
+
+        # quick compare
+        names_sorted.sort()
+        autoNames_sorted.sort()
+
+        # check and if error check in details to report
+        if names_sorted != autoNames_sorted:
+            for n in autoNames:
+                if n not in names:
+                    raise Exception(
+                        f"Naming parameter list not match the auto-detectection, some are missing : '{n}'\n"
+                        f"You gave : '{names}', detected '{autoNames}' into {file}"
+                    )
+            for n in names:
+                if n not in autoNames:
+                    raise Exception(
+                        f"Naming parameter list not match the auto-detectection, some are missing : '{n}'.\n"
+                        f"You gave : '{names}', detected '{autoNames}' into {file}"
+                    )
 
     def _makeVariableArgAsList(self, namings: str, variants) -> list:
         # make as a list
@@ -155,7 +172,7 @@ def genTests(self) -> list:
                         )
                 except Exception as e:
                     raise Exception(
-                        f"Fail to generate tests from {testfileRelPath}"
+                        f"Fail to generate tests from {testfileRelPath}.\n{e}"
                     ) from e
 
         # ok
@@ -448,7 +465,29 @@ def _runNonRegSingleRun(
                     f"Dump file {file} was overwritten on restart"
                 )
 
+    def validateTestmeJsons(self) -> bool:
+        try:
+            self.genTests()
+        except Exception as e:
+            print(f"{e}")
+            return False
+        return True
+
     def main(self, all: bool = False):
+        # make testme.json validation for pre-commit
+        if "--validate-testme-jsons" in sys.argv:
+            try:
+                tests = self.genTests()
+                for test in tests:
+                    testfile = test.values[0]["testfile"]
+                    id = test.id
+                    print(f"{testfile} => {id}")
+            except Exception as e:
+                print(f"{e}")
+                sys.exit(1)
+            return
+
+        # run
         if all:
             sys.argv.append("-all")
         idefixTest = tst.idfxTest(self.parentScritFile, name="main")

test.py

@@ -18,6 +18,14 @@
 gblIdefixPytestRunner = IdexPytestRunner(__file__)
 
 
+# if called directly as a script
+if __name__ == "__main__" and "--validate-testme-jsons" in sys.argv:
+    if gblIdefixPytestRunner.validateTestmeJsons():
+        sys.exit(0)
+    else:
+        sys.exit(1)
+
+
 # define the pytest test
 @pytest.mark.parametrize("config", gblIdefixPytestRunner.genTests())
 def test_idefix_build_run_check(config):