test: add usage of jsonschema to validate the testme.json file and get more understandable errors.

.pre-commit-config.yaml

@@ -57,3 +57,23 @@ repos:
     hooks:
       - id: cpplint
         args: [--counting=detailed, --exclude=test/*]
+
+  - repo: https://github.com/python-jsonschema/check-jsonschema
+    rev: 0.38.0
+    hooks:
+    - id: check-jsonschema
+      files: ^test/.*/testme.json$
+      args: ["--schemafile", "pytools/testme.schema.json"]
+
+  - repo: local
+    hooks:
+    - id: pytest
+      name: testme.json validation
+      entry: ./test.py --validate-testme-jsons
+      language: python
+      files: ^test/.*/testme.json$
+      pass_filenames: false
+      additional_dependencies:
+        - matplotlib >= 3.5.0
+        - pytest >= 6.0
+        - jsonschema >= 4.26.0

pytools/idfx_test.py

@@ -141,6 +141,12 @@ def __init__(self, current_test_file, name=""):
             action="store_true",
         )
 
+        parser.add_argument(
+            "--validate-testme-jsons",
+            help="Only validate the testme.json files.",
+            action="store_true",
+        )
+
         parser.add_argument(
             "--help-pytest",
             help="Display the options you can transmit directly to pytest in addition to the specific to idefix tests.",

pytools/idfx_test_run.py

@@ -13,6 +13,7 @@
 import sys
 from contextlib import contextmanager
 
+import jsonschema
 import pytest
 
 # idefix test class
@@ -45,14 +46,38 @@ def __init__(self, parentScriptFile: str):
         self.currentTestRunner: tst.idfxTest = None
         self.parentScritFile = parentScriptFile
         self.filterSubdir = os.environ.get("IDEFIX_TEST_FILTER_SUBDIR", "./test/")
+        self.testmeSchema = self._loadTestmeSchema()
+
+    def _loadTestmeSchema(self) -> dict:
+        scriptDir = os.path.dirname(__file__)
+        schemaPath = os.path.join(scriptDir, "testme.schema.json")
+        with open(schemaPath, "r") as fp:
+            return json.load(fp)
 
     def _validateNaming(self, namings: str, autoExtracted: str, file: str):
         names = namings.split(",")
-        for n in autoExtracted.split(","):
-            if not n in names:
-                raise Exception(
-                    f"Naming parameter list not match the auto-detectection, some are missinge. You gave : '{names}', detected '{autoExtracted}' into {file}"
-                )
+        names_sorted = namings.split(",")
+        autoNames = autoExtracted.split(",")
+        autoNames_sorted = autoExtracted.split(",")
+
+        # quick compare
+        names_sorted.sort()
+        autoNames_sorted.sort()
+
+        # check and if error check in details to report
+        if names_sorted != autoNames_sorted:
+            for n in autoNames:
+                if n not in names:
+                    raise Exception(
+                        f"Naming parameter list not match the auto-detectection, some are missing : '{n}'\n"
+                        f"You gave : '{names}', detected '{autoNames}' into {file}"
+                    )
+            for n in names:
+                if n not in autoNames:
+                    raise Exception(
+                        f"Naming parameter list not match the auto-detectection, some are missing : '{n}'.\n"
+                        f"You gave : '{names}', detected '{autoNames}' into {file}"
+                    )
 
     def _makeVariableArgAsList(self, namings: str, variants) -> list:
         # make as a list
@@ -98,6 +123,23 @@ def genTests(self) -> list:
                     with open(testfilePath, "r") as fp:
                         # load
                         test = json.load(fp)
+
+                        # validate via jsonschema
+                        try:
+                            jsonschema.validate(instance=test, schema=self.testmeSchema)
+                        except jsonschema.ValidationError as e:
+                            print()
+                            print(
+                                "--------------- testme.json format error ---------------"
+                            )
+                            print(f"Invalid format for : {testfilePath}\n{e}")
+                            print(
+                                "--------------------------------------------------------"
+                            )
+                            print()
+                            raise Exception(f"{testfilePath} format error !") from e
+
+                        # build generator
                         idefixTestGenerator = IdefixDirTestGenerator(
                             testfilePath, testfileDir
                         )
@@ -130,7 +172,7 @@ def genTests(self) -> list:
                         )
                 except Exception as e:
                     raise Exception(
-                        f"Fail to generate tests from {testfileRelPath}"
+                        f"Fail to generate tests from {testfileRelPath}.\n{e}"
                     ) from e
 
         # ok
@@ -423,7 +465,29 @@ def _runNonRegSingleRun(
                     f"Dump file {file} was overwritten on restart"
                 )
 
+    def validateTestmeJsons(self) -> bool:
+        try:
+            self.genTests()
+        except Exception as e:
+            print(f"{e}")
+            return False
+        return True
+
     def main(self, all: bool = False):
+        # make testme.json validation for pre-commit
+        if "--validate-testme-jsons" in sys.argv:
+            try:
+                tests = self.genTests()
+                for test in tests:
+                    testfile = test.values[0]["testfile"]
+                    id = test.id
+                    print(f"{testfile} => {id}")
+            except Exception as e:
+                print(f"{e}")
+                sys.exit(1)
+            return
+
+        # run
         if all:
             sys.argv.append("-all")
         idefixTest = tst.idfxTest(self.parentScritFile, name="main")

pytools/testme.schema.json

@@ -0,0 +1,157 @@
+{
+  "title": "testme",
+  "description": "Schema to validate the testme.json files in the test suite of Idefix.",
+  "type": "object",
+  "properties": {
+    "namings": {
+      "type": "string"
+    },
+    "default": { "$ref": "#/$defs/config" },
+    "variants": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "ini": {
+            "anyOf": [
+              { "type": "string" },
+              { "type": "array", "items": { "type": "string" } }
+            ]
+          },
+          "definitionFile": {
+            "anyOf": [
+              { "type": "string" },
+              { "type": "array", "items": { "type": "string" } }
+            ]
+          },
+          "dumpname": {
+            "anyOf": [
+              { "type": "string" },
+              { "type": "array", "items": { "type": "string" } }
+            ]
+          },
+          "tolerance": {
+            "anyOf": [
+              { "type": "number" },
+              { "type": "array", "items": { "type": "number" } }
+            ]
+          },
+          "nonRegressionTest": {
+            "anyOf": [
+              { "type": "boolean" },
+              { "type": "array", "items": { "type": "boolean" } }
+            ]
+          },
+          "standardTest": {
+            "anyOf": [
+              { "type": "boolean" },
+              { "type": "array", "items": { "type": "boolean" } }
+            ]
+          },
+          "noplot": {
+            "anyOf": [
+              { "type": "boolean" },
+              { "type": "array", "items": { "type": "boolean" } }
+            ]
+          },
+          "reconstruction": {
+            "anyOf": [
+              { "type": "integer" },
+              { "type": "array", "items": { "type": "integer" } }
+            ]
+          },
+          "dec": { "type": "array", "items": { "type": "number"} },
+          "single": {
+            "anyOf": [
+              { "type": "boolean" },
+              { "type": "array", "items": { "type": "boolean" } }
+            ]
+          },
+          "vectPot": {
+            "anyOf": [
+              { "type": "boolean" },
+              { "type": "array", "items": { "type": "boolean" } }
+            ]
+          },
+          "mpi": {
+            "anyOf": [
+              { "type": "boolean" },
+              { "type": "array", "items": { "type": "boolean" } }
+            ]
+          },
+          "mpiGpuForceCopy": {
+            "anyOf": [
+              { "type": "boolean" },
+              { "type": "array", "items": { "type": "boolean" } }
+            ]
+          },
+          "mpiMode": {
+            "anyOf": [
+              { "enum": ["Default", "Persistent", "Blocking", "NonBlocking" ] },
+              { "type": "array", "items": { "enum": ["Default", "Persistent", "Blocking", "NonBlocking" ] } }
+            ]
+          },
+          "restart": {
+            "anyOf": [
+              { "type": "boolean" },
+              { "type": "array", "items": { "type": "boolean" } }
+            ]
+          },
+          "callPyFunctionBefore": {
+            "anyOf": [
+              { "type": "string" },
+              { "type": "array", "items": { "type": "string" } }
+            ]
+          },
+          "callPyFunctionAfter": {
+            "anyOf": [
+              { "type": "string" },
+              { "type": "array", "items": { "type": "string" } }
+            ]
+          },
+          "check_file_produced": { "type": "array", "items": { "type": "string"} },
+          "restart_no_overwrite": { "type": "array", "items": { "type": "string"} },
+          "multirun": {
+            "type": "array",
+            "items": { "$ref": "#/$defs/config" }
+          }
+        }
+      }
+    }
+  },
+  "when": {
+    "type": "object",
+    "properties": {
+      "condition": { "$ref": "#/$defs/config" },
+      "apply": { "$ref": "#/$defs/config" }
+    }
+  },
+  "$defs": {
+    "config": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "ini": { "type": "string" },
+        "definitionFile": { "type": "string" },
+        "dumpname": { "type": "string" },
+        "tolerance": { "type": "number" },
+        "nonRegressionTest": { "type": "boolean" },
+        "standardTest": { "type": "boolean" },
+        "noplot": { "type": "boolean" },
+        "reconstruction": { "type": "integer" },
+        "dec": { "type": "array", "items": { "type": "number"} },
+        "single": { "type": "boolean" },
+        "vectPot": { "type": "boolean" },
+        "mpi": { "type": "boolean" },
+        "mpiGpuForceCopy": { "type": "boolean" },
+        "mpiMode": { "enum": ["Default", "Persistent", "Blocking", "NonBlocking" ] },
+        "restart": { "type": "boolean" },
+        "callPyFunctionBefore": { "type": "string" },
+        "callPyFunctionAfter": { "type": "string" },
+        "check_file_produced": { "type": "array", "items": { "type": "string"} },
+        "restart_no_overwrite": { "type": "array", "items": { "type": "string"} }
+      }
+    }
+  }
+}

test.py

@@ -18,6 +18,14 @@
 gblIdefixPytestRunner = IdexPytestRunner(__file__)
 
 
+# if called directly as a script
+if __name__ == "__main__" and "--validate-testme-jsons" in sys.argv:
+    if gblIdefixPytestRunner.validateTestmeJsons():
+        sys.exit(0)
+    else:
+        sys.exit(1)
+
+
 # define the pytest test
 @pytest.mark.parametrize("config", gblIdefixPytestRunner.genTests())
 def test_idefix_build_run_check(config):

test/HD/ShearingBox/testme.json

@@ -6,7 +6,7 @@
     "reconstruction": 2,
     "single": false,
     "mpi": false,
-    "dec": ["2","1","2"],
+    "dec": [2,1,2],
     "tolerance": 1e-15
   },
   "variants": [

test/HD/ViscousDisk/testme.json

@@ -6,7 +6,7 @@
     "reconstruction": 2,
     "single": false,
     "mpi": false,
-    "dec": ["2","1","2"],
+    "dec": [2,1,2],
     "tolerance": 3e-15
   },
   "variants": [

test/HD/ViscousFlowPastCylinder/testme.json

@@ -5,7 +5,7 @@
     "noplot": true,
     "reconstruction": 2,
     "single": false,
-    "dec": ["2","2"],
+    "dec": [2,2],
     "tolerance": 3e-14
   },
   "variants": [

test/HD/thermalDiffusion/testme.json

@@ -6,7 +6,7 @@
     "reconstruction": 2,
     "single": false,
     "mpi": false,
-    "dec": ["2","1","2"],
+    "dec": [2,1,2],
     "tolerance": 0
   },
   "variants": [

test/MHD/AmbipolarCshock3D/testme.json

@@ -12,12 +12,12 @@
       "ini": ["idefix.ini","idefix-rkl.ini"],
       "mpi": [false],
       "vectPot": [false, true],
-      "dec": ["2","1","1"]
+      "dec": [2,1,1]
     },{
       "ini": ["idefix.ini","idefix-rkl.ini"],
       "mpi": [true],
       "vectPot": [false],
-      "dec": ["2","1","1"],
+      "dec": [2,1,1],
       "tolerance": 3e-14
     }
   ],

test/MHD/FargoMHDSpherical/testme.json

@@ -5,7 +5,7 @@
     "noplot": true,
     "single": false,
     "reconstruction": 2,
-    "dec": ["2","2","2"],
+    "dec": [2,2,2],
     "standardTest": false,
     "tolerance": 1e-14
   },

test/MHD/LinearWaveTest/testme.json

@@ -3,7 +3,7 @@
   "default": {
     "dumpname": "dump.0001.dmp",
     "noplot": true,
-    "dec": ["2","2","2"],
+    "dec": [2,2,2],
     "tolerance": 2e-13
   },
   "variants": [