Move to pytest as the official and default way to run tests

.github/workflows/tests.yml

@@ -54,12 +54,10 @@ jobs:
       - name: Ensure browser is installed
         if: steps.playwright-cache.outputs.cache-hit != 'true'
         run: python -m playwright install --with-deps chromium
-      - name: Launch the API
-        run: coverage run run_studio.py &
-      - name: Run dev suite
-        run: coverage run run_tests.py dev
-      - name: Run test-studio
-        run: coverage run g2p/tests/test_studio.py
+      - name: Run most tests
+        # This will skip neural test, not having installed .[neural] yet
+        # This will also skip updating schemas, having installed pydantic>=2.9
+        run: coverage run -m pytest
 
       # Neural tests, with huggingface models cached
       - uses: actions/cache@v5
@@ -87,8 +85,6 @@ jobs:
 
       - name: Post test analyses
         run: |
-          pkill -INT coverage
-          sleep 10
           coverage combine
           coverage report
           coverage xml

Contributing.md

@@ -174,4 +174,4 @@ Then you can run all the test suites by simply invoking `pytest`:
 
     pytest
 
-We also support `unittest` if you're familiar with that, and we have a `./run_tests.py` script to run selected parts of the test suites. Run `./run_tests.py -h` for info.
+We also have a `./run_tests.py` script to run selected parts of the test suites. Run `./run_tests.py -h` for info.

g2p/tests/run.py

@@ -9,127 +9,96 @@
 """
 
 import argparse
-import os
-import re
+import io
 import sys
-from unittest import TestLoader, TestSuite, TextTestRunner
+from contextlib import redirect_stdout
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import pytest
 
 # Unit tests
 from g2p.log import LOGGER
-from g2p.tests.test_check_ipa_arpabet import CheckIpaArpabetTest
-from g2p.tests.test_cli import CliTest
-from g2p.tests.test_create_mapping import MappingCreationTest
-from g2p.tests.test_doctor import DoctorTest
-from g2p.tests.test_doctor_expensive import ExpensiveDoctorTest
-from g2p.tests.test_fallback import FallbackTest
-from g2p.tests.test_indices import IndicesTest
-from g2p.tests.test_langs import LangTest
-from g2p.tests.test_lexicon_transducer import LexiconTransducerTest
-from g2p.tests.test_mappings import MappingTest
-from g2p.tests.test_network import NetworkLiteTest, NetworkTest
-from g2p.tests.test_neural import NeuralLangTest
-from g2p.tests.test_tokenize_and_map import TokenizeAndMapTest
-from g2p.tests.test_tokenizer import TokenizerTest
-from g2p.tests.test_transducer import TransducerTest
-from g2p.tests.test_unidecode_transducer import UnidecodeTransducerTest
-from g2p.tests.test_utils import UtilsTest
-from g2p.tests.test_z_local_config import LocalConfigTest
-
-if sys.version_info >= (3, 8, 0):
-    from g2p.tests.test_api_resources import ResourceIntegrationTest
-    from g2p.tests.test_api_v2 import TestAPIV2
-
-    API_TEST_CLASSES = [ResourceIntegrationTest, TestAPIV2]
-else:
-    API_TEST_CLASSES = []
-
-LOADER = TestLoader()
-
-TRANSDUCER_TESTS = [
-    LOADER.loadTestsFromTestCase(test)
-    for test in [
-        IndicesTest,
-        TransducerTest,
-        UnidecodeTransducerTest,
-        LexiconTransducerTest,
-    ]
-]
-
-MAPPINGS_TESTS = [
-    LOADER.loadTestsFromTestCase(test)
-    for test in [
-        FallbackTest,
-        MappingCreationTest,
-        MappingTest,
-        NetworkTest,
-        NetworkLiteTest,
-        UtilsTest,
-        TokenizerTest,
-        TokenizeAndMapTest,
-        CheckIpaArpabetTest,
-    ]
-]
-
-LANGS_TESTS = [
-    LOADER.loadTestsFromTestCase(test)
-    for test in [
-        LangTest,
-    ]
-]
-
-# We are excluding these tests from the dev suite because
-# they require torch and other heavy dependencies
-# and the tests also require downloading large g2p models
-NEURAL_TESTS = [LOADER.loadTestsFromTestCase(test) for test in [NeuralLangTest]]
-
-INTEGRATION_TESTS = [
-    LOADER.loadTestsFromTestCase(test)
-    for test in [
-        CliTest,
-        DoctorTest,
-        ExpensiveDoctorTest,
-    ]
-    + API_TEST_CLASSES
-]
 
+SUITES: Dict[str, List[str]] = {
+    "all": [],  # empty list triggers complete test discovery
+    "dev": [],  # updated below this block
+    "api": ["test_api_resources", "test_api_v2"],
+    "integ": ["test_cli", "test_doctor", "test_doctor_expensive"],  # updated below
+    "langs": ["test_langs"],
+    "mappings": [
+        "test_fallback",
+        "test_create_mapping",
+        "test_mappings",
+        "test_network",
+        "test_utils",
+        "test_tokenizer",
+        "test_tokenize_and_map",
+        "test_check_ipa_arpabet",
+    ],
+    "trans": [
+        "test_indices",
+        "test_transducer",
+        "test_unidecode_transducer",
+        "test_lexicon_transducer",
+    ],
+    # Neural tests are excluded from dev and automatically skipped if neural
+    # dependencies are not installed, because they require torch and other heavy
+    # dependencies and the tests also require downloading large g2p models
+    "neural": ["test_neural"],
+    # Studio is also expensive and excluded from dev
+    "studio": ["test_studio"],
+}
+SUITES["dev"] = sum(
+    [SUITES[suite] for suite in ("api", "integ", "langs", "trans", "mappings")],
+    start=[],
+)
 # LocalConfigTest has to get run last, to avoid interactions with other test
 # cases, since it has side effects on the global database
-LAST_DEV_TEST = [
-    LOADER.loadTestsFromTestCase(test)
-    for test in [
-        LocalConfigTest,
-    ]
-]
-
-DEV_TESTS = (
-    TRANSDUCER_TESTS + MAPPINGS_TESTS + LANGS_TESTS + INTEGRATION_TESTS + LAST_DEV_TEST
-)
+SUITES["dev"] += ["test_z_local_config"]
+
+SUITES["integ"] += SUITES["api"]
 
 
-def list_tests(suite: TestSuite):
-    for subsuite in suite:
-        for match in re.finditer(r"tests=\[([^][]+)\]>", str(subsuite)):
-            for test_case in match[1].split(", "):
-                yield test_case.replace("g2p.tests.", "")
+class PytestCollectorPlugin:
+    def __init__(self):
+        self.collected = []
 
+    def pytest_collection_modifyitems(self, session, config, items):
+        self.collected.extend([item.nodeid for item in items])
+
+
+def list_tests(suite: List[str]):
+    plugin = PytestCollectorPlugin()
+    pytest_args = ["--collect-only", *suite, "-q"]
+    if sys.version_info >= (3, 10):
+        with redirect_stdout(io.StringIO()):  # broken with py 3.8/3.9...
+            pytest.main(pytest_args, plugins=[plugin])
+    else:
+        pytest.main(pytest_args, plugins=[plugin])
+    # print("===========\n", o.getvalue(), "\n================")
+    return plugin.collected
 
-def describe_suite(suite: TestSuite):
-    full_suite = LOADER.discover(os.path.dirname(__file__))
-    full_list = list(list_tests(full_suite))
-    requested_list = list(list_tests(suite))
+
+def describe_suite(suite_name, suite_filenames: List[str]):
+    full_list = list_tests([])
+    requested_list = list_tests(suite_filenames)
     requested_set = set(requested_list)
-    print("Test suite includes:", *sorted(requested_list), sep="\n")
+    print(f"Test suite '{suite_name}' includes:", *sorted(requested_list), sep="\n")
     print(
-        "\nTest suite excludes:",
+        f"\nTest suite '{suite_name}' excludes:",
         *sorted(test for test in full_list if test not in requested_set),
         sep="\n",
     )
+    print(
+        "\nTotal test cases",
+        f"found: {len(full_list)};",
+        f"included: {len(requested_list)};",
+        f"excluded: {len(full_list)-len(requested_list)}.",
+    )
 
 
-SUITES = ("all", "dev", "integ", "langs", "mappings", "trans", "neural")
-
-
-def run_tests(suite: str, describe: bool = False, verbosity: int = 3) -> bool:
+def run_tests(suite: Optional[str], describe=False, verbose=False) -> bool:
     """Run the test suite specified in suite.
 
     Args:
@@ -139,53 +108,40 @@ def run_tests(suite: str, describe: bool = False, verbosity: int = 3) -> bool:
     Returns: Bool: True iff success
     """
     if not suite:
-        LOGGER.info("No test suite specified, defaulting to dev.")
+        LOGGER.info(
+            "No test suite specified, defaulting to 'dev', which skips the slowest tests."
+        )
         suite = "dev"
 
-    if suite == "all":
-        test_suite = LOADER.discover(os.path.dirname(__file__))
-    elif suite == "trans":
-        test_suite = TestSuite(TRANSDUCER_TESTS)
-    elif suite == "langs":
-        test_suite = TestSuite(LANGS_TESTS)
-    elif suite == "mappings":
-        test_suite = TestSuite(MAPPINGS_TESTS)
-    elif suite == "integ":
-        test_suite = TestSuite(INTEGRATION_TESTS)
-    elif suite == "neural":
-        test_suite = TestSuite(NEURAL_TESTS)
-    elif suite == "dev":
-        test_suite = TestSuite(DEV_TESTS)
-    else:
+    if suite not in SUITES:
         LOGGER.error("Please specify a test suite to run among: " + ", ".join(SUITES))
         return False
 
+    test_suite = SUITES[suite]
+    tests_dir = Path(__file__).parent
+    test_suite_filenames = [str(tests_dir / f"{file}.py") for file in test_suite]
     if describe:
-        describe_suite(test_suite)
+        describe_suite(suite, test_suite_filenames)
         return True
     else:
-        runner = TextTestRunner(verbosity=verbosity)
-        success = runner.run(test_suite).wasSuccessful()
-        if not success:
-            LOGGER.error("Some tests failed. Please see log above.")
-        return success
+        pytest_args = ["--verbose"] if verbose else []
+        return 0 == pytest.main([*test_suite_filenames, *pytest_args])
 
 
-def main():
+def main() -> None:
     parser = argparse.ArgumentParser(description="Run g2p test suites.")
-    parser.add_argument("--quiet", "-q", action="store_true", help="reduce output")
+    parser.add_argument("--verbose", "-v", action="store_true", help="verbose output")
     parser.add_argument(
         "--describe", action="store_true", help="describe the selected test suite"
     )
     parser.add_argument(
         "suite",
         nargs="?",
-        default="dev",
         help="the test suite to run [dev]",
-        choices=SUITES,
+        choices=SUITES.keys(),
     )
     args = parser.parse_args()
-    result = run_tests(args.suite, args.describe, 1 if args.quiet else 3)
+    result = run_tests(args.suite, args.describe, args.verbose)
     if not result:
         sys.exit(1)
 

g2p/tests/test_api_resources.py

@@ -6,10 +6,12 @@
 import json
 import os
 import re
+import sys
 from typing import Dict, Union
-from unittest import TestCase, main
+from unittest import TestCase
 
 from fastapi.testclient import TestClient
+from pytest import main
 
 from g2p.app import APP
 from g2p.log import LOGGER
@@ -179,4 +181,4 @@ def test_g2p_conversion_with_tok(self):
 
 
 if __name__ == "__main__":
-    main()
+    main([__file__, *sys.argv])

g2p/tests/test_api_v2.py

@@ -1,15 +1,18 @@
-import unittest
+#!/usr/bin/env python
+import sys
 from contextlib import redirect_stderr
 from io import StringIO
+from unittest import TestCase
 
 from fastapi.testclient import TestClient
+from pytest import main
 
 from g2p.api_v2 import api
 
 API_CLIENT = TestClient(api)
 
 
-class TestAPIV2(unittest.TestCase):
+class TestAPIV2(TestCase):
     def test_langs(self):
         with redirect_stderr(StringIO()):
             response = API_CLIENT.get("/langs")
@@ -422,4 +425,4 @@ def test_no_path(self):
 
 
 if __name__ == "__main__":
-    unittest.main()
+    main([__file__, *sys.argv])

g2p/tests/test_check_ipa_arpabet.py

@@ -2,7 +2,10 @@
 
 """ Test Mapping langs utility functions and their use in g2p convert --check """
 
-from unittest import TestCase, main
+import sys
+from unittest import TestCase
+
+from pytest import main
 
 from g2p import make_g2p
 from g2p.log import LOGGER
@@ -114,4 +117,4 @@ def test_check_with_equiv(self):
 
 
 if __name__ == "__main__":
-    main()
+    main([__file__, *sys.argv])