use pytest in CI, and exclude tests from coverage again

.github/workflows/matrix-tests.yml

@@ -25,7 +25,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v6
-      - uses: FedericoCarboni/setup-ffmpeg@v2
+      - uses: FedericoCarboni/setup-ffmpeg@37062fbf7149fc5578d6c57e08aed62458b375d6 # @v3.1, with tool cache
         if : runner.os != 'macOS'
       - run: brew install ffmpeg
         if : runner.os == 'macOS'
@@ -37,4 +37,4 @@ jobs:
         run: |
           pip install -e .[all]
       - name: Run tests
-        run: python run_tests.py prod
+        run: python run_tests.py all

.github/workflows/tests.yml

@@ -22,7 +22,7 @@ jobs:
     steps:
       - uses: actions/checkout@v6
 
-      - uses: FedericoCarboni/setup-ffmpeg@v2
+      - uses: FedericoCarboni/setup-ffmpeg@37062fbf7149fc5578d6c57e08aed62458b375d6 # @v3.1, with tool cache
 
       - name: Set up Python
         uses: actions/setup-python@v6
@@ -39,8 +39,8 @@ jobs:
 
       - name: Run tests
         run: |
-          coverage run --parallel-mode run_tests.py prod
-          DEVELOPMENT=1 coverage run --parallel-mode -m tests.test_web_api
+          coverage run -m pytest
+          DEVELOPMENT=1 coverage run tests/test_web_api.py
           coverage combine
           coverage xml
 
@@ -100,7 +100,7 @@ jobs:
     steps:
       - uses: actions/checkout@v6
 
-      - uses: FedericoCarboni/setup-ffmpeg@v2
+      - uses: FedericoCarboni/setup-ffmpeg@37062fbf7149fc5578d6c57e08aed62458b375d6 # @v3.1, with tool cache
 
       - name: Set up Python
         uses: actions/setup-python@v6
@@ -114,7 +114,7 @@ jobs:
           pip install -e .[all]
 
       - name: Run tests on Windows
-        run: python run_tests.py prod
+        run: python run_tests.py all
 
       - name: Make sure the CLI outputs utf8 on Windows
         # Note: we're checking something CLI specific, from a prompt, so we don't want to run
@@ -157,8 +157,8 @@ jobs:
           curl http://127.0.0.1:8000/api/v1/langs | grep Cree
           kill %1
 
-      - name: Install test dependency
-        run: pip install httpx
+      - name: Install minimal test dependencies
+        run: pip install httpx pytest
 
       - name: unit test the web API
         run: python -m tests.test_web_api

README.md

@@ -160,7 +160,7 @@ or test the installed version (requires installing dev dependencies):
 
     pip install 'readalongs[dev]'  # if you installed from PyPI, or
     pip install -e '.[dev]'        # if you installed from a local clone
-    python run_tests.py dev
+    python run_tests.py
 
 recently, we also made the project compatible with Pytest, so this will run the tests too:
 

pyproject.toml

@@ -129,6 +129,16 @@ Repository = "https://github.com/ReadAlongs/Studio"
 Issues = "https://github.com/ReadAlongs/Studio/issues"
 Changelog = "https://github.com/ReadAlongs/Studio/releases"
 
+[tool.coverage.run]
+source_pkgs = ["readalongs"]
+branch = true
+parallel = true
+omit = ["readalongs/waveform2svg/*", "readalongs/epub/*"]
+exclude_also = ["if 0:", "if __name__ == .__main__.:"]
+
+[tool.coverage.report]
+precision = 2
+
 [tool.mypy]
 plugins = ["pydantic.mypy"]
 ignore_missing_imports = true
@@ -140,4 +150,5 @@ profile = "black"
 filterwarnings = [
     "ignore:'audioop' is deprecated and slated for removal in Python 3.13:DeprecationWarning",
     "ignore:.*codecs.open.. is deprecated. Use open.. instead.*:DeprecationWarning",
+    "ignore:.*Module already imported so cannot be rewritten; anyio.*",
 ]

run_tests.py

@@ -4,154 +4,121 @@
 Top-level runner for out test suites
 
 Invoke as
-   ./run.py [suite]
+   ./run_tests.py [suite]
 where [suite] can be one of:
    all: run everything, by searching the directory for all test suite files
-   prod: synonym for all
-   dev: run the standard development test suite - this is what we do in CI
+   dev: now a synonym for all (used to exclude some expensive tests)
+   api: run only the API-related tests
+   cli: run only the CLI-related tests
    e2e: run the end-to-end tests
-   other: run the other tests
 """
 
 import argparse
-import os
-import re
+import io
 import sys
-from unittest import TestLoader, TestSuite, TextTestRunner
+from contextlib import redirect_stdout
+from pathlib import Path
+from typing import List, Optional
+
+import pytest
 
 from readalongs.log import LOGGER
-from tests.test_align_cli import TestAlignCli
-from tests.test_anchors import TestAnchors
-from tests.test_api import TestAlignApi
-from tests.test_audio import TestAudio
-from tests.test_config import TestConfig
-from tests.test_dna_text import TestDNAText
-from tests.test_dna_utils import TestDNAUtils
-from tests.test_dtd import TestDTD
-from tests.test_force_align import TestForceAlignment, TestXHTML
-from tests.test_g2p_cli import TestG2pCli
-from tests.test_make_xml_cli import TestMakeXMLCli
-from tests.test_misc import TestMisc
-from tests.test_package_urls import TestPackageURLs
-from tests.test_silence import TestSilence
-from tests.test_smil import TestSmilUtilities
-from tests.test_temp_file import TestTempFile
-from tests.test_tokenize_cli import TestTokenizeCli
-from tests.test_tokenize_xml import TestTokenizer
-from tests.test_web_api import TestWebApi
-
-LOADER = TestLoader()
-
-e2e_tests = [
-    LOADER.loadTestsFromTestCase(test) for test in (TestForceAlignment, TestXHTML)
-]
-
-api_tests = [
-    LOADER.loadTestsFromTestCase(test) for test in [TestWebApi]
-]  # TODO: add some load testing with https://locust.io/
-
-other_tests = [
-    LOADER.loadTestsFromTestCase(test)
-    for test in [
-        TestAnchors,
-        TestConfig,
-        TestDNAText,
-        TestDNAUtils,
-        TestTokenizer,
-        TestTokenizeCli,
-        TestTempFile,
-        TestMakeXMLCli,
-        TestAudio,
-        TestAlignCli,
-        TestAlignApi,
-        TestG2pCli,
-        TestMisc,
-        TestSilence,
-        TestSmilUtilities,
-        TestPackageURLs,
-        TestWebApi,
-        TestDTD,
-    ]
-]
-
-
-def list_tests(suite: TestSuite):
-    for subsuite in suite:
-        for match in re.finditer(r"tests=\[([^][]+)\]>", str(subsuite)):
-            yield from match[1].split(", ")
-
-
-def describe_suite(suite: TestSuite):
-    full_suite = LOADER.discover(os.path.dirname(__file__))
-    full_list = list(list_tests(full_suite))
-    requested_list = list(list_tests(suite))
+
+SUITES = {
+    "all": [],  # relies on discovery to collect all tests
+    "dev": [],  # synonym for all
+    "api": ["test_web_api", "test_api"],
+    "cli": ["test_align_cli", "test_g2p_cli", "test_make_xml_cli", "test_tokenize_cli"],
+    "e2e": ["test_force_align", "test_align_cli"],
+}
+
+# TODO: add some load testing with https://locust.io/
+
+
+class PytestCollectorPlugin:
+    def __init__(self):
+        self.collected = []
+
+    def pytest_collection_modifyitems(self, session, config, items):
+        self.collected.extend([item.nodeid for item in items])
+
+
+def list_tests(suite: List[str]):
+    plugin = PytestCollectorPlugin()
+    pytest_args = ["--collect-only", *suite, "-q"]
+    if sys.version_info >= (3, 10):
+        with redirect_stdout(io.StringIO()):  # broken with py 3.8/3.9...
+            pytest.main(pytest_args, plugins=[plugin])
+    else:
+        pytest.main(pytest_args, plugins=[plugin])
+    # print("===========\n", o.getvalue(), "\n================")
+    return plugin.collected
+
+
+def describe_suite(suite_name, suite_filenames: List[str]):
+    full_list = list_tests([])
+    requested_list = list_tests(suite_filenames)
     requested_set = set(requested_list)
-    print("Test suite includes:", *sorted(requested_list), sep="\n")
+    print(f"Test suite '{suite_name}' includes:", *sorted(requested_list), sep="\n")
     print(
-        "\nTest suite excludes:",
+        f"\nTest suite '{suite_name}' excludes:",
         *sorted(test for test in full_list if test not in requested_set),
         sep="\n",
     )
+    print(
+        "\nTotal test cases",
+        f"found: {len(full_list)};",
+        f"included: {len(requested_list)};",
+        f"excluded: {len(full_list) - len(requested_list)}.",
+    )
 
 
-SUITES = ["all", "dev", "e2e", "prod", "api", "other"]
-
-
-def run_tests(suite: str, describe: bool = False, verbosity=3) -> bool:
+def run_tests(suite: Optional[str], describe=False, verbose=False) -> bool:
     """Run the specified test suite.
 
     Args:
         suite: one of SUITES, "dev" if the empty string
         describe: if True, list all the test cases instead of running them.
 
-    Returns: True iff success
+    Returns: Bool: True iff success
     """
 
     if not suite:
-        LOGGER.info("No test suite specified, defaulting to dev.")
+        LOGGER.info("No test suite specified, defaulting to dev, which runs all tests.")
         suite = "dev"
 
-    if suite == "e2e":
-        test_suite = TestSuite(e2e_tests)
-    elif suite == "api":
-        test_suite = TestSuite(api_tests)
-    elif suite == "dev":
-        test_suite = TestSuite(other_tests + e2e_tests)
-    elif suite in ("prod", "all"):
-        test_suite = LOADER.discover(os.path.dirname(__file__))
-    elif suite == "other":
-        test_suite = TestSuite(other_tests)
-    else:
-        LOGGER.error(
-            "Sorry, you need to select a Test Suite to run, one of: " + " ".join(SUITES)
-        )
+    if suite not in SUITES:
+        LOGGER.error("Please specify a test suite to run among: " + ", ".join(SUITES))
         return False
 
+    test_suite = SUITES[suite]
+    tests_dir = Path(__file__).parent / "tests"
+    test_suite_filenames = [str(tests_dir / f"{file}.py") for file in test_suite]
     if describe:
-        describe_suite(test_suite)
+        describe_suite(suite, test_suite_filenames)
         return True
     else:
-        runner = TextTestRunner(verbosity=verbosity)
-        success = runner.run(test_suite).wasSuccessful()
-        if not success:
-            LOGGER.error("Some tests failed. Please see log above.")
-        return success
+        pytest_args = ["--verbose"] if verbose else []
+        return 0 == pytest.main([*test_suite_filenames, *pytest_args])
 
 
-if __name__ == "__main__":
+def main() -> None:
     parser = argparse.ArgumentParser(description="Run ReadAlongs/Studio test suites.")
-    parser.add_argument("--quiet", "-q", action="store_true", help="reduce output")
+    parser.add_argument("--verbose", "-v", action="store_true", help="verbose output")
     parser.add_argument(
         "--describe", action="store_true", help="describe the selected test suite"
     )
     parser.add_argument(
         "suite",
         nargs="?",
-        default="dev",
         help="the test suite to run [dev]",
         choices=SUITES,
     )
     args = parser.parse_args()
-    result = run_tests(args.suite, args.describe, 1 if args.quiet else 3)
+    result = run_tests(args.suite, args.describe, args.verbose)
     if not result:
         sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

tests/test_align_cli.py

@@ -6,13 +6,14 @@
 
 import os
 import subprocess
+import sys
 import tempfile
 from os.path import exists, join
 from pathlib import Path
 from typing import Union
-from unittest import main
 
 from lxml.html import fromstring
+from pytest import main
 
 from readalongs._version import READALONG_FILE_FORMAT_VERSION, VERSION
 from readalongs.cli import align, langs
@@ -153,7 +154,6 @@ def test_invoke_align(self) -> None:
                 str(output),
             ],
         )
-        print("dir(result)", dir(results_output_exists))
         self.assertNotEqual(results_output_exists.exit_code, 0)
         self.assertIn(
             "already exists, use -f to overwrite", results_output_exists.output
@@ -652,4 +652,4 @@ def test_ffmpeg_is_present(self):
 
 
 if __name__ == "__main__":
-    main()
+    main([__file__, *sys.argv])

tests/test_anchors.py

@@ -3,9 +3,11 @@
 """Unit testing for the anchors functionality in readalongs align"""
 
 import os
+import sys
 from contextlib import redirect_stderr
 from io import StringIO
-from unittest import main
+
+from pytest import main
 
 from readalongs.align import align_audio
 from readalongs.log import LOGGER
@@ -101,4 +103,4 @@ def test_anchors_align_modes(self):
 
 
 if __name__ == "__main__":
-    main()
+    main([__file__, *sys.argv])