Fix pytest to run in parallel and cleanup files after (#209)

* Fix pytest to run in parallel and cleanup files after

* Fix typo in comment

* Bump webfactory ssh agent to latest version

---------

Co-authored-by: James Bensley <jwbensley@gmail.com>

.github/workflows/testing.yml

@@ -17,9 +17,9 @@ jobs:
       - name: Clone repo
         uses: actions/checkout@v3
       - name: Setup SSH Keys and known_hosts
-        uses: webfactory/ssh-agent@v0.8.0
+        uses: webfactory/ssh-agent@v0.9.0
         with:
-          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+          ssh-private-key: ${{ secrets.CI_ACTION_SSH_PRIV_KEY }}
       - name: Setup git
         run: |
           git config --global user.email "github_actions@example.com"

dnas/tests/cleanup.py

@@ -0,0 +1,17 @@
+import os
+
+import pytest
+
+
+def test_cleanup() -> None:
+    for file in [
+        "/opt/dnas_data/downloads/RRC1/rrc01.updates.20100827.0840.gz",
+        "/opt/dnas_data/downloads/RRC1/rrc01.updates.20241001.0055.gz",
+        "/opt/dnas_data/downloads/RRC23/rrc23.updates.20220421.0200.gz",
+        "/opt/dnas_data/downloads/RRC23/rrc23.updates.20220501.2305.gz",
+        "/opt/dnas_data/downloads/SYDNEY/sydney.updates.20220601.0230.bz2",
+        "/opt/dnas_data/downloads/SYDNEY/sydney.updates.20220601.0415.bz2",
+    ]:
+        if os.path.exists(file):
+            print(f"Doing to delete: {file}")
+            os.unlink(file)

dnas/tests/test_git.py

@@ -6,16 +6,13 @@
 import sys
 import unittest
 
-import pytest
-
 sys.path.append(
     os.path.join(os.path.dirname(os.path.realpath(__file__)), "../")
 )
 from dnas.config import config
 from dnas.git import git
 
 
-@pytest.mark.sequential_tests
 class test_git(unittest.TestCase):
     """
     The git tests must be run sequential, using a pytest marker to exclude
@@ -163,7 +160,7 @@ def test_clear(self: "test_git") -> None:
     def test_clone(self: "test_git") -> None:
         """
         Git clone should fail if there is an existing directory, that contains a
-        different repo. Create the base direcory with an empty .git sub-dir
+        different repo. Create the base directory with an empty .git sub-dir
         (which is invalid for git)
         """
         try:

dnas/tests/test_mrt_entry.py

@@ -14,6 +14,7 @@
 
 
 class test_mrt_entry(unittest.TestCase):
+
     def setUp(self: "test_mrt_entry") -> None:
         self.mrt_e = mrt_entry()
 
@@ -46,6 +47,8 @@ def setUp(self: "test_mrt_entry") -> None:
             if arch.NAME == "UNIT_TEST_RRC_23":
                 os.makedirs(arch.MRT_DIR, exist_ok=True)
                 self.upd_1_mrt = os.path.join(arch.MRT_DIR, self.upd_1_fn)
+                break
+        assert self.upd_1_mrt
 
         shutil.copy2(self.upd_1_path, self.upd_1_mrt)
         self.mrt_s = mrt_parser.parse_upd_dump(self.upd_1_mrt)
@@ -114,9 +117,6 @@ def test_to_json(self: "test_mrt_entry") -> None:
         self.assertIsInstance(j2, str)
         self.assertEqual(j1, j2)
 
-    def tearDown(self: "test_mrt_entry") -> None:
-        os.remove(self.upd_1_mrt)
-
 
 if __name__ == "__main__":
     unittest.main()

dnas/tests/test_mrt_parser.py

@@ -14,6 +14,7 @@
 
 
 class test_mrt_parser(unittest.TestCase):
+
     cfg = config()
 
     def setUp(self: "test_mrt_parser") -> None:

dnas/tests/test_mrt_stats.py

@@ -16,13 +16,8 @@
 from dnas.mrt_stats import mrt_stats
 
 
-@pytest.mark.sequential_tests
+@pytest.mark.mrt_stats
 class test_mrt_stats(unittest.TestCase):
-    """
-    Run these tests sequentially. When run in parallel,
-    setUp() doesn't finish before the unit tests run so they fail.
-    Probably something that can be fixed in the future.
-    """
 
     def setUp(self: "test_mrt_stats") -> None:
         """
@@ -4478,13 +4473,6 @@ def test_ts_ymd(self: "test_mrt_stats") -> None:
     def test_ts_ymd_format(self: "test_mrt_stats") -> None:
         self.assertEqual(self.upd_1_stats.ts_ymd_format(), "2022/04/21")
 
-    def tearDown(self: "test_mrt_stats") -> None:
-        os.remove(self.upd_1_mrt)
-        os.remove(self.upd_2_mrt)
-        os.remove(self.upd_3_mrt)
-        os.remove(self.upd_4_mrt)
-        os.remove(self.upd_5_mrt)
-
 
 if __name__ == "__main__":
     unittest.main()

tox.ini

@@ -43,10 +43,14 @@ markers =
 commands =
     # Ensure we have the unallocated ASN data
     python3 ./dnas/scripts/update_asn_allocations.py --debug
-    # Run up to 4 tests in parallel (those which are no marked as being sequential tests)
-    pytest -vvvv -n 4 -m "not sequential_tests" dnas/tests/ {posargs}
-    # Run the remaining tests which must be run sequentially
-    pytest -vvvv -m "sequential_tests" dnas/tests/ {posargs}
+    # Run as many test classes in parallel as we have cores.
+    # Group all tests from the same class to the same core.
+    pytest -vvvv -m "not mrt_stats" -x --numprocesses=logical --dist=loadfile dnas/tests/ {posargs}
+    # Run a single test class, this timespliting the tests
+    # across multiple cores.
+    pytest -vvvv -m "mrt_stats" -x --numprocesses=logical --dist=load dnas/tests/ {posargs}
+    # Cleanup test files
+    pytest -vvvv dnas/tests/cleanup.py {posargs}
 
 [testenv:shellcheck]
 skip_install=true