tests: Run gunittest-based tests with pytest that don’t require gunittest (#5584)

* pytest: Require pytest 8.2

* pytest: allow collecting more test files

* pytest: Ignore collecting files in `testsuite/data` folders

* pytest: Ignore running testsuite tests in normal runs

* pytest: Run gunittest-based tests in a separate step in CI

* CI(pytest): refactor pytest arguments into a file

* pytest: remove common arg from gunittest args

* pytest: Deselect crashing tests that use ctypes functions directly and note which tests run successfully

* pytest: Extract gunittest deselect list to a separate arg file

* pytest: ignore deprecated python/grass/temporal/unit_tests.py

* CI(pytest): Run gunittest-based tests without default report args

CI(pytest): Remove commented out ci args

* CI(pytest): Show short traceback for gunittest-based tests

* CI(pytest): Reset report list at end of run

* CI(pytest): Only run passing gunittest-based tests

* CI(OSGeo4w): Add folder of C:\OSGeo4W\opt\grass\grass85.bat to path for running gunittest-based tests with pytest

* tests: Deselect non-fixed Windows test

* CI(OSGeo4W): Use standard pytest args for running gunittest-based tests with pytest

Author: echoix

.github/workflows/macos.yml

@@ -97,15 +97,24 @@ jobs:
           export LD_LIBRARY_PATH=$(grass --config path)/lib:$LD_LIBRARY_PATH
           pytest \
             @.github/workflows/pytest_args_ci.txt \
-            @.github/workflows/pytest_args_parallel.txt
+            @.github/workflows/pytest_args_parallel.txt \
+            -k 'not testsuite'
       - name: Run pytest with a single worker (for tests marked with needs_solo_run)
         shell: micromamba-shell {0}
         run: |
           export PYTHONPATH=$(grass --config python_path):$PYTHONPATH
           export LD_LIBRARY_PATH=$(grass --config path)/lib:$LD_LIBRARY_PATH
           pytest \
             @.github/workflows/pytest_args_ci.txt \
-            @.github/workflows/pytest_args_not_parallel.txt
+            @.github/workflows/pytest_args_not_parallel.txt \
+            -k 'not testsuite'
+      - name: Run pytest with a single worker (for gunittest-based tests)
+        shell: micromamba-shell {0}
+        run: |
+          export PYTHONPATH=$(grass --config python_path):$PYTHONPATH
+          export LD_LIBRARY_PATH=$(grass --config path)/lib:$LD_LIBRARY_PATH
+          pytest \
+            @.github/workflows/pytest_args_gunittest.txt
 
       - name: Cache GRASS Sample Dataset
         id: cached-data

.github/workflows/osgeo4w.yml

@@ -131,7 +131,20 @@ jobs:
           call %OSGEO4W_ROOT%\opt\grass\etc\env.bat
           set PYTHONPATH=%GISBASE%\etc\python;%PYTHONPATH%
           path %GISBASE%\lib;%GISBASE%\bin;%PATH%
-          pytest @.github/workflows/pytest_args_ci.txt
+          pytest ^
+            @.github/workflows/pytest_args_ci.txt ^
+            -k "not testsuite"
+        shell: cmd /D /E:ON /V:OFF /S /C "CALL C:/OSGeo4W/OSGeo4W.bat "{0}""
+      - name: Run pytest with a single worker (for gunittest-based tests)
+        run: |
+          call %OSGEO4W_ROOT%\opt\grass\etc\env.bat
+          set PYTHONPATH=%GISBASE%\etc\python;%PYTHONPATH%
+          path %GISBASE%\lib;%GISBASE%\bin;%PATH%;%OSGEO4W_ROOT%\opt\grass
+          pytest ^
+            @.github/workflows/pytest_args_ci.txt ^
+            @.github/workflows/pytest_args_deselect.txt ^
+            @.github/workflows/pytest_args_gunittest.txt ^
+            --deselect=lib/init/testsuite/test_grass_tmp_mapset.py
         shell: cmd /D /E:ON /V:OFF /S /C "CALL C:/OSGeo4W/OSGeo4W.bat "{0}""
 
       - name: Run tests

.github/workflows/pytest.yml

@@ -91,7 +91,8 @@ jobs:
           pytest \
             @.github/workflows/pytest_args_ci.txt \
             @.github/workflows/pytest_args_cov.txt \
-            @.github/workflows/pytest_args_parallel.txt
+            @.github/workflows/pytest_args_parallel.txt \
+            -k 'not testsuite'
 
       - name: Run pytest with a single worker (for tests marked with needs_solo_run)
         run: |
@@ -102,7 +103,17 @@ jobs:
           pytest \
             @.github/workflows/pytest_args_ci.txt \
             @.github/workflows/pytest_args_cov.txt \
-            @.github/workflows/pytest_args_not_parallel.txt
+            @.github/workflows/pytest_args_not_parallel.txt \
+            -k 'not testsuite'
+      - name: Run pytest with a single worker (for gunittest-based tests)
+        run: |
+          export PYTHONPATH=`grass --config python_path`:$PYTHONPATH
+          export LD_LIBRARY_PATH=$(grass --config path)/lib:$LD_LIBRARY_PATH
+          export INITIAL_GISBASE="$(grass --config path)"
+          export INITIAL_PWD="${PWD}"
+          pytest \
+            @.github/workflows/pytest_args_cov.txt \
+            @.github/workflows/pytest_args_gunittest.txt
       - name: Fix non-standard installed script paths in coverage data
         run: |
           export PYTHONPATH=`grass --config python_path`:$PYTHONPATH

.github/workflows/pytest_args_deselect.txt

@@ -0,0 +1,16 @@
+--deselect=imagery/i.signatures/testsuite/test_i_signatures.py
+--deselect=lib/gis/testsuite/gis_lib_env_test.py
+--deselect=lib/imagery/testsuite/test_imagery_find.py
+--deselect=lib/imagery/testsuite/test_imagery_sigfile.py
+--deselect=lib/imagery/testsuite/test_imagery_signature_management.py
+--deselect=lib/imagery/testsuite/test_imagery_sigsetfile.py
+--deselect=lib/raster/testsuite/test_raster_metadata.py
+--deselect=lib/vector/Vlib/testsuite/test_vlib_intersect.py
+--deselect=python/grass/pygrass/gis/testsuite/test_gis.py
+--deselect=python/grass/pygrass/modules/interface/testsuite/test_modules.py::TestModules
+--deselect=python/grass/pygrass/vector/testsuite/test_geometry.py
+--deselect=python/grass/pygrass/vector/testsuite/test_geometry_attrs.py
+--deselect=python/grass/pygrass/vector/testsuite/test_vector.py
+--deselect=python/grass/pygrass/vector/testsuite/test_vector3d.py
+--deselect=raster/r.support/testsuite/test_r_support.py
+--deselect=scripts/r.semantic.label/testsuite/test_r_semantic_label.py

.github/workflows/pytest_args_gunittest.txt

@@ -0,0 +1,13 @@
+--ignore-glob=**/tests/**
+--ignore=gui/
+--ignore=python/grass/temporal/unit_tests.py
+lib/gis/testsuite/test_gis_lib_getl.py
+lib/init/testsuite/test_grass_tmp_mapset.py
+lib/vector/Vlib/testsuite/test_vlib_box.py
+python/grass/exceptions/testsuite/test_ScriptError.py
+python/grass/gunittest/testsuite/test_checkers.py
+python/grass/pygrass/modules/interface/testsuite/test_flag.py
+python/grass/pygrass/modules/interface/testsuite/test_parameter.py
+python/grass/pygrass/vector/testsuite/test_filters.py
+python/grass/script/testsuite/test_core_make_val.py
+python/grass/script/testsuite/test_utils.py

pyproject.toml

@@ -353,6 +353,7 @@ addopts = """
     --ignore-glob='dist.*'
     --ignore-glob='bin.*'
     --ignore-glob='*/tests/data/*'
+    --ignore-glob='*/testsuite/data/**'
     --ignore-glob='*/grass/pygrass/tests/*'
     --doctest-glob='*doctest*.txt'
     --ignore='raster/r.category/test_rcategory_doctest.txt'
@@ -361,8 +362,12 @@ markers = [
     "slow: marks tests as slow (deselect with '-m \"not slow\"')",
     "needs_solo_run: marks tests that must be run without any other tests running in parallel",
 ]
-minversion = "6.0"
-python_files = "*/tests/*_test.py */tests/test_*.py"
+minversion = "8.2"
+python_files = """
+    */tests/*_test.py
+    */tests/test_*.py
+    **/testsuite/*test*.py
+"""
 timeout = 300