ci: Run non-benchmark tests in benchmark workflow

.github/workflows/benchmark.yml

@@ -36,6 +36,9 @@ jobs:
         with:
           python-version: '3.12'
 
+      - name: Set up Graphviz
+        uses: ts-graphviz/setup-graphviz@v2
+
       - name: Create virtual environment
         run: |
           python -m venv .venv
@@ -100,4 +103,8 @@ jobs:
 
       - name: Run various benchmark tests
         working-directory: py-polars
-        run: pytest -m benchmark --durations 0 -v
+        run: pytest -m release --durations 0 -v
+
+      - name: Run non-benchmark tests
+        working-directory: py-polars
+        run: pytest -m 'not release and not debug' -n auto --dist loadgroup

.github/workflows/codecov.yml

@@ -56,7 +56,8 @@ jobs:
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@cargo-llvm-cov
 
-      - uses: Swatinem/rust-cache@v2
+      - name: Cache Rust
+        uses: Swatinem/rust-cache@v2
         with:
           save-if: ${{ github.ref_name == 'main' }}
 
@@ -85,13 +86,13 @@ jobs:
         run: maturin develop
 
       - name: Run Python tests
-        run: pytest --cov -n auto --dist loadgroup -m "not benchmark and not docs" --cov-report xml:main.xml
+        run: pytest --cov -n auto --dist loadgroup -m "not release and not docs" --cov-report xml:main.xml
         continue-on-error: true
 
       - name: Run Python tests - async reader
         env:
           POLARS_FORCE_ASYNC: 1
-        run: pytest --cov -m "not benchmark and not docs" tests/unit/io/ --cov-report xml:async.xml
+        run: pytest --cov -m "not release and not docs" tests/unit/io/ --cov-report xml:async.xml
         continue-on-error: true
 
       - name: Report coverage

.github/workflows/test-python.yml

@@ -89,13 +89,13 @@ jobs:
           # Currently skipped due to performance issues in coverage:
           # https://github.com/nedbat/coveragepy/issues/1665
           COV: ${{ !(matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12') && '--cov' || '--no-cov' }}
-        run: pytest $COV -n auto --dist loadgroup -m "not benchmark and not docs"
+        run: pytest $COV -n auto --dist loadgroup -m "not release and not docs"
 
       - name: Run tests async reader tests
         if: github.ref_name != 'main' && matrix.os != 'windows-latest'
         env:
           POLARS_FORCE_ASYNC: 1
-        run: pytest -m "not benchmark and not docs" tests/unit/io/
+        run: pytest -m "not release and not docs" tests/unit/io/
 
       - name: Check import without optional dependencies
         if: github.ref_name != 'main' && matrix.python-version == '3.12' && matrix.os == 'ubuntu-latest'

docs/development/contributing/test.md

@@ -111,10 +111,10 @@ Refer to the [benchmark workflow](https://github.com/pola-rs/polars/blob/main/.g
 ### Running other benchmark tests
 
 The other benchmark tests are run using pytest.
-Run `pytest -m benchmark --durations 0 -v` to run these tests and report run duration.
+Run `pytest -m release --durations 0 -v` to run these tests and report run duration.
 
 Note that benchmark tests are excluded by default when running `pytest`.
-You must explicitly specify `-m benchmark` to run them.
+You must explicitly specify `-m release` to run them.
 They will also be excluded when calculating test coverage.
 
 These tests _will_ be run as part of the `make test-all` make command.

py-polars/Makefile

@@ -91,7 +91,7 @@ test-all: .venv build  ## Run all tests
 
 .PHONY: coverage
 coverage: .venv build  ## Run tests and report coverage
-	$(VENV_BIN)/pytest --cov -n auto --dist loadgroup -m "not benchmark"
+	$(VENV_BIN)/pytest --cov -n auto --dist loadgroup -m "not release"
 
 .PHONY: clean
 clean:  ## Clean up caches and build artifacts

py-polars/pyproject.toml

@@ -205,12 +205,13 @@ addopts = [
   "--strict-markers",
   "--import-mode=importlib",
   # Default to running fast tests only. To run ALL tests, run: pytest -m ""
-  "-m not slow and not hypothesis and not benchmark and not write_disk and not docs",
+  "-m not slow and not hypothesis and not release and not write_disk and not docs",
 ]
 markers = [
   "write_disk: Tests that write to disk",
   "slow: Tests with a longer than average runtime.",
-  "benchmark: Tests that should be run on a Polars release build.",
+  "release: Tests that should be run on a Polars release build.",
+  "debug: Tests that should be run on a Polars debug build.",
   "docs: Documentation code snippets",
 ]
 filterwarnings = [

py-polars/tests/benchmark/test_release.py

@@ -3,7 +3,7 @@
 
 Tests in this module will be run in the CI using a release build of Polars.
 
-To run these tests: pytest -m benchmark
+To run these tests: pytest -m release
 """
 
 import time
@@ -17,7 +17,7 @@
 from polars.testing import assert_frame_equal
 
 # Mark all tests in this module as benchmark tests
-pytestmark = pytest.mark.benchmark()
+pytestmark = pytest.mark.release()
 
 
 @pytest.mark.skipif(

py-polars/tests/unit/streaming/test_streaming_sort.py

@@ -92,6 +92,7 @@ def test_ooc_sort(tmp_path: Path, monkeypatch: Any) -> None:
         assert_series_equal(out, s.sort(descending=descending))
 
 
+@pytest.mark.debug()
 @pytest.mark.write_disk()
 @pytest.mark.parametrize("spill_source", [True, False])
 def test_streaming_sort(

py-polars/tests/unit/test_cse.py

@@ -172,6 +172,7 @@ def test_schema_row_index_cse() -> None:
     assert_frame_equal(result, expected)
 
 
+@pytest.mark.debug()
 def test_cse_expr_selection_context(monkeypatch: Any, capfd: Any) -> None:
     monkeypatch.setenv("POLARS_VERBOSE", "1")
     q = pl.LazyFrame(