Refactor CI checks

.github/workflows/ci.yml

@@ -30,14 +30,16 @@ jobs:
       # Instead of running all checks in one task (i.e., `tox -m static`),
       # we'll instead run them one-by-one, so that it's easier to
       # identify and debug failing checks in the GitHub UI.
-      - name: Run static checks with pre-commit
-        shell: bash
-        env:
-          SKIP: "no-commit-to-branch"
-        run: tox -e pre-commit
+      # NOTE: We skip the pre-commit checks here since we
+      #       already use the pre-commit.ci service.
+      #- name: Run static checks with pre-commit
+      #  shell: bash
+      #  env:
+      #    SKIP: "no-commit-to-branch"
+      #  run: tox -e pre-commit-all
       - name: Run type checks with mypy
         shell: bash
-        run: tox -e mypy
+        run: tox -e mypy-safe
 
   # Run test suits for all supported platforms and Python versions
   software-tests:

docs/development/contributing.md

@@ -161,10 +161,10 @@ For more information on all the checks being run here, take a look inside the {r
 The only static check that is not run by pre-commit is [mypy](https://github.com/python/mypy), which is too expensive to run on every commit. To run mypy against all files, run:
 
 ```shell
-tox -e mypy
+tox -e mypy-incremental
 ```
 
-Just like with pytest, you can also pass extra positional arguments to mypy by running `tox -e mypy -- <MYPY_FLAGS>`.
+Just like with pytest, you can also pass extra positional arguments to mypy by running `tox -e mypy-incremental -- <MYPY_FLAGS>`.
 
 To trigger all static checks, run:
 

docs/reference/changelog.md

@@ -44,6 +44,7 @@ Thanks to {gh-user}`sstephanyy` for their contributions to this release! 🚀
 ### CI/CD
 
 - Stop sending coverage reports to Codacy ({gh-pr}`265`)
+- Improve local development experience and optimise the CI pipeline ({gh-pr}`273`)
 
 ### Miscellaneous
 

requirements/mypy.txt

@@ -8,3 +8,9 @@ types-PyYAML
 types-requests
 types-tqdm
 pandas-stubs
+
+# mypy also needs to inherit other environment dependencies in
+# order to correctly infer types for code in tests, docs, etc.
+-r cicd_utils.txt
+-r docs.txt
+-r tests.txt

requirements/tests.txt

@@ -6,3 +6,7 @@ pytest-socket
 # Coverage
 diff-cover
 pytest-cov
+
+# `cicd_utils` requirements also need to be
+# installed in order to test these utilities
+-r cicd_utils.txt

tox.ini

@@ -1,10 +1,13 @@
 [tox]
 labels =
-    static = pre-commit, mypy
+    static = pre-commit-all, mypy-safe
+    static-quick = pre-commit-quick, mypy-incremental
     tests = tests-unit, tests-e2e, tests-cicd_utils
+    upgrade-requirements = pre-commit-autoupgrade
 isolated_build = true
 requires =
     tox>=4
+    tox-uv
 
 [testenv]
 description = run the pytest tests under {basepython}
@@ -24,10 +27,7 @@ passenv =
     GITHUB_*
     PYTEST_*
 parallel_show_output = true
-deps =
-    -r requirements/tests.txt
-    # `cicd_utils` requirements also need to be installed
-    -r requirements/cicd_utils.txt
+deps = -r requirements/tests.txt
 allowlist_externals = rm, mv
 commands =
     rm -rf build/
@@ -41,7 +41,6 @@ commands =
     tests-e2e: pytest tests/e2e --cov=cicd_utils/ridgeplot_examples {env:_COV_REPORT_XML} --cov-fail-under=75 {posargs:}
     tests-e2e: diff-cover {env:_DIFFCOVER_DFLT_ARGS}
 
-    tests-cicd_utils: rm -f {env:_COV_FILES_RM}
     tests-cicd_utils: pytest tests/cicd_utils --cov=cicd_utils/cicd {env:_COV_REPORT_XML} --cov-fail-under=60 {posargs:}
     tests-cicd_utils: diff-cover {env:_DIFFCOVER_DFLT_ARGS}
 
@@ -52,7 +51,8 @@ commands =
     coverage-combine: coverage html --data-file={env:COVERAGE_FILE} --omit="cicd_utils/*,tests/*"
     coverage-combine: coverage report --data-file={env:COVERAGE_FILE} --omit="cicd_utils/*,tests/*" --fail-under=98
     coverage-combine: diff-cover {env:_DIFFCOVER_DFLT_ARGS} --fail-under=100
-[testenv:pre-commit]
+
+[testenv:pre-commit-{all,quick,autoupgrade}]
 description = run code style and linting checks with pre-commit
 passenv =
     {[testenv]passenv}
@@ -62,21 +62,21 @@ passenv =
     SSH_AUTH_SOCK
 skip_install = true
 deps = pre-commit
-# To update the pre-commit hooks, run: `pre-commit autoupdate`
 commands =
-    pre-commit run --all-files --show-diff-on-failure {posargs:}
+    all:   pre-commit run --show-diff-on-failure {posargs:}
+    quick: pre-commit run black-jupyter --all-files
+           pre-commit run ruff --all-files
+    autoupgrade: pre-commit autoupdate {posargs:}
 
-[testenv:mypy]
+[testenv:mypy-{safe,incremental}]
 description = run type checks with mypy
-deps =
-    -r requirements/mypy.txt
-    # mypy needs to inherit other environment dependencies in
-    # order to infer types for code in tests, docs, etc...
-    -r requirements/cicd_utils.txt
-    -r requirements/docs.txt
-    -r requirements/tests.txt
+deps = -r requirements/mypy.txt
+setenv =
+    {[testenv]setenv}
+    _MYPY_DFLT_ARGS=--config-file=mypy.ini --strict --enable-incomplete-feature=NewGenericSyntax
 commands =
-    mypy --config-file=mypy.ini --cache-dir=/dev/null --no-incremental --strict {posargs:}
+    safe:        mypy {env:_MYPY_DFLT_ARGS} --no-incremental --cache-dir=/dev/null {posargs:}
+    incremental: mypy {env:_MYPY_DFLT_ARGS} --incremental {posargs:}
 
 [testenv:docs-{live,static}]
 description = generate Sphinx (live/static) HTML documentation