From 699ca8a30f194759e848c2e8391f3e44ae9f464c Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Fri, 15 Nov 2024 12:00:00 +0100 Subject: [PATCH 1/4] GitHub CI/`tox`: Update CI to use Python `3.11`, `3.12` and `3.13` Signed-off-by: Bernhard Kaindl --- .github/workflows/main.yml | 6 ++++-- pyproject.toml | 5 +++-- tox.ini | 24 ++++++++++++++++-------- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 07853454..6d54fd87 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,10 +28,12 @@ jobs: fail-fast: false matrix: include: - - python-version: '3.10' - os: ubuntu-22.04 - python-version: '3.11' os: ubuntu-22.04 + - python-version: '3.12' + os: ubuntu-22.04 + - python-version: '3.13' + os: ubuntu-22.04 runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 diff --git a/pyproject.toml b/pyproject.toml index ae27d940..df7dc9fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,11 +111,12 @@ show_error_context = true error_summary = true files = ["xcp", "tests/test_*.py"] mypy_path = "stubs" -python_version = "3.10" +python_version = "3.11" warn_redundant_casts = true warn_return_any = true warn_unreachable = true warn_unused_configs = true +explicit_package_bases = true disallow_any_unimported = true disallow_any_explicit = false disallow_any_generics = true @@ -206,7 +207,7 @@ stubPath = "stubs" inputs = ["xcp", "tests", "./*.py"] keepgoing = true platform = "linux" -python_version = "3.10" +python_version = "3.11" pythonpath = ".:stubs" disable = ["ignored-type-comment"] overriding_parameter_count_checks = true diff --git a/tox.ini b/tox.ini index 0de3fed6..0491f56e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,17 @@ [tox] -# This is the order how tox runs the tests when used interactively during development. -# Run the tests which uncover issues most often first! For example: -# 1. python 2.7 and 3.10 coverage test for changed, but not covered lines and mypy check -# 2. python 3.6 test and pylint warnings from changed lines -# 3. pytype (needs Python 3.8 for best results) -# 4. pyright checks, pytest test report as markdown for GitHub Actions summary -envlist = py38-covcombine-check, py311-lint-test, py310-pytype, py311-pyright-mdreport +# Set the envlist: +# Defines the environments that tox runs by default (when -e is not used) +# +# Default order of execution when using `tox` without `-e `: +# +# 1. pytest using python 3.11 for code coverage with diff-cover and mypy check +# 2. pytype (needs Python 3.10 or newer) +# 3. pylint and pyright checks (any modern Python version is fine) +# +# .github/workflows/main.yml is set up to test with 3.11, 3.12 and 3.13 in parallel. +# Therefore, use three environments: One with 3.11, one with 3.12 and one with 3.13: +# +envlist = py311-covcp-check-mdreport, py312-cov-pytype, py313-cov-lint-pyright isolated_build = true skip_missing_interpreters = true requires = @@ -186,6 +192,8 @@ python = 3.9: py39 3.10: py310 3.11: py311 + 3.12: py312 + 3.13: py313 [check] extras = mypy @@ -207,7 +215,7 @@ commands = deps = pytype pandas commands = - python3.10 -V # Needs python <= 3.10, and 3.10 is needed to parse new "|" syntax + # Needs python >= 3.10: Needed to parse the newer syntax for "Type2 | Type2" pytype --version # Runs pytype -j auto -k --config .github/workflows/pytype.cfg and parses the output: python3 pytype_runner.py # When switching versions, update .github/workflows/pytype.cfg too! From f531be64e575e903b76f6d181152bb39bb03a986 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Fri, 15 Nov 2024 12:00:00 +0100 Subject: [PATCH 2/4] `CONTRIBUTING.md`: Add `venv` setup and using `pipx` Signed-off-by: Bernhard Kaindl --- CONTRIBUTING.md | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8ba18bc9..5d53d115 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,14 @@ +# Development setup + +## Create a virtual environment with the test dependencies + +```bash +python -m venv .venv +. .venv/bin/activate +pip install pip-tools==7.3.0 +pip-compile --extra=test,mypy,pyright,pytype,tox -o - pyproject.toml | pip install -r /dev/stdin +``` + ## Development setup on Fedora 37 On Fedora 37, the `tox` rpm installs all Python versions. @@ -6,9 +17,26 @@ But this `tox` is older, so install `tox==4.5.1` using `pip` (see below) ```bash sudo dnf install tox;sudo rpm -e tox ``` +But preferably use `tox` from the virtual environment instead. + +## Development setup on Ubuntu 24.04 + +Prefer the virtual environment. Alternatively, an option is to use `pipx`: + +```bash +sudo apt install pipx +pipx install tox; pipx install 'pytest<7';pipx install pylint +pipx inject pytest pytest-{forked,localftpserver,pythonpath,subprocess,timeout} pyfakefs pytest_httpserver six mock +pipx inject pylint pyfakefs six mock pytest{,_forked,-localftpserver} +``` + +Use the `deadsnakes` ppa to install Python versions like 3.8 and 3.11 (see below) ## Development setup on Ubuntu 22.04 +Usage of to install +other python versions. + ```bash sudo apt update sudo apt install software-properties-common python{2,3}-dev @@ -77,7 +105,7 @@ Using pip-tools, you can extract the requirements and extras from `pyptoject.tom ```bash PYTHON=python3.10 -EXTRAS=.,test,mypy,pytype,tox +EXTRAS=.,test,mypy,pyright,pytype,tox PFLAGS="--no-warn-conflicts" $PYTHON -m pip install pip-tools==7.3.0 $PYTHON -m piptools compile --extra=$EXTRAS -o - pyproject.toml | From 1d68fbd03d2a92513f73bc1f06f46ad69ef25f6e Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Fri, 15 Nov 2024 12:00:00 +0100 Subject: [PATCH 3/4] `tests/test_logger.py`: Use `pyfakefs`: don't create a file in the host Signed-off-by: Bernhard Kaindl --- tests/test_logger.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_logger.py b/tests/test_logger.py index a8bb9bfd..3a14521d 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -10,8 +10,16 @@ from xcp.logger import openLog -def test_openLog_mock_open(): - """Cover xcp.logger.openLog.open_with_codec_handling and check the arguments used for open()""" +def test_openLog_mock_open(fs): + # type(FakeFilesystem) -> None + """ + - Covers xcp.logger.openLog.open_with_codec_handling and + - checks the arguments it uses for open() + + Because needs to call openLog() which creates a logfile, this test uses + the pytest pyfakefs fixture 'fs' which wraps creating it in a virtual fs. + With it, this tests does not create a log file on the filesystem of the host. + """ fh = StringIO() with patch("xcp.compat.open", mock_open()) as open_mock: open_mock.return_value = fh From 0ac581bb2a781a543f49e3d04b38148f9401bbc8 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Fri, 15 Nov 2024 12:00:00 +0100 Subject: [PATCH 4/4] `.github/workflows/main.yml`: Cleanup obsolete code for Python 2.7/3.8 Signed-off-by: Bernhard Kaindl --- .github/workflows/main.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6d54fd87..90403a09 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,21 +58,6 @@ jobs: pip install 'virtualenv<20.22' 'tox==4.5.1' tox-gh-actions tox --workdir .github/workflows/.tox --recreate - # tox >= 4.0.0 is needed for using optional-dependencies from pyproject.toml, which is - # is not available for python <= 3.6, so use the python3.8 of Ubuntu-20.04 to install it: - - name: Run tox for 3.6 and 3.8 on ${{ matrix.os }}'s 3.8 to get 'extras' from pyproject.toml) - if: ${{ matrix.python-version == 2.7 || matrix.python-version == 3.6 }} - run: | - set -xv;curl -sSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py - python3.8 get-pip.py - # The alternative is installing python3-pip but we don't need full pip function for now: - # sudo apt-get update && sudo apt-get install -y python3-pip - # Let tox-gh-actions get the environment(s) to run tests with from tox.ini: - # Use tox==4.5.1: tox>=4 is needed for reading the extras from pyproject.toml - # Warning: tox>=4.5.2 depends on virutalenv>=20.23, which breaks Python 2.7: - python3.8 -m pip install 'virtualenv<20.22' 'tox==4.5.1' tox-gh-actions - tox --workdir .github/workflows/.tox --recreate - - name: Select the coverage file for upload if: | ( matrix.python-version == '3.6' || matrix.python-version == '3.11' ) &&