diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fd2e4f3b7..472e27ad9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,70 +6,99 @@ on: - main pull_request: +concurrency: + # For a given workflow, if we push to the same branch, cancel all previous builds on that branch except on main. + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + jobs: lint: name: Linting Suite runs-on: ubuntu-latest steps: - - name: Cancel previous runs - uses: styfle/cancel-workflow-action@0.11.0 - with: - access_token: ${{ github.token }} - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 with: python-version: "3.10" + cache: 'pip' - name: Install tox run: | - pip install tox>=4.0 + pip install tox>=4.30.3 - name: Run linting suite ⚙️ run: | tox -e lint test: - name: Testing with Python${{ matrix.python-version }} + name: Testing with (Python${{ matrix.python-version }}, ${{ matrix.os }}) needs: lint - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }}-latest strategy: matrix: + python-version: [ "3.10", "3.11", "3.12", "3.13" ] + os: ["ubuntu"] include: - - tox-env: py310-extra - python-version: "3.10" - - tox-env: py311-extra - python-version: "3.11" - - tox-env: py312-extra - python-version: "3.12" - - tox-env: py313-extra - python-version: "3.13" + # - python-version: "3.11" + # os: "windows" + - python-version: "3.11" + os: "macos" steps: - - uses: actions/checkout@v4 - - name: Install packages 📦 - run: | - sudo apt-get update - sudo apt-get -y install libnetcdf-dev libhdf5-dev - - uses: actions/setup-python@v5 - name: Setup Python ${{ matrix.python-version }} - with: - python-version: ${{ matrix.python-version }} - - name: Install tox 📦 - run: pip install "tox>=4.0" - - name: Run tests with tox ⚙️ - run: tox -e ${{ matrix.tox-env }} - - name: Run coveralls ⚙️ - if: matrix.python-version == 3.10 - uses: AndreMiras/coveralls-python-action@develop + - uses: actions/checkout@v6 + - name: Install packages 📦 (Linux) + if: ${{ matrix.os == 'ubuntu' }} + run: | + sudo apt-get update + sudo apt-get -y install libnetcdf-dev libhdf5-dev + - name: Install packages 📦 (macOS) + if: ${{ matrix.os == 'macos' }} + uses: tecolicom/actions-use-homebrew-tools@b9c066b79607fa3d71e0be05d7003bb75fd9ff34 # v1.3 + with: + tools: hdf5 netcdf + cache: "yes" + - uses: actions/setup-python@v6 + name: Setup Python ${{ matrix.python-version }} + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install tox 📦 + run: | + python -m pip install "tox>=4.30.3" "tox-gh>=1.5" + - name: Run tests with tox ⚙️ + run: | + tox + env: + TOX_GH_MAJOR_MINOR: ${{ matrix.python-version }} + - name: Report Coverage + uses: coverallsapp/github-action@v2 + with: + flag-name: run-${{ matrix.python-version }} + parallel: true docs: name: Build docs 🏗️ needs: lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 name: Setup Python 3.10 with: python-version: "3.10" + cache: 'pip' - name: Build documentation 🏗️ run: | pip install -e .[dev] cd docs && make html + + finish: + name: Finish + needs: + - test + - docs + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Coveralls Finished + uses: coverallsapp/github-action@v2 + with: + parallel-finished: true diff --git a/.gitignore b/.gitignore index 123e0315a..56fc153ec 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ docs/_build *.orig .coverage .pytest_cache +coverage.lcov \ No newline at end of file diff --git a/.readthedocs.yml b/.readthedocs.yml index 317d25e57..efc2f6e88 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -10,7 +10,7 @@ formats: - epub build: - os: ubuntu-22.04 + os: "ubuntu-24.04" tools: python: "3.11" # Use standard CPython version; `mambaforge-22.9` is not valid here diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index be0c2d9b5..83e275616 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -150,7 +150,7 @@ for stable releases and managed exclusively by the PyWPS team. # clone the repository locally $ git clone git@github.com:USERNAME/pywps.git $ cd pywps - $ pip install -e . && pip install -r requirements.txt + $ pip install -e . # add the main PyWPS development branch to keep up to date with upstream changes $ git remote add upstream https://github.com/geopython/pywps.git diff --git a/README.md b/README.md index a035da75e..41df0b028 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ As of PyWPS 4.0.0, PyWPS is released under an ## Dependencies -See [requirements.txt](requirements.txt) file +See [pyproject.toml](pyproject.toml) file ## Install @@ -34,11 +34,11 @@ $ pip install . ## Run tests ```bash -pip install -r requirements-dev.txt +pip install ".[dev]" # run unit tests python -m pytest tests # run code coverage -python -m coverage run --source=pywps -m unittest tests +python -m coverage run --source=pywps -m pytest python -m coverage report -m ``` diff --git a/docs/conf.py b/docs/conf.py index a4becd261..59668a73e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -81,9 +81,6 @@ def __getattr__(cls, name): MOCK_MODULES = ['lxml', 'lxml.etree', 'lxml.builder'] -# with open('../requirements.txt') as f: -# MOCK_MODULES = f.read().splitlines() - for mod_name in MOCK_MODULES: sys.modules[mod_name] = Mock() diff --git a/docs/extensions.rst b/docs/extensions.rst index 84446726f..d96fab2a0 100644 --- a/docs/extensions.rst +++ b/docs/extensions.rst @@ -40,7 +40,7 @@ The scheduler extension uses the `DRMAA`_ library to talk to the different scheduler systems. Install the additional Python dependencies using pip:: - $ pip install -r requirements-processing.txt # drmaa + $ pip install ".[processing]" # drmaa If you are using the `conda `_ package manager you can install the dependencies with:: @@ -95,7 +95,6 @@ Slurm. Docker Container Extension --------------------------- - .. todo:: This extension is on our wish list. In can be used to encapsulate and control the execution of a process. It enhances also the use case of Web Processing Services in a cloud computing infrastructure. diff --git a/docs/install.rst b/docs/install.rst index 421e2d660..b51c7cb6a 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -12,7 +12,7 @@ Installation Dependencies and requirements ----------------------------- -PyWPS runs on Python 3.7 or higher. PyWPS is currently tested and developed on Linux (mostly Ubuntu). +PyWPS runs on Python 3.10 or higher. PyWPS is currently tested and developed on Linux (mostly Ubuntu). In the documentation we take this distribution as reference. Prior to installing PyWPS, Git and the Python bindings for GDAL must be @@ -61,15 +61,9 @@ Manual installation $ tar zxf pywps-x.y.z.tar.gz $ cd pywps-x.y.z/ - Then install the package dependencies using pip:: + To install PyWPS system-wide with all dependencies run:: - $ pip install -r requirements.txt - $ pip install -r requirements-gdal.txt # for GDAL Python bindings (if python-gdal is not already installed by `apt-get`) - $ pip install -r requirements-dev.txt # for developer tasks - - To install PyWPS system-wide run:: - - $ sudo pip install . + $ sudo pip install ".[dev,extra] # for developer tasks and GDAL Python bindings (if python-gdal is not already installed by `apt-get`) For Developers Installation of the source code using Git and Python's virtualenv tool:: diff --git a/environment.yml b/environment.yml index 552d89a07..eed5704e7 100644 --- a/environment.yml +++ b/environment.yml @@ -4,20 +4,27 @@ channels: dependencies: - python >=3.10,<3.14 - pip >=25.0 + - humanize >=4.5.0 + - jinja2 >=3.1.0 + - jsonschema >=4.20.0 + - lxml >=6.0.2 + - markupsafe >=3.0.3 + - numpy >=1.22.2 - owslib >=0.35.0 + - python-dateutil - requests >=2.32.5 - - werkzeug >=3.1.4 - sqlalchemy >=2.0.44 - - lxml >=6.0.2 - urllib3 >=2.5.0 - - markupsafe >=3.0.3 - - numpy >=1.22.2 - - zarr <3 + - werkzeug >=3.1.4 + # extras - fiona - geotiff + - netcdf4 + - tifffile <=2025.5.10 + - zarr <3.0 # tests - pytest - - ruff >=0.5.7 + - ruff >=0.14.0 # docs - sphinx >=7.0.0 diff --git a/pyproject.toml b/pyproject.toml index 69c40b3e8..cf96b962b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,124 @@ [build-system] -requires = ["setuptools"] +requires = ["setuptools>=80", "wheel"] build-backend = "setuptools.build_meta" +[project] +name = "pywps" +description = "PyWPS is an implementation of the Web Processing Service standard from the Open Geospatial Consortium. PyWPS is written in Python." +readme = { file = "README.md", content-type = "text/markdown" } +license = { text = "MIT" } +authors = [ + { name = "Jachym Cepicky", email = "jachym.cepicky@gmail.com" } +] +maintainers = [ + { name = "Jachym Cepicky", email = "jachym.cepicky@gmail.com" } +] +keywords = ["PyWPS", "WPS", "OGC", "processing"] +requires-python = ">=3.10" +dynamic = ["version"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Scientific/Engineering :: GIS", +] +urls = {Homepage = "https://pywps.org", Repository = "https://github.com/geopython/pywps"} +dependencies = [ + "humanize >=4.5.0", + "jinja2 >=3.1.0", + "jsonschema >=4.20.0", + "lxml >=6.0.2", + "markupsafe >=3.0.3", + "numpy >=1.22.2", # not directly required, pinned by Snyk to avoid a vulnerability + "owslib >=0.35.0", + "python-dateutil", + "requests >=2.32.5", + "sqlalchemy >=2.0.44", + "urllib3 >=2.5.0", # not directly required, pinned by Snyk to avoid a vulnerability + "werkzeug >=3.1.4", +] + +[project.optional-dependencies] +dev = [ + "bump-my-version", + "coverage", + "docutils", + "pylint", + "pytest", + "pytest-cov", + "ruff >=0.14.0", + "sphinx", + "tox >=4.30.3", + "twine", + "wheel" +] +extra = [ + "fiona", + "geotiff", + "netCDF4", + "tifffile <=2025.5.10", + "zarr <3.0"] +processing = ["drmaa"] +s3 = ["boto3"] + +[project.scripts] +joblauncher = "pywps.processing.job:launcher" + +[tool.bumpversion] +current_version = "4.7.0" +commit = false +tag = false +parse = "(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)" +serialize = ["{major}.{minor}.{patch}"] + +[[tool.bumpversion.files]] +filename = "pywps/__init__.py" +search = '__version__ = "{current_version}"' +replace = '__version__ = "{new_version}"' + +[[tool.bumpversion.files]] +filename = "VERSION.txt" +search = "{current_version}" +replace = "{new_version}" + +[tool.coverage.run] +relative_files = true + +[tool.pytest.ini_options] +addopts = [ + "--import-mode=importlib", +] +xfail_strict = true +pythonpath = ["tests"] +testpaths = ["tests"] +markers = [ + "online: marks tests requiring network", + "requires_fiona: marks tests requiring fiona module", + "requires_geotiff: marks tests requiring geotiff module", + "requires_netcdf4: marks tests requiring netcdf4 module", +] + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +exclude = ["docs", "tests", "tests.*"] + +[tool.setuptools.dynamic] +version = { file = ["VERSION.txt"] } + [tool.ruff] lint.select = ["E", "W", "F", "C90"] # Flake8-equivalent rule families lint.ignore = ["F401", "E402", "C901"] - line-length = 120 exclude = ["tests"] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 1f1f6bb94..000000000 --- a/pytest.ini +++ /dev/null @@ -1,4 +0,0 @@ -[pytest] -addopts = --import-mode=importlib -pythonpath = tests -testpaths = tests diff --git a/pywps/configuration.py b/pywps/configuration.py index 455519f13..33796059e 100755 --- a/pywps/configuration.py +++ b/pywps/configuration.py @@ -275,16 +275,21 @@ def get_size_mb(mbsize): import re - units = re.compile("[gmkb].*") - newsize = float(re.sub(units, '', size)) - - if size.find("g") > -1: - newsize *= 1024 - elif size.find("m") > -1: - newsize *= 1 - elif size.find("k") > -1: - newsize /= 1024 - else: - newsize *= 1 - LOGGER.debug('Calculated real size of {} is {}'.format(mbsize, newsize)) - return newsize + match = re.fullmatch(r"([0-9]*\.?[0-9]+)\s*(g|m|k|gb|mb|b)?", size) + if not match: + raise ValueError(f"Invalid size format: {mbsize}") + + value = float(match.group(1)) + unit = match.group(2) + + if unit in ("g", "gb"): + value *= 1024 + elif unit in ("m", "mb"): + value *= 1 + elif unit in ("k", "kb"): + value /= 1024 + elif unit in ("b"): + value /= 1024**2 + + LOGGER.debug('Calculated real size of {} is {}'.format(mbsize, value)) + return value diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index e7c0b4657..000000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,12 +0,0 @@ -bump2version -coverage -coveralls -docutils -ruff -pylint -pytest -pytest-cov -sphinx -tox>=4.0 -twine -wheel diff --git a/requirements-extra.txt b/requirements-extra.txt deleted file mode 100644 index 7d438f194..000000000 --- a/requirements-extra.txt +++ /dev/null @@ -1 +0,0 @@ -netCDF4 diff --git a/requirements-processing.txt b/requirements-processing.txt deleted file mode 100644 index 91ffcad8d..000000000 --- a/requirements-processing.txt +++ /dev/null @@ -1 +0,0 @@ -drmaa diff --git a/requirements-s3.txt b/requirements-s3.txt deleted file mode 100644 index 1db657b6b..000000000 --- a/requirements-s3.txt +++ /dev/null @@ -1 +0,0 @@ -boto3 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index dc85b1c08..000000000 --- a/requirements.txt +++ /dev/null @@ -1,16 +0,0 @@ -markupsafe -sqlalchemy -fiona -geotiff -tifffile <=2025.5.10 -zarr <3 -humanize -jinja2 -jsonschema -lxml -owslib -python-dateutil -requests -werkzeug -urllib3>=2.5.0 # not directly required, pinned by Snyk to avoid a vulnerability -numpy>=1.22.2 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 96b6f1ed8..000000000 --- a/setup.cfg +++ /dev/null @@ -1,18 +0,0 @@ -[bumpversion] -current_version = 4.7.0 -commit = False -tag = False -parse = (?P\d+)\.(?P\d+).(?P\d+) -serialize = - {major}.{minor}.{patch} - -[bumpversion:file:pywps/__init__.py] -search = __version__ = "{current_version}" -replace = __version__ = "{new_version}" - -[bumpversion:file:VERSION.txt] -search = {current_version} -replace = {new_version} - -[coverage:run] -relative_files = True diff --git a/setup.py b/setup.py deleted file mode 100644 index b280f852a..000000000 --- a/setup.py +++ /dev/null @@ -1,73 +0,0 @@ -################################################################## -# Copyright 2018 Open Source Geospatial Foundation and others # -# licensed under MIT, Please consult LICENSE.txt for details # -################################################################## - -from setuptools import find_packages, setup - -with open("VERSION.txt") as ff: - VERSION = ff.read().strip() - -DESCRIPTION = ( - "PyWPS is an implementation of the Web Processing Service " - "standard from the Open Geospatial Consortium. PyWPS is " - "written in Python." -) - -with open("README.md") as ff: - LONG_DESCRIPTION = ff.read() - -KEYWORDS = "PyWPS WPS OGC processing" - -with open("requirements.txt") as fr: - INSTALL_REQUIRES = fr.read().splitlines() - -with open("requirements-dev.txt") as frd: - DEV_REQUIRES = frd.read().splitlines() - -CONFIG = { - "name": "pywps", - "version": VERSION, - "description": DESCRIPTION, - "long_description": LONG_DESCRIPTION, - "long_description_content_type": "text/markdown", - "keywords": KEYWORDS, - "license": "MIT", - "platforms": "all", - "author": "Jachym Cepicky", - "author_email": "jachym.cepicky@gmail.com", - "maintainer": "Jachym Cepicky", - "maintainer_email": "jachym.cepicky@gmail.com", - "url": "https://pywps.org", - "download_url": "https://github.com/geopython/pywps", - "classifiers": [ - "Development Status :: 5 - Production/Stable", - "Environment :: Web Environment", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Topic :: Scientific/Engineering :: GIS", - ], - "install_requires": INSTALL_REQUIRES, - "extras_require": dict( - dev=DEV_REQUIRES, - ), - "python_requires": ">=3.10,<4", - "packages": find_packages(exclude=["docs", "tests.*", "tests"]), - "include_package_data": True, - "scripts": [], - "entry_points": { - "console_scripts": [ - "joblauncher=pywps.processing.job:launcher", - ] - }, -} - -setup(**CONFIG) diff --git a/tests/__init__.py b/tests/__obsolete_init__.py similarity index 97% rename from tests/__init__.py rename to tests/__obsolete_init__.py index 560137a2f..e492d1e4b 100644 --- a/tests/__init__.py +++ b/tests/__obsolete_init__.py @@ -19,7 +19,6 @@ import test_exceptions import test_inout import test_literaltypes -import validator import test_ows import test_formats import test_dblog @@ -27,7 +26,7 @@ import test_service import test_process import test_processing -import test_assync +import test_async import test_grass_location import test_storage import test_filestorage @@ -92,7 +91,7 @@ def load_tests(loader=None, tests=None, pattern=None): test_service.load_tests(), test_process.load_tests(), test_processing.load_tests(), - test_assync.load_tests(), + test_async.load_tests(), test_grass_location.load_tests(), test_storage.load_tests(), test_filestorage.load_tests(), diff --git a/tests/processes/__init__.py b/tests/common.py similarity index 100% rename from tests/processes/__init__.py rename to tests/common.py diff --git a/tests/test_assync.py b/tests/test_async.py similarity index 94% rename from tests/test_assync.py rename to tests/test_async.py index 38c39cb01..d6ea5b869 100644 --- a/tests/test_assync.py +++ b/tests/test_async.py @@ -6,14 +6,13 @@ from basic import TestBase import pytest import time +import platform from pywps import Service, configuration from pywps import get_ElementMakerForVersion from pywps.tests import client_for, assert_response_accepted, assert_response_success -from processes import Sleep +from common import Sleep from owslib.wps import WPSExecution from pathlib import Path -from tempfile import TemporaryDirectory -from pywps import dblog VERSION = "1.0.0" @@ -27,7 +26,7 @@ def setUp(self) -> None: # Running processes using the MultiProcessing scheduler and a file-based database configuration.CONFIG.set('processing', 'mode', 'distributed') - @pytest.mark.xfail(reason="async fails") + @pytest.mark.xfail(platform.system() == "Darwin", reason="Response failures on macOS.") def test_async(self): client = client_for(Service(processes=[Sleep()])) wps = WPSExecution() @@ -48,7 +47,6 @@ def test_async(self): # Parse response to extract the status file path url = resp.xml.xpath("//@statusLocation")[0] - print(url) # OWSlib only reads from URLs, not local files. So we need to read the response manually. p = Path(configuration.get_config_value('server', 'outputpath')) / url.split('/')[-1] diff --git a/tests/test_assync_inout.py b/tests/test_async_inout.py similarity index 100% rename from tests/test_assync_inout.py rename to tests/test_async_inout.py diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 1b3901404..5935062af 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -9,6 +9,7 @@ import os import random +import pytest from pywps import configuration @@ -51,7 +52,6 @@ def test_dont_expand_value_without_env_variable(self): configuration.CONFIG.read_string("[envinterpolationsection]\nuser=" + key) assert key == configuration.CONFIG["envinterpolationsection"]["user"] - def load_tests(loader=None, tests=None, pattern=None): """Load the tests and return the test suite for this file.""" import unittest @@ -62,3 +62,17 @@ def load_tests(loader=None, tests=None, pattern=None): loader.loadTestsFromTestCase(TestEnvInterpolation), ] return unittest.TestSuite(suite_list) + + +@pytest.mark.parametrize( +"input_value, expected", +[ + ("1024k", 1.0), + ("2g", 2048.0), + ("20MB", 20.0), + ("1GB", 1024.0), # case-insensitive + ("104857600B", 100.0), +], +) +def test_get_size(input_value, expected): + assert configuration.get_size_mb(input_value) == expected diff --git a/tests/test_execute.py b/tests/test_execute.py index bcc2f0a30..c9611d367 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -22,12 +22,11 @@ from io import StringIO +netCDF4 = None try: import netCDF4 except ImportError: - WITH_NC4 = False -else: - WITH_NC4 = True + pass DATA_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data') @@ -235,10 +234,10 @@ def get_output(doc): class ExecuteTest(TestBase): """Test for Exeucte request KVP request""" - @pytest.mark.xfail(reason="test.opendap.org is offline") + @pytest.mark.online + @pytest.mark.requires_netcdf4 + @pytest.mark.skipif(netCDF4 is None, reason='netCDF4 libraries are required for this test') def test_dods(self): - if not WITH_NC4: - self.skipTest('netCDF4 not installed') my_process = create_complex_nc_process() service = Service(processes=[my_process]) @@ -281,8 +280,11 @@ class FakeRequest(): language = "en-US" request = FakeRequest() - resp = service.execute('my_opendap_process', request, 'fakeuuid') + + if resp.outputs["conventions"].data is None: + pytest.xfail("Network is likely unavailable or test.opendap.org is offline") + self.assertEqual(resp.outputs['conventions'].data, 'CF-1.0') self.assertEqual(resp.outputs['outdods'].url, href) self.assertTrue(resp.outputs['outdods'].as_reference) diff --git a/tests/test_inout.py b/tests/test_inout.py index 6d532892a..b4d027d00 100644 --- a/tests/test_inout.py +++ b/tests/test_inout.py @@ -15,6 +15,7 @@ import json from pywps import inout import base64 +import pytest from pywps import Format, FORMATS from pywps.app.Common import Metadata @@ -128,6 +129,7 @@ def test_file(self): with self.assertRaises(TypeError): self.iohandler[0].data = '5' + @pytest.mark.online def test_url(self): if not service_ok('https://demo.mapserver.org'): self.skipTest("mapserver is unreachable") @@ -552,6 +554,7 @@ def test_base64(self): b = self.complex_out.base64 self.assertEqual(base64.b64decode(b).decode(), self.data) + @pytest.mark.online def test_url_handler(self): wfsResource = 'http://demo.mapserver.org/cgi-bin/wfs?' \ 'service=WFS&version=1.1.0&' \ @@ -798,6 +801,7 @@ def test_json(self): ) +@pytest.mark.online class TestMetaLink(TestBase): def setUp(self) -> None: diff --git a/tests/test_ows.py b/tests/test_ows.py index de354c65d..188908054 100644 --- a/tests/test_ows.py +++ b/tests/test_ows.py @@ -14,6 +14,7 @@ from pywps import get_ElementMakerForVersion import pywps.configuration as config from pywps.tests import client_for, assert_response_success, service_ok +import pytest wfsResource = 'https://demo.mapserver.org/cgi-bin/wfs?service=WFS&version=1.1.0&request=GetFeature&typename=continents&maxfeatures=10' # noqa wcsResource = 'https://demo.mapserver.org/cgi-bin/wcs?service=WCS&version=1.0.0&request=GetCoverage&coverage=ndvi&crs=EPSG:4326&bbox=-92,42,-85,45&format=image/tiff&width=400&height=300' # noqa @@ -91,6 +92,7 @@ def sum_one(request, response): supported_formats=[get_format('GEOTIFF')])], grass_location='epsg:4326') + @pytest.mark.online def test_wfs(self): if not service_ok('https://demo.mapserver.org'): self.skipTest("mapserver is unreachable") @@ -117,6 +119,7 @@ def test_wfs(self): # . the inclusion of output # . the type of output + @pytest.mark.online def test_wcs(self): if not config.CONFIG.get('grass', 'gisbase'): self.skipTest('GRASS lib not found') diff --git a/tests/test_processing.py b/tests/test_processing.py index c7fa2f046..92a6d1f05 100644 --- a/tests/test_processing.py +++ b/tests/test_processing.py @@ -18,7 +18,7 @@ from pywps.app import WPSRequest from pywps.response.execute import ExecuteResponse -from processes import Greeter, InOut, BBox +from common import Greeter, InOut, BBox class GreeterProcessingTest(TestBase): diff --git a/tests/test_s3storage.py b/tests/test_s3storage.py index 8ee694fab..448ff9f1c 100644 --- a/tests/test_s3storage.py +++ b/tests/test_s3storage.py @@ -4,7 +4,7 @@ ################################################################## from basic import TestBase -from pywps.inout.storage.s3 import S3StorageBuilder, S3Storage +from pywps.inout.storage.s3 import S3StorageBuilder from pywps.inout.storage import STORE_TYPE from pywps.inout.basic import ComplexOutput @@ -41,7 +41,7 @@ def test_write(self, uploadData): configuration.CONFIG.set('s3', 'prefix', 'wps') storage = S3StorageBuilder().build() - url = storage.write('Bar Baz', 'out.txt', data_format=FORMATS.TEXT) + storage.write('Bar Baz', 'out.txt', data_format=FORMATS.TEXT) called_args = uploadData.call_args[0] diff --git a/tests/test_storage.py b/tests/test_storage.py index 59ac081cc..ca888f810 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -3,7 +3,6 @@ # licensed under MIT, Please consult LICENSE.txt for details # ################################################################## from basic import TestBase -import pytest from pywps.inout.storage.builder import StorageBuilder from pywps.inout.storage.file import FileStorage @@ -58,3 +57,18 @@ def test_recursive_directory_creation(self): fn = FakeOutput(self.tmpdir.name) storage.store(fn) assert os.path.exists(self.opath) + + +def load_tests(loader=None, tests=None, pattern=None): + """Load local tests + """ + import unittest + + if not loader: + loader = unittest.TestLoader() + suite_list = [ + loader.loadTestsFromTestCase(TestDefaultStorageBuilder), + loader.loadTestsFromTestCase(TestS3StorageBuilder), + loader.loadTestsFromTestCase(TestFileStorageBuilder) + ] + return unittest.TestSuite(suite_list) \ No newline at end of file diff --git a/tests/validator/__init__.py b/tests/validator/__init__.py deleted file mode 100644 index 721932794..000000000 --- a/tests/validator/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -################################################################## -# Copyright 2018 Open Source Geospatial Foundation and others # -# licensed under MIT, Please consult LICENSE.txt for details # -################################################################## diff --git a/tests/validator/test_complexvalidators.py b/tests/validator/test_complexvalidators.py index 032fe027e..5c01ff6ae 100644 --- a/tests/validator/test_complexvalidators.py +++ b/tests/validator/test_complexvalidators.py @@ -7,6 +7,7 @@ """ from basic import TestBase +import importlib.util as ilu import pytest from pywps.validator.mode import MODE from pywps.validator.complexvalidator import ( @@ -27,12 +28,9 @@ import os -try: - import netCDF4 # noqa -except ImportError: - WITH_NC4 = False -else: - WITH_NC4 = True +HAS_NETCDF4 = bool(ilu.find_spec("netCDF4")) +HAS_GEOTIFF = bool(ilu.find_spec("geotiff")) +HAS_FIONA = bool(ilu.find_spec("fiona")) class ValidateTest(TestBase): @@ -69,9 +67,11 @@ class data_format(object): return fake_input + @pytest.mark.online + @pytest.mark.requires_fiona + @pytest.mark.skipif(not HAS_FIONA, reason="fiona libraries are required for this test") def test_gml_validator(self): - """Test GML validator - """ + """Test GML validator""" gml_input = self.get_input('gml/point.gml', 'point.xsd', FORMATS.GML.mime_type) self.assertTrue(validategml(gml_input, MODE.NONE), 'NONE validation') self.assertTrue(validategml(gml_input, MODE.SIMPLE), 'SIMPLE validation') @@ -79,26 +79,40 @@ def test_gml_validator(self): # self.assertTrue(validategml(gml_input, MODE.VERYSTRICT), 'VERYSTRICT validation') gml_input.stream.close() + @pytest.mark.online + @pytest.mark.skipif(HAS_FIONA, reason="fiona libraries must not be installed for this test") + def test_no_gml_validator(self): + """Test GML validator""" + gml_input = self.get_input('gml/point.gml', 'point.xsd', FORMATS.GML.mime_type) + self.assertTrue(validategml(gml_input, MODE.NONE), 'NONE validation') + self.assertTrue(validategml(gml_input, MODE.SIMPLE), 'SIMPLE validation') + self.assertFalse(validategml(gml_input, MODE.STRICT), 'STRICT validation') + # self.assertTrue(validategml(gml_input, MODE.VERYSTRICT), 'VERYSTRICT validation') + gml_input.stream.close() + + @pytest.mark.online + @pytest.mark.requires_fiona @pytest.mark.xfail(reason="gml verystrict validation fails") + @pytest.mark.skipif(not HAS_FIONA, reason="fiona libraries are required for this test") def test_gml_validator_verystrict(self): - """Test GML validator - """ + """Test GML validator""" gml_input = self.get_input('gml/point.gml', 'point.xsd', FORMATS.GML.mime_type) self.assertTrue(validategml(gml_input, MODE.VERYSTRICT), 'VERYSTRICT validation') gml_input.stream.close() + def test_json_validator(self): - """Test GeoJSON validator - """ + """Test GeoJSON validator""" json_input = self.get_input('json/point.geojson', None, FORMATS.JSON.mime_type) self.assertTrue(validatejson(json_input, MODE.NONE), 'NONE validation') self.assertTrue(validatejson(json_input, MODE.SIMPLE), 'SIMPLE validation') self.assertTrue(validatejson(json_input, MODE.STRICT), 'STRICT validation') json_input.stream.close() + @pytest.mark.requires_fiona + @pytest.mark.skipif(not HAS_FIONA, reason="fiona libraries are required for this test") def test_geojson_validator(self): - """Test GeoJSON validator - """ + """Test GeoJSON validator""" geojson_input = self.get_input('json/point.geojson', 'json/schema/geojson.json', FORMATS.GEOJSON.mime_type) self.assertTrue(validategeojson(geojson_input, MODE.NONE), 'NONE validation') @@ -107,9 +121,26 @@ def test_geojson_validator(self): self.assertTrue(validategeojson(geojson_input, MODE.VERYSTRICT), 'VERYSTRICT validation') geojson_input.stream.close() + + @pytest.mark.skipif(HAS_FIONA, reason="fiona libraries must not be installed for this test") + def test_no_geojson_validator(self): + """Test GeoJSON validator""" + geojson_input = self.get_input('json/point.geojson', 'json/schema/geojson.json', + FORMATS.GEOJSON.mime_type) + self.assertTrue(validategeojson(geojson_input, MODE.NONE), 'NONE validation') + self.assertTrue(validategeojson(geojson_input, MODE.SIMPLE), 'SIMPLE validation') + + self.assertFalse(validategeojson(geojson_input, MODE.STRICT), 'STRICT validation') + + # FIXME: MODE.VERYSTRICT should fail here + self.assertTrue(validategeojson(geojson_input, MODE.VERYSTRICT), 'VERYSTRICT validation') + + geojson_input.stream.close() + + @pytest.mark.requires_fiona + @pytest.mark.skipif(not HAS_FIONA, reason="fiona libraries are required for this test") def test_shapefile_validator(self): - """Test ESRI Shapefile validator - """ + """Test ESRI Shapefile validator""" shapefile_input = self.get_input('shp/point.shp.zip', None, FORMATS.SHP.mime_type) self.assertTrue(validateshapefile(shapefile_input, MODE.NONE), 'NONE validation') @@ -117,9 +148,30 @@ def test_shapefile_validator(self): self.assertTrue(validateshapefile(shapefile_input, MODE.STRICT), 'STRICT validation') shapefile_input.stream.close() + @pytest.mark.skipif(HAS_FIONA, reason="fiona libraries must not be installed for this test") + def test_no_shapefile_validator(self): + """Test ESRI Shapefile validator""" + shapefile_input = self.get_input('shp/point.shp.zip', None, + FORMATS.SHP.mime_type) + self.assertTrue(validateshapefile(shapefile_input, MODE.NONE), 'NONE validation') + self.assertTrue(validateshapefile(shapefile_input, MODE.SIMPLE), 'SIMPLE validation') + self.assertFalse(validateshapefile(shapefile_input, MODE.STRICT), 'STRICT validation') + shapefile_input.stream.close() + + @pytest.mark.skipif(HAS_GEOTIFF, reason="geotiff libraries must not be installed for this test") + def test_no_geotiff_validator(self): + """Test GeoTIFF validator""" + geotiff_input = self.get_input('geotiff/dem.tiff', None, + FORMATS.GEOTIFF.mime_type) + self.assertTrue(validategeotiff(geotiff_input, MODE.NONE), 'NONE validation') + self.assertTrue(validategeotiff(geotiff_input, MODE.SIMPLE), 'SIMPLE validation') + self.assertFalse(validategeotiff(geotiff_input, MODE.STRICT), 'STRICT validation') + geotiff_input.stream.close() + + @pytest.mark.requires_geotiff + @pytest.mark.skipif(not HAS_GEOTIFF, reason="geotiff libraries are required for this test") def test_geotiff_validator(self): - """Test GeoTIFF validator - """ + """Test GeoTIFF validator""" geotiff_input = self.get_input('geotiff/dem.tiff', None, FORMATS.GEOTIFF.mime_type) self.assertTrue(validategeotiff(geotiff_input, MODE.NONE), 'NONE validation') @@ -127,39 +179,56 @@ def test_geotiff_validator(self): self.assertTrue(validategeotiff(geotiff_input, MODE.STRICT), 'STRICT validation') geotiff_input.stream.close() + @pytest.mark.requires_netcdf4 + @pytest.mark.skipif(not HAS_NETCDF4, reason="NetCDF4 libraries are required for this test") def test_netcdf_validator(self): - """Test netCDF validator - """ + """Test netCDF validator""" + netcdf_input = self.get_input('netcdf/time.nc', None, FORMATS.NETCDF.mime_type) + self.assertTrue(validatenetcdf(netcdf_input, MODE.NONE), 'NONE validation') + self.assertTrue(validatenetcdf(netcdf_input, MODE.SIMPLE), 'SIMPLE validation') + netcdf_input.stream.close() + + self.assertTrue(validatenetcdf(netcdf_input, MODE.STRICT), 'STRICT validation') + netcdf_input.file = 'grub.nc' + self.assertFalse(validatenetcdf(netcdf_input, MODE.STRICT)) + + @pytest.mark.skipif(HAS_NETCDF4, reason="NetCDF4 libraries must not be installed for this test") + def test_no_netcdf_validator(self): + """Test netCDF validator""" netcdf_input = self.get_input('netcdf/time.nc', None, FORMATS.NETCDF.mime_type) self.assertTrue(validatenetcdf(netcdf_input, MODE.NONE), 'NONE validation') self.assertTrue(validatenetcdf(netcdf_input, MODE.SIMPLE), 'SIMPLE validation') netcdf_input.stream.close() - if WITH_NC4: - self.assertTrue(validatenetcdf(netcdf_input, MODE.STRICT), 'STRICT validation') - netcdf_input.file = 'grub.nc' - self.assertFalse(validatenetcdf(netcdf_input, MODE.STRICT)) - else: - self.assertFalse(validatenetcdf(netcdf_input, MODE.STRICT), 'STRICT validation') - - @pytest.mark.xfail(reason="test.opendap.org is offline") + + self.assertFalse(validatenetcdf(netcdf_input, MODE.STRICT), 'STRICT validation') + + @pytest.mark.online + @pytest.mark.requires_netcdf4 + @pytest.mark.skipif(not HAS_NETCDF4, reason="NetCDF4 libraries are required for this test") def test_dods_validator(self): opendap_input = ComplexInput('dods', 'opendap test', [FORMATS.DODS,]) opendap_input.url = "http://test.opendap.org:80/opendap/netcdf/examples/sresa1b_ncar_ccsm3_0_run1_200001.nc" self.assertTrue(validatedods(opendap_input, MODE.NONE), 'NONE validation') self.assertTrue(validatedods(opendap_input, MODE.SIMPLE), 'SIMPLE validation') - if WITH_NC4: - self.assertTrue(validatedods(opendap_input, MODE.STRICT), 'STRICT validation') - opendap_input.url = 'Faulty url' - self.assertFalse(validatedods(opendap_input, MODE.STRICT)) - else: - self.assertFalse(validatedods(opendap_input, MODE.STRICT), 'STRICT validation') + self.assertTrue(validatedods(opendap_input, MODE.STRICT), 'STRICT validation') + opendap_input.url = 'Faulty url' + self.assertFalse(validatedods(opendap_input, MODE.STRICT)) + @pytest.mark.online + @pytest.mark.skipif(HAS_NETCDF4, reason="NetCDF4 libraries must not be installed for this test") def test_dods_default(self): opendap_input = ComplexInput('dods', 'opendap test', [FORMATS.DODS,], default='http://test.opendap.org', default_type=SOURCE_TYPE.URL, mode=MODE.SIMPLE) + opendap_input.url = "http://test.opendap.org:80/opendap/netcdf/examples/sresa1b_ncar_ccsm3_0_run1_200001.nc" + self.assertTrue(validatedods(opendap_input, MODE.NONE), 'NONE validation') + self.assertTrue(validatedods(opendap_input, MODE.SIMPLE), 'SIMPLE validation') + + with pytest.warns(UserWarning) as record: + self.assertFalse(validatedods(opendap_input, MODE.STRICT), 'STRICT validation') + assert "Complex validation requires netCDF4 support." in record[0].message.args[0] def test_fail_validator(self): fake_input = self.get_input('point.xsd', 'point.xsd', FORMATS.SHP.mime_type) diff --git a/tox.ini b/tox.ini index b8d0b61df..41848e6c1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,16 @@ [tox] -min_version = 4.0 envlist = py{310,311,312,313}{-extra,}, lint -requires = pip >=25.2 opts = --verbose +[gh] +python = + 3.10 = py3.10 + 3.11 = py3.11-extra + 3.12 = py3.12 + 3.13 = py3.13-extra + [testenv:lint] skip_install = true extras = @@ -16,7 +21,7 @@ commands = [testenv] setenv = - PYTEST_ADDOPTS = "--color=yes" + PYTEST_ADDOPTS = "--color=yes" "--cov=pywps" "--cov-report=lcov" PYTHONPATH = {toxinidir} COV_CORE_SOURCE = passenv = @@ -26,10 +31,11 @@ passenv = download = True install_command = python -m pip install --no-user {opts} {packages} -extras = dev -deps = - extra: -rrequirements-extra.txt +extras = + dev + extra: extra commands = ; # magic for gathering the GDAL version within tox ; sh -c 'pip install GDAL=="$(gdal-config --version)" --global-option=build_ext --global-option="-I/usr/include/gdal"' - pytest --cov + pytest {posargs} + coverage report