From 5e36c30e5ce77f2d81d1e24739d8aefecc2a9fd8 Mon Sep 17 00:00:00 2001 From: Christian Baur Date: Thu, 8 Dec 2022 16:43:26 +0100 Subject: [PATCH 01/39] #75 fixing whitespaces and code format --- {{cookiecutter.project_slug}}/.gitlab-ci.yml | 16 ++++------- {{cookiecutter.project_slug}}/Dockerfile__pip | 4 +++ {{cookiecutter.project_slug}}/README.md | 8 ++++-- {{cookiecutter.project_slug}}/pyproject.toml | 7 ++--- .../{{cookiecutter.module_name}}/__main__.py | 1 + .../{{cookiecutter.module_name}}/main__cli.py | 28 ++++++++++++++----- .../util__hocon.py | 7 +++++ .../util__yaml.py | 6 ++++ 8 files changed, 54 insertions(+), 23 deletions(-) diff --git a/{{cookiecutter.project_slug}}/.gitlab-ci.yml b/{{cookiecutter.project_slug}}/.gitlab-ci.yml index 0c0e110..316e614 100644 --- a/{{cookiecutter.project_slug}}/.gitlab-ci.yml +++ b/{{cookiecutter.project_slug}}/.gitlab-ci.yml @@ -27,6 +27,7 @@ stages: - deploy-prod # Build and Test + {% if cookiecutter.package_manager == 'poetry' -%} build-wheel: image: python:3-slim @@ -38,8 +39,7 @@ build-wheel: expire_in: 6 mos script: - pip install poetry==$POETRY_VERSION - - poetry build -f wheel -{% else -%} + - poetry build -f wheel {% else -%} build-wheel: image: python:3-slim stage: build @@ -49,8 +49,7 @@ build-wheel: - dist/{{ cookiecutter.module_name }}-*.whl expire_in: 6 mos script: - - python setup.py dist -{%- endif %} + - python setup.py dist {%- endif %} {% if cookiecutter.package_manager == 'poetry' -%} test-unit: @@ -68,8 +67,7 @@ test-unit: - poetry install --no-root - source `poetry env info --path`/bin/activate script: - - pytest tests -{% elif cookiecutter.package_manager == 'conda' -%} + - pytest tests {% elif cookiecutter.package_manager == 'conda' -%} test-unit: stage: test image: continuumio/miniconda3 @@ -88,8 +86,7 @@ test-unit: - source activate .venv - pip install dist/{{ cookiecutter.module_name }}-*.whl script: - - pytest tests -{% elif cookiecutter.package_manager == 'pip' -%} + - pytest tests {% elif cookiecutter.package_manager == 'pip' -%} test-unit: stage: test image: python:3-slim @@ -106,8 +103,7 @@ test-unit: - pip install -r requirements.txt -r requirements-dev.txt - pip install dist/{{ cookiecutter.module_name }}-*.whl script: - - pytest tests -{%- endif %} + - pytest tests {%- endif %} # Nonprod deployments diff --git a/{{cookiecutter.project_slug}}/Dockerfile__pip b/{{cookiecutter.project_slug}}/Dockerfile__pip index c0cb5f8..5fcb9db 100644 --- a/{{cookiecutter.project_slug}}/Dockerfile__pip +++ b/{{cookiecutter.project_slug}}/Dockerfile__pip @@ -6,9 +6,11 @@ ARG PYTHON_IMAGE_TAG=3.7-stretch FROM python:${PYTHON_IMAGE_TAG} as py-build WORKDIR /build + # install and store the requirements (this stage will be re-used unless requirements.txt changes) COPY requirements.txt . RUN pip install -r requirements.txt && pip wheel -r requirements.txt -w deps + # add all other source files, force rebuild only from here COPY . . RUN python setup.py install && python setup.py bdist_wheel @@ -24,9 +26,11 @@ LABEL maintainer="{{ cookiecutter.company_name }}" LABEL maintainer="{{ cookiecutter.full_name }}" {% endif %} WORKDIR /app + # install all dependencies from wheel packages in the 'deps' folder COPY --from=py-build /build/deps/ deps/ RUN [ -n "$(ls -A deps)" ] && pip install deps/*.whl && rm -rf deps || echo "no dependencies to install" + # install the application from a wheel package in the 'dist' folder COPY --from=py-build /build/dist/ dist/ RUN pip install dist/*.whl && rm -rf dist diff --git a/{{cookiecutter.project_slug}}/README.md b/{{cookiecutter.project_slug}}/README.md index 79d34da..936c6e0 100644 --- a/{{cookiecutter.project_slug}}/README.md +++ b/{{cookiecutter.project_slug}}/README.md @@ -30,13 +30,15 @@ To set up your local development environment, please use a fresh virtual environ pip install -r requirements.txt -r requirements-dev.txt pip install -e . -The first command will install all requirements for the application and to execute tests. With the second command, you'll get an editable installation of the module, so that imports work properly. +The first command will install all requirements for the application and to execute tests. +With the second command, you'll get an editable installation of the module, so that imports work properly. {% elif cookiecutter.package_manager == 'poetry' %} To set up your local development environment, please run: poetry install -Behind the scenes, this creates a virtual environment and installs `{{ cookiecutter.module_name }}` along with its dependencies into a new virtualenv. Whenever you run `poetry run `, that `` is actually run inside the virtualenv managed by poetry. +Behind the scenes, this creates a virtual environment and installs `{{ cookiecutter.module_name }}` along with its dependencies into a new virtualenv. +Whenever you run `poetry run `, that `` is actually run inside the virtualenv managed by poetry. {% endif -%} {% if cookiecutter.create_cli == 'yes' %} @@ -85,6 +87,7 @@ This way, you'll always use the latest version of your module code in your noteb Assuming you already have Jupyter installed, you can make your virtual environment available as a separate kernel by running: {{ install_command }} ipykernel + {{ py_command }} -m ipykernel install --user --name="{{ cookiecutter.project_slug }}" Note that we mainly use notebooks for experiments, visualizations and reports. Every piece of functionality that is meant to be reused should go into module code and be imported into notebooks. @@ -102,6 +105,7 @@ this will clean up the build folder and then run the `bdist_wheel` command. Before contributing, please set up the pre-commit hooks to reduce errors and ensure consistency pip install -U pre-commit + pre-commit install If you run into any issues, you can remove the hooks again with `pre-commit uninstall`. diff --git a/{{cookiecutter.project_slug}}/pyproject.toml b/{{cookiecutter.project_slug}}/pyproject.toml index 966770f..8b241c6 100644 --- a/{{cookiecutter.project_slug}}/pyproject.toml +++ b/{{cookiecutter.project_slug}}/pyproject.toml @@ -5,13 +5,12 @@ description = "{{ cookiecutter.project_short_description }}" authors = ["{{ cookiecutter.full_name }} <{{ cookiecutter.email }}>"] license = "Proprietary" packages = [ - { include = "{{ cookiecutter.module_name }}", from = "src" }, -] + { include = "{{ cookiecutter.module_name }}", from = "src" }, ] include = ["src/{{ cookiecutter.module_name }}/res/*"] {% if cookiecutter.create_cli == "yes" %} [tool.poetry.scripts] -{{ cookiecutter.project_slug }} = "{{ cookiecutter.module_name }}.main:app" -{% endif %} +{{ cookiecutter.project_slug }} = "{{ cookiecutter.module_name }}.main:app" {% endif %} + [tool.poetry.dependencies] python = "^3.7"{% if cookiecutter.config_file == 'hocon' %} pyhocon = "^0.3.59"{% elif cookiecutter.config_file == 'yaml' %} diff --git a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/__main__.py b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/__main__.py index e692541..1828e2a 100644 --- a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/__main__.py +++ b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/__main__.py @@ -1,3 +1,4 @@ {% if cookiecutter.create_cli == 'yes' %}from {{ cookiecutter.module_name }}.main import app + app(){% else %}from {{ cookiecutter.module_name }}.main import main main(){% endif %} diff --git a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/main__cli.py b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/main__cli.py index b184458..4d1c892 100644 --- a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/main__cli.py +++ b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/main__cli.py @@ -2,15 +2,14 @@ import typer -from {{ cookiecutter.module_name }} import __title__ -from {{ cookiecutter.module_name }} import __version__{% if cookiecutter.config_file != 'none' %} -from {{ cookiecutter.module_name }} import util{% endif %} +from {{ cookiecutter.module_name }} import __title__ , __version__{% if cookiecutter.config_file != 'none' %}, util{% endif %} logger = logging.getLogger('{{ cookiecutter.module_name }}') app = typer.Typer( name='{{ cookiecutter.module_name }}', - help="{{ cookiecutter.project_short_description }}") + help="{{ cookiecutter.project_short_description }}" +) def version_callback(version: bool): @@ -20,10 +19,22 @@ def version_callback(version: bool): ConfigOption = typer.Option( - {% if cookiecutter.config_file == 'yaml' %}...{% else %}None{% endif %}, '-c', '--config', metavar='PATH', help="path to the program configuration") + {% if cookiecutter.config_file == 'yaml' %}...{% else %}None{% endif %}, + '-c', + '--config', + metavar='PATH', + help="path to the program configuration" +) + + VersionOption = typer.Option( - None, '-v', '--version', callback=version_callback, is_eager=True, - help="print the program version and exit") + None, + '-v', + '--version', + callback=version_callback, + is_eager=True, + help="print the program version and exit" +) @app.command() @@ -35,8 +46,11 @@ def main(config_file: str = ConfigOption, version: bool = VersionOption): This docstring is where you describe what your command line application does. Try running `python -m {{ cookiecutter.module_name }} --help` to see how this shows up in the command line. """ + {% if cookiecutter.config_file != 'none' %}config = util.load_config(config_file) + util.logging_setup(config){% endif %} + logger.info("Looks like you're all set up. Let's get going!") # TODO your journey starts here diff --git a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/util__hocon.py b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/util__hocon.py index 8212aeb..3d3eb6f 100644 --- a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/util__hocon.py +++ b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/util__hocon.py @@ -17,6 +17,7 @@ def get_resource_string(path: str, decode=True) -> Union[str, bytes]: :param decode: if true, decode the file contents as string (otherwise return bytes) :return: the contents of the resource file (as string or bytes) """ + s = pkg_resources.resource_string(__name__.split('.')[0], path) return s.decode(errors='ignore') if decode else s @@ -29,8 +30,10 @@ def load_config(config_file: Union[str, Path] = None) -> ConfigTree: :param config_file: path of the config file to load :return: the parsed config """ + base_config_str = get_resource_string('res/default.conf') config: ConfigTree = ConfigFactory.parse_string(base_config_str) + if config_file: try: config = ConfigFactory.parse_file(config_file).with_fallback(config) @@ -45,10 +48,14 @@ def logging_setup(config: ConfigTree): :param config: the parsed config tree """ + fmt = config.get('logging.format') + if config.get_bool('logging.enabled'): level = logging._nameToLevel[config.get('logging.level').upper()] else: level = logging.NOTSET + logging.basicConfig(format=fmt, level=logging.WARNING) + logger.setLevel(level) diff --git a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/util__yaml.py b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/util__yaml.py index ad44e50..8aedcf2 100644 --- a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/util__yaml.py +++ b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/util__yaml.py @@ -17,6 +17,7 @@ def get_resource_string(path: str, decode=True) -> Union[str, bytes]: :param decode: if true, decode the file contents as string (otherwise return bytes) :return: the contents of the resource file (as string or bytes) """ + s = pkg_resources.resource_string(__name__.split('.')[0], path) return s.decode(errors='ignore') if decode else s @@ -28,6 +29,7 @@ def load_config(config_file: Union[str, Path]) -> Dict[str, Any]: :param config_file: path of the config file to load :return: the parsed config as dictionary """ + with open(config_file, 'r') as fp: return yaml.safe_load(fp) @@ -38,11 +40,15 @@ def logging_setup(config: Dict): :param config: the parsed config tree """ + log_conf = config['logging'] fmt = log_conf['format'] + if log_conf['enabled']: level = logging._nameToLevel[log_conf['level'].upper()] else: level = logging.NOTSET + logging.basicConfig(format=fmt, level=logging.WARNING) + logger.setLevel(level) From a6ffc4c2e73534fa0a28680a63e25132e3ec9be0 Mon Sep 17 00:00:00 2001 From: Christian Baur Date: Fri, 9 Dec 2022 10:12:00 +0100 Subject: [PATCH 02/39] #refined pr --- {{cookiecutter.project_slug}}/Dockerfile__pip | 4 ---- {{cookiecutter.project_slug}}/pyproject.toml | 3 +-- .../src/{{cookiecutter.module_name}}/main__cli.py | 3 --- .../src/{{cookiecutter.module_name}}/util__hocon.py | 7 ------- 4 files changed, 1 insertion(+), 16 deletions(-) diff --git a/{{cookiecutter.project_slug}}/Dockerfile__pip b/{{cookiecutter.project_slug}}/Dockerfile__pip index 5fcb9db..c0cb5f8 100644 --- a/{{cookiecutter.project_slug}}/Dockerfile__pip +++ b/{{cookiecutter.project_slug}}/Dockerfile__pip @@ -6,11 +6,9 @@ ARG PYTHON_IMAGE_TAG=3.7-stretch FROM python:${PYTHON_IMAGE_TAG} as py-build WORKDIR /build - # install and store the requirements (this stage will be re-used unless requirements.txt changes) COPY requirements.txt . RUN pip install -r requirements.txt && pip wheel -r requirements.txt -w deps - # add all other source files, force rebuild only from here COPY . . RUN python setup.py install && python setup.py bdist_wheel @@ -26,11 +24,9 @@ LABEL maintainer="{{ cookiecutter.company_name }}" LABEL maintainer="{{ cookiecutter.full_name }}" {% endif %} WORKDIR /app - # install all dependencies from wheel packages in the 'deps' folder COPY --from=py-build /build/deps/ deps/ RUN [ -n "$(ls -A deps)" ] && pip install deps/*.whl && rm -rf deps || echo "no dependencies to install" - # install the application from a wheel package in the 'dist' folder COPY --from=py-build /build/dist/ dist/ RUN pip install dist/*.whl && rm -rf dist diff --git a/{{cookiecutter.project_slug}}/pyproject.toml b/{{cookiecutter.project_slug}}/pyproject.toml index 8b241c6..79e21ab 100644 --- a/{{cookiecutter.project_slug}}/pyproject.toml +++ b/{{cookiecutter.project_slug}}/pyproject.toml @@ -4,8 +4,7 @@ version = "0.1.0" description = "{{ cookiecutter.project_short_description }}" authors = ["{{ cookiecutter.full_name }} <{{ cookiecutter.email }}>"] license = "Proprietary" -packages = [ - { include = "{{ cookiecutter.module_name }}", from = "src" }, ] +packages = [{ include = "{{ cookiecutter.module_name }}", from = "src" }, ] include = ["src/{{ cookiecutter.module_name }}/res/*"] {% if cookiecutter.create_cli == "yes" %} [tool.poetry.scripts] diff --git a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/main__cli.py b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/main__cli.py index 4d1c892..755a70e 100644 --- a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/main__cli.py +++ b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/main__cli.py @@ -46,11 +46,8 @@ def main(config_file: str = ConfigOption, version: bool = VersionOption): This docstring is where you describe what your command line application does. Try running `python -m {{ cookiecutter.module_name }} --help` to see how this shows up in the command line. """ - {% if cookiecutter.config_file != 'none' %}config = util.load_config(config_file) - util.logging_setup(config){% endif %} - logger.info("Looks like you're all set up. Let's get going!") # TODO your journey starts here diff --git a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/util__hocon.py b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/util__hocon.py index 3d3eb6f..8212aeb 100644 --- a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/util__hocon.py +++ b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/util__hocon.py @@ -17,7 +17,6 @@ def get_resource_string(path: str, decode=True) -> Union[str, bytes]: :param decode: if true, decode the file contents as string (otherwise return bytes) :return: the contents of the resource file (as string or bytes) """ - s = pkg_resources.resource_string(__name__.split('.')[0], path) return s.decode(errors='ignore') if decode else s @@ -30,10 +29,8 @@ def load_config(config_file: Union[str, Path] = None) -> ConfigTree: :param config_file: path of the config file to load :return: the parsed config """ - base_config_str = get_resource_string('res/default.conf') config: ConfigTree = ConfigFactory.parse_string(base_config_str) - if config_file: try: config = ConfigFactory.parse_file(config_file).with_fallback(config) @@ -48,14 +45,10 @@ def logging_setup(config: ConfigTree): :param config: the parsed config tree """ - fmt = config.get('logging.format') - if config.get_bool('logging.enabled'): level = logging._nameToLevel[config.get('logging.level').upper()] else: level = logging.NOTSET - logging.basicConfig(format=fmt, level=logging.WARNING) - logger.setLevel(level) From e77f65fb05e378ef9e72b4f8489d78552a55f1e0 Mon Sep 17 00:00:00 2001 From: Christian Baur Date: Fri, 23 Dec 2022 16:35:33 +0100 Subject: [PATCH 03/39] #refined pr v2 - deleted whitespaces in functions --- .../src/{{cookiecutter.module_name}}/util__yaml.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/util__yaml.py b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/util__yaml.py index 8aedcf2..78b62e8 100644 --- a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/util__yaml.py +++ b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/util__yaml.py @@ -13,11 +13,10 @@ def get_resource_string(path: str, decode=True) -> Union[str, bytes]: Load a package resource (i.e. a file from within this package) :param path: the path, starting at the root of the current module (e.g. 'res/default.conf'). - must be a string, not a Path object! + must be a string, not a Path object! :param decode: if true, decode the file contents as string (otherwise return bytes) :return: the contents of the resource file (as string or bytes) """ - s = pkg_resources.resource_string(__name__.split('.')[0], path) return s.decode(errors='ignore') if decode else s @@ -29,7 +28,6 @@ def load_config(config_file: Union[str, Path]) -> Dict[str, Any]: :param config_file: path of the config file to load :return: the parsed config as dictionary """ - with open(config_file, 'r') as fp: return yaml.safe_load(fp) @@ -40,15 +38,11 @@ def logging_setup(config: Dict): :param config: the parsed config tree """ - log_conf = config['logging'] fmt = log_conf['format'] - if log_conf['enabled']: level = logging._nameToLevel[log_conf['level'].upper()] else: level = logging.NOTSET - logging.basicConfig(format=fmt, level=logging.WARNING) - logger.setLevel(level) From 6dd5b990fa19cd76b43c3b174730aa191a865aa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carsten=20B=C3=B6hm?= Date: Thu, 8 Dec 2022 11:28:18 +0100 Subject: [PATCH 04/39] Use mambaforge as base image --- .../Dockerfile__conda | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/{{cookiecutter.project_slug}}/Dockerfile__conda b/{{cookiecutter.project_slug}}/Dockerfile__conda index 118796a..ec136af 100644 --- a/{{cookiecutter.project_slug}}/Dockerfile__conda +++ b/{{cookiecutter.project_slug}}/Dockerfile__conda @@ -1,21 +1,19 @@ -ARG PYTHON_IMAGE_TAG=4.8.2 +ARG IMAGE_TAG=22.9.0-2 + +FROM condaforge/mambaforge:${IMAGE_TAG} + +LABEL maintainer="{{ cookiecutter.company_name if cookiecutter.company_name else cookiecutter.full_name }}" -FROM continuumio/miniconda3:${PYTHON_IMAGE_TAG} -{%- if cookiecutter.company_name %} -LABEL maintainer="{{ cookiecutter.company_name }}" -{% else %} -LABEL maintainer="{{ cookiecutter.full_name }}" -{% endif -%} WORKDIR /{{cookiecutter.module_name}} COPY . . RUN conda config --set channel_priority strict && \ - conda env create -n {{cookiecutter.module_name}}_env -f environment.yml + mamba env create -n {{cookiecutter.module_name}}_env -f environment.yml # Make RUN commands use the new environment (see: https://pythonspeed.com/articles/activate-conda-dockerfile/) -SHELL ["conda", "run", "-n", "{{cookiecutter.module_name}}_env", "/bin/bash", "-c"] +SHELL ["mamba", "run", "-n", "{{cookiecutter.module_name}}_env", "/bin/bash", "-c"] RUN python setup.py install # ENTRYPOINT doesn't use the same shell as RUN so you need the conda stuff -ENTRYPOINT ["conda", "run", "-n", "{{cookiecutter.module_name}}_env", "python", "-OO", "-m", "{{ cookiecutter.module_name }}"] +ENTRYPOINT ["mamba", "run", "-n", "{{cookiecutter.module_name}}_env", "python", "-OO", "-m", "{{ cookiecutter.module_name }}"] From 3decf1f96036ce78fb49af888ce5096cd90930fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carsten=20B=C3=B6hm?= Date: Fri, 20 Jan 2023 11:40:49 +0100 Subject: [PATCH 05/39] Optimise Dockerfile for caching --- {{cookiecutter.project_slug}}/Dockerfile__conda | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/{{cookiecutter.project_slug}}/Dockerfile__conda b/{{cookiecutter.project_slug}}/Dockerfile__conda index ec136af..e703509 100644 --- a/{{cookiecutter.project_slug}}/Dockerfile__conda +++ b/{{cookiecutter.project_slug}}/Dockerfile__conda @@ -5,7 +5,7 @@ FROM condaforge/mambaforge:${IMAGE_TAG} LABEL maintainer="{{ cookiecutter.company_name if cookiecutter.company_name else cookiecutter.full_name }}" WORKDIR /{{cookiecutter.module_name}} -COPY . . +COPY environment.yml . RUN conda config --set channel_priority strict && \ mamba env create -n {{cookiecutter.module_name}}_env -f environment.yml @@ -13,6 +13,7 @@ RUN conda config --set channel_priority strict && \ # Make RUN commands use the new environment (see: https://pythonspeed.com/articles/activate-conda-dockerfile/) SHELL ["mamba", "run", "-n", "{{cookiecutter.module_name}}_env", "/bin/bash", "-c"] +COPY . . RUN python setup.py install # ENTRYPOINT doesn't use the same shell as RUN so you need the conda stuff From 9257b4fb713b56e510b7f6876f9cb9f93abf8de9 Mon Sep 17 00:00:00 2001 From: Maximilian Pensel Date: Wed, 25 Jan 2023 11:54:43 +0100 Subject: [PATCH 06/39] Change metadata version reference to project_slug --- .../src/{{cookiecutter.module_name}}/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/version.py b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/version.py index c207133..5c86dd1 100644 --- a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/version.py +++ b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/version.py @@ -3,5 +3,5 @@ except ModuleNotFoundError: import importlib_metadata -__version__ = importlib_metadata.version("{{ cookiecutter.module_name }}") +__version__ = importlib_metadata.version("{{ cookiecutter.project_slug }}") {%- else %}__version__ = '0.1.0'{% endif %} From a3a1962c00e151ae3b089608bf56ef58e6a1ae1a Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 13 Feb 2023 17:38:21 +0100 Subject: [PATCH 07/39] Begin to add and adapt files for az-devops option --- README.md | 1 + cd/build-dev.yml | 47 +++++++++++++++ cd/build.yml | 54 +++++++++++++++++ cd/delete-old-images.yml | 39 +++++++++++++ cd/trigger.yml | 68 ++++++++++++++++++++++ ci/test-pipeline.yml | 119 ++++++++++++++++++++++++++++++++++++++ cookiecutter.json | 2 +- hooks/post_gen_project.py | 11 +++- 8 files changed, 339 insertions(+), 2 deletions(-) create mode 100644 cd/build-dev.yml create mode 100644 cd/build.yml create mode 100644 cd/delete-old-images.yml create mode 100644 cd/trigger.yml create mode 100644 ci/test-pipeline.yml diff --git a/README.md b/README.md index 15b7d60..59e5cef 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ Unfortunately, cookiecutter does not allow us to show any description of the opt * Select your `ci_pipeline` - `none` (default): Don't use any CI/CD pipeline. - `gitlab`: If you plan to use GitLab, this option will add a CI/CD Pipeline definition for [GitLab CI/CD](https://docs.gitlab.com/ee/ci/). The pipeline includes basic steps to build, test and deploy your code. The deployment steps do nothing but echoing a String, as deployment is very project-specific. + - `az-devops`: If you plan to use [Azure DevOps](https://azure.microsoft.com/en-us/products/devops), this option will add a CI pipeline and templates for CD pipelines. For the CD pipelines to work, you need to add project specific information. * `create_cli` (yes or no): if you plan to build an application with a command line interface (CLI), select *yes* here. This will integrate a template for the CLI into your project - minimal boilerplate guaranteed! (We're leveraging the awesome [typer](https://typer.tiangolo.com/) library for this.) * `config_file`: select your preferred config format. It is best practice to store your configuration separate from your code, even for small projects, but because there are a gazillion ways to do this, each project seems to reinvents the wheel. We want to provide a few options to set you up with a working configuration: - `yaml`: use [YAML](https://yaml.org/) as your configuration file format. Easy to read and write, widely adopted, relies on the [PyYAML](https://pyyaml.org/) package. diff --git a/cd/build-dev.yml b/cd/build-dev.yml new file mode 100644 index 0000000..17c03d8 --- /dev/null +++ b/cd/build-dev.yml @@ -0,0 +1,47 @@ +# Pipeline which builds the docker image for DEV and pushes it to DEV Container Registry +# +# expected parameters: +# ACR: Azure Container Registry ID to push image to +# ACR_SUB: Service Connection to authenticate against the Azure Container Registry +# REPOSITORY: The repository to push to in the Container Registry +# SUBSCRIPTION: Service Connection used for Azure CLI Task +# IMAGE_COUNT: Number of last images to retain, all others are deleted + +parameters: +- name: ACR + type: string +- name: ACR_SUB + type: string +- name: REPOSITORY + type: string +- name: SUBSCRIPTION + type: string +- name: IMAGE_COUNT + type: number + +jobs: +- job: build-push-docker-image + steps: + - checkout: self + + - task: Docker@2 + inputs: + containerRegistry: ${{ parameters.ACR_SUB }} + repository: ${{ parameters.REPOSITORY }} + command: build + Dockerfile: '**/Dockerfile' + + - task: Docker@2 + inputs: + containerRegistry: ${{ parameters.ACR_SUB }} + repository: ${{ parameters.REPOSITORY }} + command: push + Dockerfile: '**/Dockerfile' + tags: $(Build.BuildId) + + - template: template_delete_old_images.yml + parameters: + ACR: ${{ parameters.ACR }} + SUBSCRIPTION: ${{ parameters.SUBSCRIPTION }} + REPOSITORY: ${{ parameters.REPOSITORY }} + IMAGE_COUNT: ${{ parameters.IMAGE_COUNT}} diff --git a/cd/build.yml b/cd/build.yml new file mode 100644 index 0000000..4ece11c --- /dev/null +++ b/cd/build.yml @@ -0,0 +1,54 @@ +# Pipeline which copies docker image from the Container Registry in a (previous) environment to the next environment +# This will be used to deploy the image DEV -> QA -> PROD +# +# expected parameters: +# ACR_PREVIOUS : Azure Container Registry to pull Docker image from +# ACR_NEXT: Azure Container Registry to push Docker image to +# ACR_SUB_PREVIOUS: Service Connection used for authentication with Container Registry OLD +# ACR_SUB_NEXT: Service Connection used for authentication with Container Registry NEW +# REPOSITORY: The repository to push and pull to/from in the Container Registries, should be the same for all environments to avoid complexity +# IMAGE_COUNT: Number of last images to retain, all others are deleted + +parameters: + - name: ACR_PREVIOUS + type: string + - name: ACR_NEXT + type: string + - name: ACR_SUB_PREVIOUS + type: string + - name: ACR_SUB_NEXT + type: string + - name: REPOSITORY + type: string + - name: IMAGE_COUNT + type: number + - name: SUBSCRIPTION + type: string + +steps: + +- task: Docker@2 + displayName: Pull image from container repository of previous environment + inputs: + containerRegistry: ${{ parameters.ACR_SUB_PREVIOUS }} + repository: ${{ parameters.REPOSITORY }} + command: pull + arguments: ${{ parameters.ACR_PREVIOUS }}/${{ parameters.REPOSITORY }}:$(Build.BuildId) + +- bash: docker tag ${{ parameters.ACR_PREVIOUS }}/${{ parameters.REPOSITORY }}:$(Build.BuildId) ${{ parameters.ACR_NEXT }}/${{ parameters.REPOSITORY }}:$(Build.BuildId) + displayName: Promote Docker image to registry of next environment + +- task: Docker@2 + displayName: Push Image to container repository of next environment + inputs: + containerRegistry: ${{ parameters.ACR_SUB_NEXT }} + repository: ${{ parameters.REPOSITORY }} + command: push + tags: $(Build.BuildId) + +- template: template_delete_old_images.yml + parameters: + ACR: ${{ parameters.ACR_NEXT }} + SUBSCRIPTION: ${{ parameters.SUBSCRIPTION }} + REPOSITORY: ${{ parameters.REPOSITORY }} + IMAGE_COUNT: ${{ parameters.IMAGE_COUNT}} diff --git a/cd/delete-old-images.yml b/cd/delete-old-images.yml new file mode 100644 index 0000000..59d26e3 --- /dev/null +++ b/cd/delete-old-images.yml @@ -0,0 +1,39 @@ +# This template encapsulates the functionality to delete old images, +# so only a pre-specified number of images is present in the registry +# +# Expected Parameters +# ACR: Azure Container Registry containing the images to delete +# REPOSITORY: Specific Repo in the Registry +# IMAGE_COUNT: Number of historic images to retain +# ACR_SUB: Azure Service Connection to use for authentication against registry + +parameters: +- name: ACR + type: string +- name: REPOSITORY + type: string +- name: IMAGE_COUNT + type: number +- name: SUBSCRIPTION + type: string + +steps: +- task: AzureCLI@2 + displayName: 'Clean-up old Docker image' + inputs: + azureSubscription: ${{ parameters.SUBSCRIPTION }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + ACR=${{ parameters.ACR }} + REPOSITORY=${{ parameters.REPOSITORY }} + # Number of newest images in the repository that will not be deleted + COUNT=${{ parameters.IMAGE_COUNT }} + + OLD_IMAGES=$(az acr repository show-tags --name $ACR --repository $REPOSITORY --orderby time_asc -o tsv | head -n -$COUNT) + echo "$OLD_IMAGES" + for OLD_IMAGE in $OLD_IMAGES + do + az acr repository delete --name $ACR --image $REPOSITORY:$OLD_IMAGE --yes + done + arguments: '-failOnStandardError false' diff --git a/cd/trigger.yml b/cd/trigger.yml new file mode 100644 index 0000000..f55e1f6 --- /dev/null +++ b/cd/trigger.yml @@ -0,0 +1,68 @@ +trigger: + branches: + include: + - master + paths: + include: + - src/{{ cookiecutter.module_name }} + +pool: + vmImage: 'Ubuntu-latest' + +stages: + + - stage: dev-build + displayName: Build and Push Docker Image to DEV + jobs: + - template: template-build-dev.yml + parameters: + ACR: # complete with the identifier of your azure container registry on DEV + ACR_SUB: # complete with name of docker registry service connection on DEV + REPOSITORY: # complete with name of repository + SUBSCRIPTION: # complete with name of service connection + IMAGE_COUNT: 5 + + - stage: qa-build + dependsOn: dev-build + displayName: Copy Docker Image from DEV to QA + jobs: + + - deployment: devtoqa + displayName: Copy Docker Image from DEV to QA + # environment is only set, so that an approval process can be defined + environment: # complete with Azure Dev Ops environment + strategy: + runOnce: + deploy: + steps: + - template: template-build.yml + parameters: + ACR_PREVIOUS : # complete with the identifier of your azure container registry on DEV + ACR_NEXT: # complete with the identifier of your azure container registry on QA + ACR_SUB_PREVIOUS: # complete with name of docker registry service connection on DEV + ACR_SUB_NEXT: # complete with name of docker registry service connection on QA + REPOSITORY: # complete with name of repository + SUBSCRIPTION: # complete with name of service connection + IMAGE_COUNT: 5 + + - stage: prod-build + dependsOn: qa-build + displayName: Copy Docker Image from QA to PROD + jobs: + - deployment: qatoprod + displayName: Copy Docker Image from QA to PROD + # environment is only set, so that an approval process can be defined + environment: # complete with Azure Dev Ops environment + strategy: + runOnce: + deploy: + steps: + - template: template-build.yml + parameters: + ACR_PREVIOUS : # complete with the identifier of your azure container registry on QA + ACR_NEXT: # complete with the identifier of your azure container registry on PROD + ACR_SUB_PREVIOUS: # complete with name of docker registry service connection on QA + ACR_SUB_NEXT: # complete with name of docker registry service connection on PROD + REPOSITORY: # complete with name of repository + SUBSCRIPTION: # complete with name of service connection + IMAGE_COUNT: 5 diff --git a/ci/test-pipeline.yml b/ci/test-pipeline.yml new file mode 100644 index 0000000..44979b8 --- /dev/null +++ b/ci/test-pipeline.yml @@ -0,0 +1,119 @@ +# trigger on pull requests needs to be set manually in Azure DevOps +trigger: none + +pool: + name: Azure Pipelines + vmImage: ubuntu-latest + +jobs: + +- job: Test + variables: + VENV_FOLDER: $(Pipeline.Workspace)/venv + {%- if cookiecutter.package_manager == 'pip' %} + PIP_PACKAGE_FOLDER: $(Pipeline.Workspace)/venv/lib + {%- elif cookiecutter.package_manager == 'conda' %} + CONDA_PACKAGE_FOLDER: usr/share/miniconda/envs + {%- endif %} + PACKAGE_NAME: '{{ cookiecutter.project_slug }}' + + steps: + - checkout: self + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.8' + inputs: + versionSpec: 3.8 + addToPath: true + + - task: Cache@2 + inputs: + key: 'pip | venv | $(Agent.OS) | requirements.txt | requirements-dev.txt' + path: $(PIP_PACKAGE_FOLDER) + displayName: Cache pip packages + + {%- if cookiecutter.package_manager == 'conda' %} + - task: Cache@2 + inputs: + key: 'conda | $(Agent.OS) | environment.yml | envionment-dev.yml' + path: $(PIP_PACKAGE_FOLDER) + displayName: Cache conda packages + {%- endif %} + + - task: Bash@3 + displayName: 'Create venv' + inputs: + targetType: 'inline' + script: | + set -uex + python3 -m venv $(VENV_FOLDER) + source $(VENV_FOLDER)/bin/activate + + # install system dependencies (build tools for python libs) + - task: Bash@3 + displayName: 'Install system dependencies' + inputs: + targetType: 'inline' + script: | + set -uex + sudo apt-get update + sudo apt-get install -y build-essential + + # fill in or delete if no Azuzre Keyvault is used + - task: AzureKeyVault@1 + inputs: + azureSubscription: '' + KeyVaultName: '' + SecretsFilter: '' + RunAsPreJob: false + + - task: Bash@3 + inputs: + targetType: 'inline' + script: | + set -uex + source $(VENV_FOLDER)/bin/activate + # resolve issues with old cached versions of pip + # Open question: Still needed? + python -m pip install --upgrade pip || (curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python get-pip.py) + python -m pip install wheel + + # install python app dependencies + python -m pip install -r requirements.txt -r requirements-dev.txt -U + + # build and install wheel + python setup.py dist + python -m pip install --force-reinstall dist/*.whl + displayName: 'Setup environment' + + # run tests with coverage information + - task: Bash@3 + displayName: 'pytest (with coverage)' + # define env variables as needed + env: + PACKAGE_NAME: $(PACKAGE_NAME) + inputs: + targetType: 'inline' + script: | + set -uex + source $(VENV_FOLDER)/bin/activate + + # run pytest + python -m pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=$PACKAGE_NAME --cov-report=xml:coverage-reports/cov.xml --cov-report=html + + - task: PublishCodeCoverageResults@1 + displayName: 'Publish code coverage' + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: coverage-reports/cov.xml + additionalCodeCoverageFiles: '$(System.DefaultWorkingDirectory)/junit/' + condition: succeededOrFailed() + + # publish pytest results + - task: PublishTestResults@2 + inputs: + testResultsFormat: 'JUnit' + testResultsFiles: '$(System.DefaultWorkingDirectory)/**/*-results.xml' + testRunTitle: 'Publish pytest results' + failTaskOnFailedTests: false + condition: succeededOrFailed() diff --git a/cookiecutter.json b/cookiecutter.json index affc78c..f71aa40 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -9,7 +9,7 @@ "package_manager": ["conda", "pip", "poetry"], "use_notebooks": ["yes", "no"], "use_docker": ["no", "yes"], - "ci_pipeline": ["none", "gitlab"], + "ci_pipeline": ["none", "gitlab", "az-devops"], "create_cli": ["no", "yes"], "config_file": ["yaml", "hocon", "none"], "code_formatter": ["none", "black"], diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index 4c14940..34cbc49 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -71,7 +71,14 @@ ".gitlab-ci.yml", } -files_ci_all = files_ci_gitlab +files_ci_devops = { + 'ci/test-pipeline.yml', + 'cd/build-dev.yml', + 'cd/build.yml', + 'cd/trigger.yml' +} + +files_ci_all = [files_ci_gitlab , files_ci_devops] folders_editor = [ '.idea__editor', @@ -167,6 +174,8 @@ def handle_ci(): ci_pipeline = '{{ cookiecutter.ci_pipeline }}' if ci_pipeline == "gitlab": _delete_files(files_ci_all - files_ci_gitlab) + elif ci_pipeline == "az-devops": + _delete_files(files_ci_all - files_ci_devops) elif ci_pipeline == 'none': _delete_files(files_ci_all) From dd51238010afc5e8e31a4cdfd6d1e6620a0f1f26 Mon Sep 17 00:00:00 2001 From: Alexander Fottner Date: Wed, 15 Feb 2023 10:50:26 +0100 Subject: [PATCH 08/39] complete draft of test pipeline for pip and conda --- ci/test-pipeline.yml | 136 ++++++++++++++++++++++++++++--------------- 1 file changed, 89 insertions(+), 47 deletions(-) diff --git a/ci/test-pipeline.yml b/ci/test-pipeline.yml index 44979b8..199aa61 100644 --- a/ci/test-pipeline.yml +++ b/ci/test-pipeline.yml @@ -11,80 +11,123 @@ jobs: variables: VENV_FOLDER: $(Pipeline.Workspace)/venv {%- if cookiecutter.package_manager == 'pip' %} - PIP_PACKAGE_FOLDER: $(Pipeline.Workspace)/venv/lib + PIP_CACHE_DIR: $(Pipeline.Workspace)/venv/lib {%- elif cookiecutter.package_manager == 'conda' %} - CONDA_PACKAGE_FOLDER: usr/share/miniconda/envs + CONDA_PKGS_DIRS: usr/share/miniconda/envs {%- endif %} PACKAGE_NAME: '{{ cookiecutter.project_slug }}' steps: - checkout: self - + - - task: UsePythonVersion@0 - displayName: 'Use Python 3.8' + displayName: Use Python 3.8 inputs: versionSpec: 3.8 addToPath: true - - task: Cache@2 - inputs: - key: 'pip | venv | $(Agent.OS) | requirements.txt | requirements-dev.txt' - path: $(PIP_PACKAGE_FOLDER) - displayName: Cache pip packages + - task: Bash@3 + displayName: Install system dependencies + inputs: + targetType: inline + script: | + set -uex + sudo apt update + sudo apt install -y build-essential - {%- if cookiecutter.package_manager == 'conda' %} - - task: Cache@2 + # fill in or delete if no Azuzre Keyvault is used + - task: AzureKeyVault@1 inputs: - key: 'conda | $(Agent.OS) | environment.yml | envionment-dev.yml' - path: $(PIP_PACKAGE_FOLDER) - displayName: Cache conda packages - {%- endif %} + azureSubscription: '' + KeyVaultName: '' + SecretsFilter: '' + RunAsPreJob: false + {% if cookiecutter.package_manager == 'conda' -%} - task: Bash@3 - displayName: 'Create venv' + displayName: Add conda to path inputs: - targetType: 'inline' + targetType: inline script: | - set -uex - python3 -m venv $(VENV_FOLDER) - source $(VENV_FOLDER)/bin/activate + echo "##vso[task.prependpath]$CONDA/bin" + {%- endif %} + + {%- if cookiecutter.package_manager == 'pip' %} + - task: Cache@2 + inputs: + key: 'pip | venv | $(Agent.OS) | requirements.txt | requirements-dev.txt' + path: $(PIP_CACHE_DIR) + displayName: Cache pip packages - # install system dependencies (build tools for python libs) - task: Bash@3 - displayName: 'Install system dependencies' + displayName: Create venv inputs: - targetType: 'inline' + targetType: inline script: | set -uex - sudo apt-get update - sudo apt-get install -y build-essential + python -m venv $(VENV_FOLDER) + source $(VENV_FOLDER)/bin/activate - # fill in or delete if no Azuzre Keyvault is used - - task: AzureKeyVault@1 - inputs: - azureSubscription: '' - KeyVaultName: '' - SecretsFilter: '' - RunAsPreJob: false + - task: Bash@3 + displayName: Setup environment + inputs: + targetType: inline + script: | + set -uex + source $(VENV_FOLDER)/bin/activate + # resolve issues with old cached versions of pip + # Open question: Still needed? + python -m pip install --upgrade pip || (curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python get-pip.py) + python -m pip install wheel + + # install python app dependencies + python -m pip install -r requirements.txt -r requirements-dev.txt -U + + # build and install wheel + python setup.py dist + python -m pip install --force-reinstall dist/*.whl - task: Bash@3 + displayName: 'pytest (with coverage)' + # define env variables as needed + env: + PACKAGE_NAME: $(PACKAGE_NAME) inputs: - targetType: 'inline' + targetType: inline script: | set -uex source $(VENV_FOLDER)/bin/activate - # resolve issues with old cached versions of pip - # Open question: Still needed? - python -m pip install --upgrade pip || (curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python get-pip.py) - python -m pip install wheel - # install python app dependencies - python -m pip install -r requirements.txt -r requirements-dev.txt -U + # run pytest + python -m pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=$PACKAGE_NAME --cov-report=xml:coverage-reports/cov.xml --cov-report=html + + {% elif cookiecutter.package_manager == 'conda' -%} + - task: Cache@2 + displayName: Cache conda packages + inputs: + key: 'conda | $(Agent.OS) | environment.yml | environment-dev.yml' + path: $(CONDA_PKGS_DIRS) + cacheHitVar: CONDA_CACHE_RESTORED + + - task: Bash@3 + displayName: 'Create conda environment' + inputs: + targetType: 'inline' + script: | + conda env create -n .venv -f environment-dev.yml environment.yml + source activate .venv + python -m pip install wheel + condition: eq(variables.CONDA_CACHE_RESTORED, 'false') - # build and install wheel - python setup.py dist - python -m pip install --force-reinstall dist/*.whl - displayName: 'Setup environment' + - task: Bash@3 + displayName: Install project as python package + inputs: + targetType: 'inline' + script: | + set -uex + source activate $CONDA_ENV + python setup.py dist + python -m pip install --force-reinstall dist/*.whl # run tests with coverage information - task: Bash@3 @@ -93,13 +136,12 @@ jobs: env: PACKAGE_NAME: $(PACKAGE_NAME) inputs: - targetType: 'inline' + targetType: inline script: | set -uex - source $(VENV_FOLDER)/bin/activate - - # run pytest + source activate .venv python -m pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=$PACKAGE_NAME --cov-report=xml:coverage-reports/cov.xml --cov-report=html + {%- endif %} - task: PublishCodeCoverageResults@1 displayName: 'Publish code coverage' From ebd4274a842b7c45dc91709affd087ff0d0f56c2 Mon Sep 17 00:00:00 2001 From: AlexF1994 Date: Mon, 20 Feb 2023 15:36:10 +0100 Subject: [PATCH 09/39] move append of conda to path to proper if else position --- ci/test-pipeline.yml | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/ci/test-pipeline.yml b/ci/test-pipeline.yml index 199aa61..84208c2 100644 --- a/ci/test-pipeline.yml +++ b/ci/test-pipeline.yml @@ -43,15 +43,6 @@ jobs: SecretsFilter: '' RunAsPreJob: false - {% if cookiecutter.package_manager == 'conda' -%} - - task: Bash@3 - displayName: Add conda to path - inputs: - targetType: inline - script: | - echo "##vso[task.prependpath]$CONDA/bin" - {%- endif %} - {%- if cookiecutter.package_manager == 'pip' %} - task: Cache@2 inputs: @@ -102,6 +93,13 @@ jobs: python -m pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=$PACKAGE_NAME --cov-report=xml:coverage-reports/cov.xml --cov-report=html {% elif cookiecutter.package_manager == 'conda' -%} + - task: Bash@3 + displayName: Add conda to path + inputs: + targetType: inline + script: | + echo "##vso[task.prependpath]$CONDA/bin" + - task: Cache@2 displayName: Cache conda packages inputs: @@ -125,7 +123,7 @@ jobs: targetType: 'inline' script: | set -uex - source activate $CONDA_ENV + source activate .venv python setup.py dist python -m pip install --force-reinstall dist/*.whl From cd476d783e1eb7813b088b6f9d1f9e5c67df7751 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 20 Feb 2023 16:59:13 +0100 Subject: [PATCH 10/39] Move DevOps files into project_slug dir --- {cd => {{cookiecutter.project_slug}}/cd}/build-dev.yml | 0 {cd => {{cookiecutter.project_slug}}/cd}/build.yml | 0 {cd => {{cookiecutter.project_slug}}/cd}/delete-old-images.yml | 0 {cd => {{cookiecutter.project_slug}}/cd}/trigger.yml | 0 {ci => {{cookiecutter.project_slug}}/ci}/test-pipeline.yml | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename {cd => {{cookiecutter.project_slug}}/cd}/build-dev.yml (100%) rename {cd => {{cookiecutter.project_slug}}/cd}/build.yml (100%) rename {cd => {{cookiecutter.project_slug}}/cd}/delete-old-images.yml (100%) rename {cd => {{cookiecutter.project_slug}}/cd}/trigger.yml (100%) rename {ci => {{cookiecutter.project_slug}}/ci}/test-pipeline.yml (100%) diff --git a/cd/build-dev.yml b/{{cookiecutter.project_slug}}/cd/build-dev.yml similarity index 100% rename from cd/build-dev.yml rename to {{cookiecutter.project_slug}}/cd/build-dev.yml diff --git a/cd/build.yml b/{{cookiecutter.project_slug}}/cd/build.yml similarity index 100% rename from cd/build.yml rename to {{cookiecutter.project_slug}}/cd/build.yml diff --git a/cd/delete-old-images.yml b/{{cookiecutter.project_slug}}/cd/delete-old-images.yml similarity index 100% rename from cd/delete-old-images.yml rename to {{cookiecutter.project_slug}}/cd/delete-old-images.yml diff --git a/cd/trigger.yml b/{{cookiecutter.project_slug}}/cd/trigger.yml similarity index 100% rename from cd/trigger.yml rename to {{cookiecutter.project_slug}}/cd/trigger.yml diff --git a/ci/test-pipeline.yml b/{{cookiecutter.project_slug}}/ci/test-pipeline.yml similarity index 100% rename from ci/test-pipeline.yml rename to {{cookiecutter.project_slug}}/ci/test-pipeline.yml From a5ed727e5a958d5673e36aaf1b65cc27b97dd184 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 20 Feb 2023 17:04:40 +0100 Subject: [PATCH 11/39] Fix declaration of all ci files --- hooks/post_gen_project.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index 34cbc49..6320dcf 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -78,7 +78,7 @@ 'cd/trigger.yml' } -files_ci_all = [files_ci_gitlab , files_ci_devops] +files_ci_all = files_ci_gitlab | files_ci_devops folders_editor = [ '.idea__editor', @@ -174,10 +174,14 @@ def handle_ci(): ci_pipeline = '{{ cookiecutter.ci_pipeline }}' if ci_pipeline == "gitlab": _delete_files(files_ci_all - files_ci_gitlab) + os.rmdir('ci') + os.rmdir('cd') elif ci_pipeline == "az-devops": _delete_files(files_ci_all - files_ci_devops) elif ci_pipeline == 'none': _delete_files(files_ci_all) + os.rmdir('ci') + os.rmdir('cd') def print_success(): From 6c85536d12fd5c7164783f4771b370dc43be6978 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 20 Feb 2023 17:48:52 +0100 Subject: [PATCH 12/39] Copy Build and Delete templates without rendering --- cookiecutter.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cookiecutter.json b/cookiecutter.json index f71aa40..0896978 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -13,5 +13,10 @@ "create_cli": ["no", "yes"], "config_file": ["yaml", "hocon", "none"], "code_formatter": ["none", "black"], - "editor_settings": ["none", "vscode", "pycharm"] + "editor_settings": ["none", "vscode", "pycharm"], + "_copy_without_render": [ + "cd/build-dev.yml", + "cd/build.yml", + "cd/delete-old-images.yml" + ] } From 7dba22aa82a93791a4279ee5c95fdf342fca7507 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 20 Feb 2023 17:56:44 +0100 Subject: [PATCH 13/39] Bf - Add forgotten devops file path --- hooks/post_gen_project.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index 6320dcf..93ad34c 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -75,7 +75,8 @@ 'ci/test-pipeline.yml', 'cd/build-dev.yml', 'cd/build.yml', - 'cd/trigger.yml' + 'cd/trigger.yml', + 'cd/delete-old-images.yml' } files_ci_all = files_ci_gitlab | files_ci_devops From 85f5237239481e1744bd35975a5ea4cf851b9f4d Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 20 Feb 2023 18:38:15 +0100 Subject: [PATCH 14/39] Bf - delete empty step --- {{cookiecutter.project_slug}}/ci/test-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{cookiecutter.project_slug}}/ci/test-pipeline.yml b/{{cookiecutter.project_slug}}/ci/test-pipeline.yml index 84208c2..8face33 100644 --- a/{{cookiecutter.project_slug}}/ci/test-pipeline.yml +++ b/{{cookiecutter.project_slug}}/ci/test-pipeline.yml @@ -19,7 +19,7 @@ jobs: steps: - checkout: self - - + - task: UsePythonVersion@0 displayName: Use Python 3.8 inputs: From 2ba857007f4eb20f8240c1506e9a9570e36310b4 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 20 Feb 2023 18:48:40 +0100 Subject: [PATCH 15/39] Fix indentation --- .gitignore | 1 + cookiecutter-server.yml | 15 +++++++++++++++ .../ci/test-pipeline.yml | 16 ++++++++-------- 3 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 cookiecutter-server.yml diff --git a/.gitignore b/.gitignore index 346d97c..4fd53ac 100644 --- a/.gitignore +++ b/.gitignore @@ -68,4 +68,5 @@ venv*/ # Pyenv .python-version +serve/ # project-specific stuff diff --git a/cookiecutter-server.yml b/cookiecutter-server.yml new file mode 100644 index 0000000..b1bba9e --- /dev/null +++ b/cookiecutter-server.yml @@ -0,0 +1,15 @@ +full_name: Jane Doe +company_name: '' +email: contact@alexanderthamm.com +project_name: Test AZ-DevOps Pip +project_slug: '{{ cookiecutter.project_name | slugify }}' +module_name: '{{ cookiecutter.project_slug | slugify | replace(''-'', ''_'') }}' +project_short_description: A short summary of the project +package_manager: pip +use_notebooks: 'no' +use_docker: 'no' +ci_pipeline: 'az-devops' +create_cli: 'no' +config_file: yaml +code_formatter: none +editor_settings: none diff --git a/{{cookiecutter.project_slug}}/ci/test-pipeline.yml b/{{cookiecutter.project_slug}}/ci/test-pipeline.yml index 8face33..6e2440a 100644 --- a/{{cookiecutter.project_slug}}/ci/test-pipeline.yml +++ b/{{cookiecutter.project_slug}}/ci/test-pipeline.yml @@ -19,7 +19,7 @@ jobs: steps: - checkout: self - + - task: UsePythonVersion@0 displayName: Use Python 3.8 inputs: @@ -27,13 +27,13 @@ jobs: addToPath: true - task: Bash@3 - displayName: Install system dependencies - inputs: - targetType: inline - script: | - set -uex - sudo apt update - sudo apt install -y build-essential + displayName: Install system dependencies + inputs: + targetType: inline + script: | + set -uex + sudo apt update + sudo apt install -y build-essential # fill in or delete if no Azuzre Keyvault is used - task: AzureKeyVault@1 From 256ec3ec4b71fe94d61e580be9e290beecb19b18 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 20 Feb 2023 18:52:01 +0100 Subject: [PATCH 16/39] Revert "Fix indentation" This reverts commit 2ba857007f4eb20f8240c1506e9a9570e36310b4. --- .gitignore | 1 - cookiecutter-server.yml | 15 --------------- .../ci/test-pipeline.yml | 16 ++++++++-------- 3 files changed, 8 insertions(+), 24 deletions(-) delete mode 100644 cookiecutter-server.yml diff --git a/.gitignore b/.gitignore index 4fd53ac..346d97c 100644 --- a/.gitignore +++ b/.gitignore @@ -68,5 +68,4 @@ venv*/ # Pyenv .python-version -serve/ # project-specific stuff diff --git a/cookiecutter-server.yml b/cookiecutter-server.yml deleted file mode 100644 index b1bba9e..0000000 --- a/cookiecutter-server.yml +++ /dev/null @@ -1,15 +0,0 @@ -full_name: Jane Doe -company_name: '' -email: contact@alexanderthamm.com -project_name: Test AZ-DevOps Pip -project_slug: '{{ cookiecutter.project_name | slugify }}' -module_name: '{{ cookiecutter.project_slug | slugify | replace(''-'', ''_'') }}' -project_short_description: A short summary of the project -package_manager: pip -use_notebooks: 'no' -use_docker: 'no' -ci_pipeline: 'az-devops' -create_cli: 'no' -config_file: yaml -code_formatter: none -editor_settings: none diff --git a/{{cookiecutter.project_slug}}/ci/test-pipeline.yml b/{{cookiecutter.project_slug}}/ci/test-pipeline.yml index 6e2440a..8face33 100644 --- a/{{cookiecutter.project_slug}}/ci/test-pipeline.yml +++ b/{{cookiecutter.project_slug}}/ci/test-pipeline.yml @@ -19,7 +19,7 @@ jobs: steps: - checkout: self - + - task: UsePythonVersion@0 displayName: Use Python 3.8 inputs: @@ -27,13 +27,13 @@ jobs: addToPath: true - task: Bash@3 - displayName: Install system dependencies - inputs: - targetType: inline - script: | - set -uex - sudo apt update - sudo apt install -y build-essential + displayName: Install system dependencies + inputs: + targetType: inline + script: | + set -uex + sudo apt update + sudo apt install -y build-essential # fill in or delete if no Azuzre Keyvault is used - task: AzureKeyVault@1 From 605bd584f078fce16240d2376bf38a1e3169cecb Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 20 Feb 2023 19:01:34 +0100 Subject: [PATCH 17/39] Fix indentation --- {{cookiecutter.project_slug}}/cd/build.yml | 45 ++++---- .../cd/delete-old-images.yml | 52 ++++----- {{cookiecutter.project_slug}}/cd/trigger.yml | 102 +++++++++--------- .../ci/test-pipeline.yml | 90 ++++++++-------- 4 files changed, 143 insertions(+), 146 deletions(-) diff --git a/{{cookiecutter.project_slug}}/cd/build.yml b/{{cookiecutter.project_slug}}/cd/build.yml index 4ece11c..087b5cf 100644 --- a/{{cookiecutter.project_slug}}/cd/build.yml +++ b/{{cookiecutter.project_slug}}/cd/build.yml @@ -10,30 +10,29 @@ # IMAGE_COUNT: Number of last images to retain, all others are deleted parameters: - - name: ACR_PREVIOUS - type: string - - name: ACR_NEXT - type: string - - name: ACR_SUB_PREVIOUS - type: string - - name: ACR_SUB_NEXT - type: string - - name: REPOSITORY - type: string - - name: IMAGE_COUNT - type: number - - name: SUBSCRIPTION - type: string +- name: ACR_PREVIOUS + type: string +- name: ACR_NEXT + type: string +- name: ACR_SUB_PREVIOUS + type: string +- name: ACR_SUB_NEXT + type: string +- name: REPOSITORY + type: string +- name: IMAGE_COUNT + type: number +- name: SUBSCRIPTION + type: string steps: - - task: Docker@2 displayName: Pull image from container repository of previous environment inputs: - containerRegistry: ${{ parameters.ACR_SUB_PREVIOUS }} - repository: ${{ parameters.REPOSITORY }} - command: pull - arguments: ${{ parameters.ACR_PREVIOUS }}/${{ parameters.REPOSITORY }}:$(Build.BuildId) + containerRegistry: ${{ parameters.ACR_SUB_PREVIOUS }} + repository: ${{ parameters.REPOSITORY }} + command: pull + arguments: ${{ parameters.ACR_PREVIOUS }}/${{ parameters.REPOSITORY }}:$(Build.BuildId) - bash: docker tag ${{ parameters.ACR_PREVIOUS }}/${{ parameters.REPOSITORY }}:$(Build.BuildId) ${{ parameters.ACR_NEXT }}/${{ parameters.REPOSITORY }}:$(Build.BuildId) displayName: Promote Docker image to registry of next environment @@ -41,10 +40,10 @@ steps: - task: Docker@2 displayName: Push Image to container repository of next environment inputs: - containerRegistry: ${{ parameters.ACR_SUB_NEXT }} - repository: ${{ parameters.REPOSITORY }} - command: push - tags: $(Build.BuildId) + containerRegistry: ${{ parameters.ACR_SUB_NEXT }} + repository: ${{ parameters.REPOSITORY }} + command: push + tags: $(Build.BuildId) - template: template_delete_old_images.yml parameters: diff --git a/{{cookiecutter.project_slug}}/cd/delete-old-images.yml b/{{cookiecutter.project_slug}}/cd/delete-old-images.yml index 59d26e3..0dd7969 100644 --- a/{{cookiecutter.project_slug}}/cd/delete-old-images.yml +++ b/{{cookiecutter.project_slug}}/cd/delete-old-images.yml @@ -8,32 +8,32 @@ # ACR_SUB: Azure Service Connection to use for authentication against registry parameters: -- name: ACR - type: string -- name: REPOSITORY - type: string -- name: IMAGE_COUNT - type: number -- name: SUBSCRIPTION - type: string + - name: ACR + type: string + - name: REPOSITORY + type: string + - name: IMAGE_COUNT + type: number + - name: SUBSCRIPTION + type: string steps: -- task: AzureCLI@2 - displayName: 'Clean-up old Docker image' - inputs: - azureSubscription: ${{ parameters.SUBSCRIPTION }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - ACR=${{ parameters.ACR }} - REPOSITORY=${{ parameters.REPOSITORY }} - # Number of newest images in the repository that will not be deleted - COUNT=${{ parameters.IMAGE_COUNT }} + - task: AzureCLI@2 + displayName: 'Clean-up old Docker image' + inputs: + azureSubscription: ${{ parameters.SUBSCRIPTION }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + ACR=${{ parameters.ACR }} + REPOSITORY=${{ parameters.REPOSITORY }} + # Number of newest images in the repository that will not be deleted + COUNT=${{ parameters.IMAGE_COUNT }} - OLD_IMAGES=$(az acr repository show-tags --name $ACR --repository $REPOSITORY --orderby time_asc -o tsv | head -n -$COUNT) - echo "$OLD_IMAGES" - for OLD_IMAGE in $OLD_IMAGES - do - az acr repository delete --name $ACR --image $REPOSITORY:$OLD_IMAGE --yes - done - arguments: '-failOnStandardError false' + OLD_IMAGES=$(az acr repository show-tags --name $ACR --repository $REPOSITORY --orderby time_asc -o tsv | head -n -$COUNT) + echo "$OLD_IMAGES" + for OLD_IMAGE in $OLD_IMAGES + do + az acr repository delete --name $ACR --image $REPOSITORY:$OLD_IMAGE --yes + done + arguments: '-failOnStandardError false' diff --git a/{{cookiecutter.project_slug}}/cd/trigger.yml b/{{cookiecutter.project_slug}}/cd/trigger.yml index f55e1f6..6bc7c37 100644 --- a/{{cookiecutter.project_slug}}/cd/trigger.yml +++ b/{{cookiecutter.project_slug}}/cd/trigger.yml @@ -10,59 +10,57 @@ pool: vmImage: 'Ubuntu-latest' stages: +- stage: dev-build + displayName: Build and Push Docker Image to DEV + jobs: + - template: template-build-dev.yml + parameters: + ACR: # complete with the identifier of your azure container registry on DEV + ACR_SUB: # complete with name of docker registry service connection on DEV + REPOSITORY: # complete with name of repository + SUBSCRIPTION: # complete with name of service connection + IMAGE_COUNT: 5 - - stage: dev-build - displayName: Build and Push Docker Image to DEV - jobs: - - template: template-build-dev.yml - parameters: - ACR: # complete with the identifier of your azure container registry on DEV - ACR_SUB: # complete with name of docker registry service connection on DEV - REPOSITORY: # complete with name of repository - SUBSCRIPTION: # complete with name of service connection - IMAGE_COUNT: 5 - - - stage: qa-build - dependsOn: dev-build +- stage: qa-build + dependsOn: dev-build + displayName: Copy Docker Image from DEV to QA + jobs: + - deployment: devtoqa displayName: Copy Docker Image from DEV to QA - jobs: - - - deployment: devtoqa - displayName: Copy Docker Image from DEV to QA - # environment is only set, so that an approval process can be defined - environment: # complete with Azure Dev Ops environment - strategy: - runOnce: - deploy: - steps: - - template: template-build.yml - parameters: - ACR_PREVIOUS : # complete with the identifier of your azure container registry on DEV - ACR_NEXT: # complete with the identifier of your azure container registry on QA - ACR_SUB_PREVIOUS: # complete with name of docker registry service connection on DEV - ACR_SUB_NEXT: # complete with name of docker registry service connection on QA - REPOSITORY: # complete with name of repository - SUBSCRIPTION: # complete with name of service connection - IMAGE_COUNT: 5 + # environment is only set, so that an approval process can be defined + environment: # complete with Azure Dev Ops environment + strategy: + runOnce: + deploy: + steps: + - template: template-build.yml + parameters: + ACR_PREVIOUS : # complete with the identifier of your azure container registry on DEV + ACR_NEXT: # complete with the identifier of your azure container registry on QA + ACR_SUB_PREVIOUS: # complete with name of docker registry service connection on DEV + ACR_SUB_NEXT: # complete with name of docker registry service connection on QA + REPOSITORY: # complete with name of repository + SUBSCRIPTION: # complete with name of service connection + IMAGE_COUNT: 5 - - stage: prod-build - dependsOn: qa-build +- stage: prod-build + dependsOn: qa-build + displayName: Copy Docker Image from QA to PROD + jobs: + - deployment: qatoprod displayName: Copy Docker Image from QA to PROD - jobs: - - deployment: qatoprod - displayName: Copy Docker Image from QA to PROD - # environment is only set, so that an approval process can be defined - environment: # complete with Azure Dev Ops environment - strategy: - runOnce: - deploy: - steps: - - template: template-build.yml - parameters: - ACR_PREVIOUS : # complete with the identifier of your azure container registry on QA - ACR_NEXT: # complete with the identifier of your azure container registry on PROD - ACR_SUB_PREVIOUS: # complete with name of docker registry service connection on QA - ACR_SUB_NEXT: # complete with name of docker registry service connection on PROD - REPOSITORY: # complete with name of repository - SUBSCRIPTION: # complete with name of service connection - IMAGE_COUNT: 5 + # environment is only set, so that an approval process can be defined + environment: # complete with Azure Dev Ops environment + strategy: + runOnce: + deploy: + steps: + - template: template-build.yml + parameters: + ACR_PREVIOUS : # complete with the identifier of your azure container registry on QA + ACR_NEXT: # complete with the identifier of your azure container registry on PROD + ACR_SUB_PREVIOUS: # complete with name of docker registry service connection on QA + ACR_SUB_NEXT: # complete with name of docker registry service connection on PROD + REPOSITORY: # complete with name of repository + SUBSCRIPTION: # complete with name of service connection + IMAGE_COUNT: 5 diff --git a/{{cookiecutter.project_slug}}/ci/test-pipeline.yml b/{{cookiecutter.project_slug}}/ci/test-pipeline.yml index 8face33..b5fa26a 100644 --- a/{{cookiecutter.project_slug}}/ci/test-pipeline.yml +++ b/{{cookiecutter.project_slug}}/ci/test-pipeline.yml @@ -27,13 +27,13 @@ jobs: addToPath: true - task: Bash@3 - displayName: Install system dependencies - inputs: - targetType: inline - script: | - set -uex - sudo apt update - sudo apt install -y build-essential + displayName: Install system dependencies + inputs: + targetType: inline + script: | + set -uex + sudo apt update + sudo apt install -y build-essential # fill in or delete if no Azuzre Keyvault is used - task: AzureKeyVault@1 @@ -60,23 +60,23 @@ jobs: source $(VENV_FOLDER)/bin/activate - task: Bash@3 - displayName: Setup environment - inputs: - targetType: inline - script: | - set -uex - source $(VENV_FOLDER)/bin/activate - # resolve issues with old cached versions of pip - # Open question: Still needed? - python -m pip install --upgrade pip || (curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python get-pip.py) - python -m pip install wheel - - # install python app dependencies - python -m pip install -r requirements.txt -r requirements-dev.txt -U - - # build and install wheel - python setup.py dist - python -m pip install --force-reinstall dist/*.whl + displayName: Setup environment + inputs: + targetType: inline + script: | + set -uex + source $(VENV_FOLDER)/bin/activate + # resolve issues with old cached versions of pip + # Open question: Still needed? + python -m pip install --upgrade pip || (curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python get-pip.py) + python -m pip install wheel + + # install python app dependencies + python -m pip install -r requirements.txt -r requirements-dev.txt -U + + # build and install wheel + python setup.py dist + python -m pip install --force-reinstall dist/*.whl - task: Bash@3 displayName: 'pytest (with coverage)' @@ -94,11 +94,11 @@ jobs: {% elif cookiecutter.package_manager == 'conda' -%} - task: Bash@3 - displayName: Add conda to path - inputs: - targetType: inline - script: | - echo "##vso[task.prependpath]$CONDA/bin" + displayName: Add conda to path + inputs: + targetType: inline + script: | + echo "##vso[task.prependpath]$CONDA/bin" - task: Cache@2 displayName: Cache conda packages @@ -108,24 +108,24 @@ jobs: cacheHitVar: CONDA_CACHE_RESTORED - task: Bash@3 - displayName: 'Create conda environment' - inputs: - targetType: 'inline' - script: | - conda env create -n .venv -f environment-dev.yml environment.yml - source activate .venv - python -m pip install wheel - condition: eq(variables.CONDA_CACHE_RESTORED, 'false') + displayName: 'Create conda environment' + inputs: + targetType: 'inline' + script: | + conda env create -n .venv -f environment-dev.yml environment.yml + source activate .venv + python -m pip install wheel + condition: eq(variables.CONDA_CACHE_RESTORED, 'false') - task: Bash@3 - displayName: Install project as python package - inputs: - targetType: 'inline' - script: | - set -uex - source activate .venv - python setup.py dist - python -m pip install --force-reinstall dist/*.whl + displayName: Install project as python package + inputs: + targetType: 'inline' + script: | + set -uex + source activate .venv + python setup.py dist + python -m pip install --force-reinstall dist/*.whl # run tests with coverage information - task: Bash@3 From 03f4a3c82628279583e025ec98ec29cd1dabd5e6 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 20 Feb 2023 19:38:21 +0100 Subject: [PATCH 18/39] Bf - delete single quotes --- {{cookiecutter.project_slug}}/ci/test-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/{{cookiecutter.project_slug}}/ci/test-pipeline.yml b/{{cookiecutter.project_slug}}/ci/test-pipeline.yml index b5fa26a..9ab62bb 100644 --- a/{{cookiecutter.project_slug}}/ci/test-pipeline.yml +++ b/{{cookiecutter.project_slug}}/ci/test-pipeline.yml @@ -110,7 +110,7 @@ jobs: - task: Bash@3 displayName: 'Create conda environment' inputs: - targetType: 'inline' + targetType: inline script: | conda env create -n .venv -f environment-dev.yml environment.yml source activate .venv @@ -120,7 +120,7 @@ jobs: - task: Bash@3 displayName: Install project as python package inputs: - targetType: 'inline' + targetType: inline script: | set -uex source activate .venv From 0a225f407f830550d91c873e80f16fdc61f9ba41 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 21 Feb 2023 13:21:56 +0100 Subject: [PATCH 19/39] Fix format --- {{cookiecutter.project_slug}}/cd/trigger.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/{{cookiecutter.project_slug}}/cd/trigger.yml b/{{cookiecutter.project_slug}}/cd/trigger.yml index 6bc7c37..2f8ce86 100644 --- a/{{cookiecutter.project_slug}}/cd/trigger.yml +++ b/{{cookiecutter.project_slug}}/cd/trigger.yml @@ -35,7 +35,7 @@ stages: steps: - template: template-build.yml parameters: - ACR_PREVIOUS : # complete with the identifier of your azure container registry on DEV + ACR_PREVIOUS: # complete with the identifier of your azure container registry on DEV ACR_NEXT: # complete with the identifier of your azure container registry on QA ACR_SUB_PREVIOUS: # complete with name of docker registry service connection on DEV ACR_SUB_NEXT: # complete with name of docker registry service connection on QA @@ -57,7 +57,7 @@ stages: steps: - template: template-build.yml parameters: - ACR_PREVIOUS : # complete with the identifier of your azure container registry on QA + ACR_PREVIOUS: # complete with the identifier of your azure container registry on QA ACR_NEXT: # complete with the identifier of your azure container registry on PROD ACR_SUB_PREVIOUS: # complete with name of docker registry service connection on QA ACR_SUB_NEXT: # complete with name of docker registry service connection on PROD From eca9c11efb6518e65620231644499846d59d1ca8 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 21 Feb 2023 14:25:00 +0100 Subject: [PATCH 20/39] Fix template file names --- {{cookiecutter.project_slug}}/cd/trigger.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/{{cookiecutter.project_slug}}/cd/trigger.yml b/{{cookiecutter.project_slug}}/cd/trigger.yml index 2f8ce86..a319075 100644 --- a/{{cookiecutter.project_slug}}/cd/trigger.yml +++ b/{{cookiecutter.project_slug}}/cd/trigger.yml @@ -13,7 +13,7 @@ stages: - stage: dev-build displayName: Build and Push Docker Image to DEV jobs: - - template: template-build-dev.yml + - template: build-dev.yml parameters: ACR: # complete with the identifier of your azure container registry on DEV ACR_SUB: # complete with name of docker registry service connection on DEV @@ -33,7 +33,7 @@ stages: runOnce: deploy: steps: - - template: template-build.yml + - template: build.yml parameters: ACR_PREVIOUS: # complete with the identifier of your azure container registry on DEV ACR_NEXT: # complete with the identifier of your azure container registry on QA @@ -55,7 +55,7 @@ stages: runOnce: deploy: steps: - - template: template-build.yml + - template: build.yml parameters: ACR_PREVIOUS: # complete with the identifier of your azure container registry on QA ACR_NEXT: # complete with the identifier of your azure container registry on PROD From 5fba90478590bd387d28742c71ebd1f80e98db42 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 22 Feb 2023 07:40:29 +0100 Subject: [PATCH 21/39] Fix template file names --- {{cookiecutter.project_slug}}/cd/build-dev.yml | 2 +- {{cookiecutter.project_slug}}/cd/build.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/{{cookiecutter.project_slug}}/cd/build-dev.yml b/{{cookiecutter.project_slug}}/cd/build-dev.yml index 17c03d8..7583764 100644 --- a/{{cookiecutter.project_slug}}/cd/build-dev.yml +++ b/{{cookiecutter.project_slug}}/cd/build-dev.yml @@ -39,7 +39,7 @@ jobs: Dockerfile: '**/Dockerfile' tags: $(Build.BuildId) - - template: template_delete_old_images.yml + - template: delete_old_images.yml parameters: ACR: ${{ parameters.ACR }} SUBSCRIPTION: ${{ parameters.SUBSCRIPTION }} diff --git a/{{cookiecutter.project_slug}}/cd/build.yml b/{{cookiecutter.project_slug}}/cd/build.yml index 087b5cf..e41b486 100644 --- a/{{cookiecutter.project_slug}}/cd/build.yml +++ b/{{cookiecutter.project_slug}}/cd/build.yml @@ -45,7 +45,7 @@ steps: command: push tags: $(Build.BuildId) -- template: template_delete_old_images.yml +- template: delete_old_images.yml parameters: ACR: ${{ parameters.ACR_NEXT }} SUBSCRIPTION: ${{ parameters.SUBSCRIPTION }} From 9aa9a8cf8aea116216f703128f27417397aa5ba7 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 22 Feb 2023 07:43:08 +0100 Subject: [PATCH 22/39] Fix template file names --- {{cookiecutter.project_slug}}/cd/build-dev.yml | 2 +- {{cookiecutter.project_slug}}/cd/build.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/{{cookiecutter.project_slug}}/cd/build-dev.yml b/{{cookiecutter.project_slug}}/cd/build-dev.yml index 7583764..00b83ae 100644 --- a/{{cookiecutter.project_slug}}/cd/build-dev.yml +++ b/{{cookiecutter.project_slug}}/cd/build-dev.yml @@ -39,7 +39,7 @@ jobs: Dockerfile: '**/Dockerfile' tags: $(Build.BuildId) - - template: delete_old_images.yml + - template: delete-old-images.yml parameters: ACR: ${{ parameters.ACR }} SUBSCRIPTION: ${{ parameters.SUBSCRIPTION }} diff --git a/{{cookiecutter.project_slug}}/cd/build.yml b/{{cookiecutter.project_slug}}/cd/build.yml index e41b486..4602395 100644 --- a/{{cookiecutter.project_slug}}/cd/build.yml +++ b/{{cookiecutter.project_slug}}/cd/build.yml @@ -45,7 +45,7 @@ steps: command: push tags: $(Build.BuildId) -- template: delete_old_images.yml +- template: delete-old-images.yml parameters: ACR: ${{ parameters.ACR_NEXT }} SUBSCRIPTION: ${{ parameters.SUBSCRIPTION }} From ced26203b4f8e8afb7e06dbeb6bce3e2702768bd Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 22 Feb 2023 07:46:45 +0100 Subject: [PATCH 23/39] Fix template file names --- {{cookiecutter.project_slug}}/cd/trigger.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/{{cookiecutter.project_slug}}/cd/trigger.yml b/{{cookiecutter.project_slug}}/cd/trigger.yml index a319075..80f5765 100644 --- a/{{cookiecutter.project_slug}}/cd/trigger.yml +++ b/{{cookiecutter.project_slug}}/cd/trigger.yml @@ -10,7 +10,7 @@ pool: vmImage: 'Ubuntu-latest' stages: -- stage: dev-build +- stage: dev_build displayName: Build and Push Docker Image to DEV jobs: - template: build-dev.yml @@ -21,8 +21,8 @@ stages: SUBSCRIPTION: # complete with name of service connection IMAGE_COUNT: 5 -- stage: qa-build - dependsOn: dev-build +- stage: qa_build + dependsOn: dev_build displayName: Copy Docker Image from DEV to QA jobs: - deployment: devtoqa @@ -43,8 +43,8 @@ stages: SUBSCRIPTION: # complete with name of service connection IMAGE_COUNT: 5 -- stage: prod-build - dependsOn: qa-build +- stage: prod_build + dependsOn: qa_build displayName: Copy Docker Image from QA to PROD jobs: - deployment: qatoprod From 830ce74568186fea76f813cbed44897b2e1e38cf Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 22 Feb 2023 07:46:56 +0100 Subject: [PATCH 24/39] Revert "Fix template file names" This reverts commit ced26203b4f8e8afb7e06dbeb6bce3e2702768bd. --- {{cookiecutter.project_slug}}/cd/trigger.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/{{cookiecutter.project_slug}}/cd/trigger.yml b/{{cookiecutter.project_slug}}/cd/trigger.yml index 80f5765..a319075 100644 --- a/{{cookiecutter.project_slug}}/cd/trigger.yml +++ b/{{cookiecutter.project_slug}}/cd/trigger.yml @@ -10,7 +10,7 @@ pool: vmImage: 'Ubuntu-latest' stages: -- stage: dev_build +- stage: dev-build displayName: Build and Push Docker Image to DEV jobs: - template: build-dev.yml @@ -21,8 +21,8 @@ stages: SUBSCRIPTION: # complete with name of service connection IMAGE_COUNT: 5 -- stage: qa_build - dependsOn: dev_build +- stage: qa-build + dependsOn: dev-build displayName: Copy Docker Image from DEV to QA jobs: - deployment: devtoqa @@ -43,8 +43,8 @@ stages: SUBSCRIPTION: # complete with name of service connection IMAGE_COUNT: 5 -- stage: prod_build - dependsOn: qa_build +- stage: prod-build + dependsOn: qa-build displayName: Copy Docker Image from QA to PROD jobs: - deployment: qatoprod From f4cd38ef4ae1be23544dc3c5f0b54a0fefa07d30 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 22 Feb 2023 07:49:19 +0100 Subject: [PATCH 25/39] Fix stage names --- {{cookiecutter.project_slug}}/cd/trigger.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/{{cookiecutter.project_slug}}/cd/trigger.yml b/{{cookiecutter.project_slug}}/cd/trigger.yml index a319075..80f5765 100644 --- a/{{cookiecutter.project_slug}}/cd/trigger.yml +++ b/{{cookiecutter.project_slug}}/cd/trigger.yml @@ -10,7 +10,7 @@ pool: vmImage: 'Ubuntu-latest' stages: -- stage: dev-build +- stage: dev_build displayName: Build and Push Docker Image to DEV jobs: - template: build-dev.yml @@ -21,8 +21,8 @@ stages: SUBSCRIPTION: # complete with name of service connection IMAGE_COUNT: 5 -- stage: qa-build - dependsOn: dev-build +- stage: qa_build + dependsOn: dev_build displayName: Copy Docker Image from DEV to QA jobs: - deployment: devtoqa @@ -43,8 +43,8 @@ stages: SUBSCRIPTION: # complete with name of service connection IMAGE_COUNT: 5 -- stage: prod-build - dependsOn: qa-build +- stage: prod_build + dependsOn: qa_build displayName: Copy Docker Image from QA to PROD jobs: - deployment: qatoprod From ae02d161804dd3cc9a5d8791fa3cbee6c1b9d6e6 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 22 Feb 2023 08:33:47 +0100 Subject: [PATCH 26/39] Fix other names --- {{cookiecutter.project_slug}}/cd/build-dev.yml | 2 +- {{cookiecutter.project_slug}}/cd/trigger.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/{{cookiecutter.project_slug}}/cd/build-dev.yml b/{{cookiecutter.project_slug}}/cd/build-dev.yml index 00b83ae..fbc062f 100644 --- a/{{cookiecutter.project_slug}}/cd/build-dev.yml +++ b/{{cookiecutter.project_slug}}/cd/build-dev.yml @@ -20,7 +20,7 @@ parameters: type: number jobs: -- job: build-push-docker-image +- job: build_push_docker_image steps: - checkout: self diff --git a/{{cookiecutter.project_slug}}/cd/trigger.yml b/{{cookiecutter.project_slug}}/cd/trigger.yml index 80f5765..a0ee95f 100644 --- a/{{cookiecutter.project_slug}}/cd/trigger.yml +++ b/{{cookiecutter.project_slug}}/cd/trigger.yml @@ -25,7 +25,7 @@ stages: dependsOn: dev_build displayName: Copy Docker Image from DEV to QA jobs: - - deployment: devtoqa + - deployment: dev_to_qa displayName: Copy Docker Image from DEV to QA # environment is only set, so that an approval process can be defined environment: # complete with Azure Dev Ops environment @@ -47,7 +47,7 @@ stages: dependsOn: qa_build displayName: Copy Docker Image from QA to PROD jobs: - - deployment: qatoprod + - deployment: qa_to_prod displayName: Copy Docker Image from QA to PROD # environment is only set, so that an approval process can be defined environment: # complete with Azure Dev Ops environment From 3d743bc7b49ca6707057cafb21d633c856300855 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 22 Feb 2023 09:28:07 +0100 Subject: [PATCH 27/39] Fix conda test-pipeline --- .../ci/test-pipeline.yml | 40 +++++++++---------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/{{cookiecutter.project_slug}}/ci/test-pipeline.yml b/{{cookiecutter.project_slug}}/ci/test-pipeline.yml index 9ab62bb..da809ae 100644 --- a/{{cookiecutter.project_slug}}/ci/test-pipeline.yml +++ b/{{cookiecutter.project_slug}}/ci/test-pipeline.yml @@ -13,7 +13,8 @@ jobs: {%- if cookiecutter.package_manager == 'pip' %} PIP_CACHE_DIR: $(Pipeline.Workspace)/venv/lib {%- elif cookiecutter.package_manager == 'conda' %} - CONDA_PKGS_DIRS: usr/share/miniconda/envs + CONDA_ENV_NAME: test-env + CONDA_PKGS_DIRS: /usr/share/miniconda/envs/$(CONDA_ENV_NAME) {%- endif %} PACKAGE_NAME: '{{ cookiecutter.project_slug }}' @@ -79,7 +80,7 @@ jobs: python -m pip install --force-reinstall dist/*.whl - task: Bash@3 - displayName: 'pytest (with coverage)' + displayName: pytest (with coverage) # define env variables as needed env: PACKAGE_NAME: $(PACKAGE_NAME) @@ -93,13 +94,6 @@ jobs: python -m pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=$PACKAGE_NAME --cov-report=xml:coverage-reports/cov.xml --cov-report=html {% elif cookiecutter.package_manager == 'conda' -%} - - task: Bash@3 - displayName: Add conda to path - inputs: - targetType: inline - script: | - echo "##vso[task.prependpath]$CONDA/bin" - - task: Cache@2 displayName: Cache conda packages inputs: @@ -108,12 +102,14 @@ jobs: cacheHitVar: CONDA_CACHE_RESTORED - task: Bash@3 - displayName: 'Create conda environment' + displayName: Create conda environment inputs: targetType: inline script: | - conda env create -n .venv -f environment-dev.yml environment.yml - source activate .venv + set -uex + conda env create -n $(CONDA_ENV_NAME) -f environment-dev.yml environment.yml + source /usr/share/miniconda/etc/profile.d/conda.sh + conda activate $(CONDA_ENV_NAME) python -m pip install wheel condition: eq(variables.CONDA_CACHE_RESTORED, 'false') @@ -123,13 +119,14 @@ jobs: targetType: inline script: | set -uex - source activate .venv + source /usr/share/miniconda/etc/profile.d/conda.sh + conda activate $(CONDA_ENV_NAME) python setup.py dist python -m pip install --force-reinstall dist/*.whl # run tests with coverage information - task: Bash@3 - displayName: 'pytest (with coverage)' + displayName: pytest (with coverage) # define env variables as needed env: PACKAGE_NAME: $(PACKAGE_NAME) @@ -137,23 +134,24 @@ jobs: targetType: inline script: | set -uex - source activate .venv + source /usr/share/miniconda/etc/profile.d/conda.sh + conda activate $(CONDA_ENV_NAME) python -m pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=$PACKAGE_NAME --cov-report=xml:coverage-reports/cov.xml --cov-report=html {%- endif %} - task: PublishCodeCoverageResults@1 - displayName: 'Publish code coverage' + displayName: Publish code coverage inputs: codeCoverageTool: Cobertura - summaryFileLocation: coverage-reports/cov.xml - additionalCodeCoverageFiles: '$(System.DefaultWorkingDirectory)/junit/' + summaryFileLocation: $(System.DefaultWorkingDirectory)/coverage-reports/cov.xml + additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/junit/ condition: succeededOrFailed() # publish pytest results - task: PublishTestResults@2 inputs: - testResultsFormat: 'JUnit' - testResultsFiles: '$(System.DefaultWorkingDirectory)/**/*-results.xml' - testRunTitle: 'Publish pytest results' + testResultsFormat: JUnit + testResultsFiles: $(System.DefaultWorkingDirectory)/**/*-results.xml + testRunTitle: Publish pytest results failTaskOnFailedTests: false condition: succeededOrFailed() From 910b6820ef3a68edb6d9e10507d2ee90d77bf098 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 27 Feb 2023 18:10:32 +0100 Subject: [PATCH 28/39] cd option conditional on docker option --- hooks/post_gen_project.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index 93ad34c..d2f4f2d 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -73,13 +73,16 @@ files_ci_devops = { 'ci/test-pipeline.yml', +} + +files_cd_devops = { 'cd/build-dev.yml', 'cd/build.yml', 'cd/trigger.yml', 'cd/delete-old-images.yml' } -files_ci_all = files_ci_gitlab | files_ci_devops +files_ci_all = files_ci_gitlab | files_ci_devops | files_cd_devops folders_editor = [ '.idea__editor', @@ -173,12 +176,16 @@ def handle_editor_settings(): def handle_ci(): ci_pipeline = '{{ cookiecutter.ci_pipeline }}' + use_docker = '{{ cookiecutter.use_docker }}' if ci_pipeline == "gitlab": _delete_files(files_ci_all - files_ci_gitlab) os.rmdir('ci') os.rmdir('cd') elif ci_pipeline == "az-devops": - _delete_files(files_ci_all - files_ci_devops) + _delete_files(files_ci_all - files_ci_devops - files_cd_devops) + if use_docker == 'no': + _delete_files(files_cd_devops) + os.rmdir('cd') elif ci_pipeline == 'none': _delete_files(files_ci_all) os.rmdir('ci') From d3551dfbd800425c7f9e0522040d50107cdbcc91 Mon Sep 17 00:00:00 2001 From: AlexF1994 Date: Tue, 14 Mar 2023 09:47:38 +0100 Subject: [PATCH 29/39] add tests for az-devops option --- tests/test_options.py | 69 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/tests/test_options.py b/tests/test_options.py index 83ed6cf..4c0913f 100644 --- a/tests/test_options.py +++ b/tests/test_options.py @@ -1,6 +1,27 @@ from pathlib import Path from .util import assert_file_contains, check_project +import pytest + + +@pytest.fixture +def az_devops_cd_files(): + return [ + 'cd/build.yml', + 'cd/build-dev.yml', + 'cd/delete-old-images.yml', + 'cd/trigger.yml' + ] + + +@pytest.fixture +def az_devops_ci_files(): + return ['ci/test-pipeline.yml'] + + +@pytest.fixture +def az_devops_files(az_devops_ci_files, az_devops_cd_files): + return az_devops_ci_files + az_devops_cd_files def test_base(): @@ -61,10 +82,26 @@ def test_docker_poetry(): def test_docker_no(): check_project( - settings={'use_docker': 'no'}, + settings={ + 'use_docker': 'no' + }, files_non_existent=['Dockerfile', 'docker-compose.yml', '.dockerignore']) +def test_docker_no_az_devops(az_devops_cd_files): + check_project( + settings={ + 'use_docker': 'no', + 'ci_pipeline': 'az-devops' + }, + files_non_existent=[ + 'Dockerfile', + 'docker-compose.yml', + '.dockerignore', + ].extend(az_devops_cd_files) + ) + + def test_cli_yes(): check_project( settings={'create_cli': 'yes'}, @@ -197,6 +234,7 @@ def test_poetry_regression(): run_pytest=True, ) + def test_gitlab_pip(): check_project( settings={ @@ -206,6 +244,7 @@ def test_gitlab_pip(): files_existent=[".gitlab-ci.yml"] ) + def test_gitlab_conda(): check_project( settings={ @@ -215,6 +254,7 @@ def test_gitlab_conda(): files_existent=[".gitlab-ci.yml"] ) + def test_gitlab_poetry(): check_project( settings={ @@ -224,10 +264,33 @@ def test_gitlab_poetry(): files_existent=[".gitlab-ci.yml"] ) -def test_no_ci_pipeline(): + +def test_az_devops_pip(az_devops_files): + check_project( + settings={ + "package_manager": "pip", + "ci_pipeline": "az-devops", + "use_docker": "yes" + }, + files_existent=az_devops_files + ) + + +def test_az_devops_conda(az_devops_files): + check_project( + settings={ + "package_manager": "conda", + "ci_pipeline": "az-devops", + "use_docker": "yes" + }, + files_existent=az_devops_files + ) + + +def test_no_ci_pipeline(az_devops_files): check_project( settings={ "ci_pipeline": "none" }, - files_non_existent=[".gitlab-ci.yml"] + files_non_existent=[".gitlab-ci.yml"] + az_devops_files ) From 1188501f49053824d55a346fc2c0b648d256b360 Mon Sep 17 00:00:00 2001 From: Sebastian Straub Date: Thu, 10 Aug 2023 16:42:55 +0200 Subject: [PATCH 30/39] add .DS_Store and pyenv to gitignore --- .gitignore | 11 +++++++---- {{cookiecutter.project_slug}}/.gitignore | 6 ++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 432a694..041aa8c 100644 --- a/.gitignore +++ b/.gitignore @@ -55,19 +55,22 @@ target/ # Jupyter Notebook .ipynb_checkpoints +# pyenv +.python-version + # virtualenv .env .venv venv*/ +# OS specific +.DS_Store + # IDE settings .vscode/ .idea/ *.iws -# Pyenv -.python-version - # project-specific stuff serve/ -cookiecutter-server.yml \ No newline at end of file +cookiecutter-server.yml diff --git a/{{cookiecutter.project_slug}}/.gitignore b/{{cookiecutter.project_slug}}/.gitignore index 781bee0..0a25ded 100644 --- a/{{cookiecutter.project_slug}}/.gitignore +++ b/{{cookiecutter.project_slug}}/.gitignore @@ -55,11 +55,17 @@ target/ # Jupyter Notebook .ipynb_checkpoints +# pyenv +.python-version + # virtualenv .env .venv venv*/ +# OS specific +.DS_Store + # IDE settings .vscode/ .idea/ From 77f47ccb08f6728ef302f32abbf23ff3ad67ff61 Mon Sep 17 00:00:00 2001 From: Sebastian Straub Date: Thu, 10 Aug 2023 17:38:30 +0200 Subject: [PATCH 31/39] set min version to python 3.8 (3.7 is EOL) --- .github/workflows/tests-conda.yml | 2 +- .github/workflows/tests-pip.yml | 2 +- .pre-commit-config.yaml | 1 - README.md | 4 ++-- cookiecutter.json | 2 +- hooks/pre_gen_project.py | 6 +++--- tests/version_check.py | 2 +- {{cookiecutter.project_slug}}/.pre-commit-config.yaml | 2 +- {{cookiecutter.project_slug}}/docker-compose.yml | 2 +- {{cookiecutter.project_slug}}/pyproject.toml | 1 - {{cookiecutter.project_slug}}/setup.py | 2 +- .../src/{{cookiecutter.module_name}}/version.py | 7 ++----- 12 files changed, 14 insertions(+), 19 deletions(-) diff --git a/.github/workflows/tests-conda.yml b/.github/workflows/tests-conda.yml index 4ea7e1d..cfbfe1c 100644 --- a/.github/workflows/tests-conda.yml +++ b/.github/workflows/tests-conda.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7, 3.11] + python-version: [3.8, 3.12] os: [ubuntu-latest, windows-latest] name: "Test: Python ${{ matrix.python-version }}, conda, ${{ matrix.os }}" steps: diff --git a/.github/workflows/tests-pip.yml b/.github/workflows/tests-pip.yml index 0eb5218..5614204 100644 --- a/.github/workflows/tests-pip.yml +++ b/.github/workflows/tests-pip.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7, 3.11] + python-version: [3.8, 3.12] os: [ubuntu-latest, windows-latest] name: "Test: Python ${{ matrix.python-version }}, pip, ${{ matrix.os }}" steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 37341c9..968b50d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,6 @@ repos: rev: v4.4.0 hooks: - id: check-added-large-files - - id: check-ast - id: check-merge-conflict - id: end-of-file-fixer - id: mixed-line-ending diff --git a/README.md b/README.md index 1a02ab9..1e69045 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # AT Python Template [![build](https://img.shields.io/github/actions/workflow/status/at-gmbh/at-python-template/tests-pip.yml?branch=master)](https://github.com/at-gmbh/at-python-template/actions?query=branch%3Amaster+) -![Python Version](https://img.shields.io/badge/python-3.7%20--%203.11-blue) +![Python Version](https://img.shields.io/badge/python-3.8%20--%203.12-blue) [![License](https://img.shields.io/github/license/at-gmbh/at-python-template)](https://github.com/at-gmbh/at-python-template/blob/master/LICENSE) ![GitHub Repo stars](https://img.shields.io/github/stars/at-gmbh/at-python-template?style=social) @@ -13,7 +13,7 @@ This is the official Python Project Template of Alexander Thamm GmbH (AT). It is 2. `cookiecutter https://github.com/at-gmbh/at-python-template` 3. profit! -This will install or update cookiecutter on your system and create a new project in the current folder using the AT Python Template. Please note: Python 3.7 or higher is required. +This will install or update cookiecutter on your system and create a new project in the current folder using the AT Python Template. Please note: Python 3.8 or higher is required. > This template requires `cookiecutter>=1.7.2`. If you experience issues installing it into your default conda environment, we recommend to create a new clean environment with nothing but the `cookiecutter` package installed. diff --git a/cookiecutter.json b/cookiecutter.json index be82fea..663becc 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -9,7 +9,7 @@ "package_manager": ["conda", "pip", "poetry"], "use_notebooks": ["no", "yes"], "use_docker": ["no", "yes"], - "ci_pipeline": ["none", "gitlab"], + "ci_pipeline": ["none", "gitlab"], "create_cli": ["no", "yes"], "config_file": ["none", "hocon", "yaml"], "code_formatter": ["none", "black"], diff --git a/hooks/pre_gen_project.py b/hooks/pre_gen_project.py index 71c5b0b..7cf604e 100644 --- a/hooks/pre_gen_project.py +++ b/hooks/pre_gen_project.py @@ -19,9 +19,9 @@ with warnings.catch_warnings(): warnings.simplefilter("ignore", category=DeprecationWarning) - # check Python version (3.7 or higher) - if StrictVersion(platform.python_version()) < StrictVersion("3.7.0"): - print("ERROR: You are using Python {}, but Python 3.7 or higher is required " + # check Python version (3.8 or higher) + if StrictVersion(platform.python_version()) < StrictVersion("3.8.0"): + print("ERROR: You are using Python {}, but Python 3.8 or higher is required " "to use this template".format(platform.python_version())) sys.exit(1) # check cookiecutter version (1.7.2 or higher) diff --git a/tests/version_check.py b/tests/version_check.py index 81a7c4e..1caecc9 100644 --- a/tests/version_check.py +++ b/tests/version_check.py @@ -23,7 +23,7 @@ shutil.rmtree(temp_dir, ignore_errors=True) # handle possible issues & give proper return codes -if b'Python 3.7 or higher' in stdout or b'successfully created' in stdout: +if b'Python 3.8 or higher' in stdout or b'successfully created' in stdout: if actual_fail == expect_fail: print("Python {} {} as expected".format( platform.python_version(), diff --git a/{{cookiecutter.project_slug}}/.pre-commit-config.yaml b/{{cookiecutter.project_slug}}/.pre-commit-config.yaml index 64337cd..32091a9 100644 --- a/{{cookiecutter.project_slug}}/.pre-commit-config.yaml +++ b/{{cookiecutter.project_slug}}/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: rev: stable hooks: - id: black - language_version: python3.7 + language_version: python3.8 exclude: ^notebooks{% else %} - repo: https://github.com/PyCQA/flake8 rev: '5.0.4' diff --git a/{{cookiecutter.project_slug}}/docker-compose.yml b/{{cookiecutter.project_slug}}/docker-compose.yml index a55f316..d7c7871 100644 --- a/{{cookiecutter.project_slug}}/docker-compose.yml +++ b/{{cookiecutter.project_slug}}/docker-compose.yml @@ -6,6 +6,6 @@ services: build: context: .{% if cookiecutter.package_manager != 'poetry' %} args: - PYTHON_IMAGE_TAG: {% if cookiecutter.package_manager == 'conda' %}"4.8.2"{% else %}"3.7-stretch"{% endif %}{% endif %} + PYTHON_IMAGE_TAG: {% if cookiecutter.package_manager == 'conda' %}"4.8.2"{% else %}"3.8-stretch"{% endif %}{% endif %} command: '{% if cookiecutter.create_cli == 'yes' %}--help{% endif %}' tty: true diff --git a/{{cookiecutter.project_slug}}/pyproject.toml b/{{cookiecutter.project_slug}}/pyproject.toml index 402f11b..b03fe76 100644 --- a/{{cookiecutter.project_slug}}/pyproject.toml +++ b/{{cookiecutter.project_slug}}/pyproject.toml @@ -15,7 +15,6 @@ python = "^3.10"{% if cookiecutter.config_file == 'hocon' %} pyhocon = "^0.3.59"{% elif cookiecutter.config_file == 'yaml' %} PyYAML = "^6.0"{% endif %}{% if cookiecutter.create_cli == 'yes' %} typer = {extras = ["all"], version = "^0.7.0"}{% endif %} -importlib-metadata = {version = "^1.0", python = "<3.8"} [tool.poetry.dev-dependencies]{% if cookiecutter.code_formatter == 'black' %} black = "^22.10"{% endif %} diff --git a/{{cookiecutter.project_slug}}/setup.py b/{{cookiecutter.project_slug}}/setup.py index ade8190..8ff3637 100644 --- a/{{cookiecutter.project_slug}}/setup.py +++ b/{{cookiecutter.project_slug}}/setup.py @@ -35,5 +35,5 @@ def read(fname): 'pre-commit', ], platforms='any', - python_requires='>=3.7', + python_requires='>=3.8', ) diff --git a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/version.py b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/version.py index 5c86dd1..c5441b0 100644 --- a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/version.py +++ b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/version.py @@ -1,7 +1,4 @@ -{% if cookiecutter.package_manager == 'poetry' %}try: - import importlib.metadata as importlib_metadata -except ModuleNotFoundError: - import importlib_metadata +{% if cookiecutter.package_manager == 'poetry' %}from importlib.metadata import version -__version__ = importlib_metadata.version("{{ cookiecutter.project_slug }}") +__version__ = version("{{ cookiecutter.project_slug }}") {%- else %}__version__ = '0.1.0'{% endif %} From e89e17ef6a51ace74dc49a3c119357f4e0d7b29b Mon Sep 17 00:00:00 2001 From: Sebastian Straub Date: Thu, 10 Aug 2023 17:41:07 +0200 Subject: [PATCH 32/39] revert to 3.11 as max supported --- .github/workflows/tests-conda.yml | 2 +- .github/workflows/tests-pip.yml | 2 +- README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests-conda.yml b/.github/workflows/tests-conda.yml index cfbfe1c..592b102 100644 --- a/.github/workflows/tests-conda.yml +++ b/.github/workflows/tests-conda.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.8, 3.12] + python-version: [3.8, 3.11] os: [ubuntu-latest, windows-latest] name: "Test: Python ${{ matrix.python-version }}, conda, ${{ matrix.os }}" steps: diff --git a/.github/workflows/tests-pip.yml b/.github/workflows/tests-pip.yml index 5614204..bf28953 100644 --- a/.github/workflows/tests-pip.yml +++ b/.github/workflows/tests-pip.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.8, 3.12] + python-version: [3.8, 3.11] os: [ubuntu-latest, windows-latest] name: "Test: Python ${{ matrix.python-version }}, pip, ${{ matrix.os }}" steps: diff --git a/README.md b/README.md index 1e69045..0226343 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # AT Python Template [![build](https://img.shields.io/github/actions/workflow/status/at-gmbh/at-python-template/tests-pip.yml?branch=master)](https://github.com/at-gmbh/at-python-template/actions?query=branch%3Amaster+) -![Python Version](https://img.shields.io/badge/python-3.8%20--%203.12-blue) +![Python Version](https://img.shields.io/badge/python-3.8%20--%203.11-blue) [![License](https://img.shields.io/github/license/at-gmbh/at-python-template)](https://github.com/at-gmbh/at-python-template/blob/master/LICENSE) ![GitHub Repo stars](https://img.shields.io/github/stars/at-gmbh/at-python-template?style=social) From 97abe83870a6411f4e8168d4326b01245a66a9e5 Mon Sep 17 00:00:00 2001 From: Christian Baur Date: Thu, 24 Aug 2023 19:59:13 +0200 Subject: [PATCH 33/39] added human readable prompts --- cookiecutter.json | 56 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/cookiecutter.json b/cookiecutter.json index be82fea..4e410eb 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -9,9 +9,61 @@ "package_manager": ["conda", "pip", "poetry"], "use_notebooks": ["no", "yes"], "use_docker": ["no", "yes"], - "ci_pipeline": ["none", "gitlab"], + "ci_pipeline": ["none", "gitlab"], "create_cli": ["no", "yes"], "config_file": ["none", "hocon", "yaml"], "code_formatter": ["none", "black"], - "editor_settings": ["none", "pycharm", "vscode"] + "editor_settings": ["none", "pycharm", "vscode"], + "__prompts__": { + "full_name": "What is your [bold yellow]full name[/]?", + "company_name": "Enter your [bold yellow]company name[/]. Leave empty if not applicable.", + "email": "What's your [bold yellow]email address[/]?", + "project_name": "Name your [bold yellow]project[/]:", + "project_short_description": "Provide a [bold yellow]short description[/] for the project:", + "package_manager": { + "__prompt__": "Which [bold yellow]package manager[/] would you like to use?", + "conda": "Conda", + "pip": "Pip", + "poetry": "Poetry" + }, + "use_notebooks": { + "__prompt__": "Do you want to include [bold yellow]Jupyter Notebooks[/] in your project?", + "no": "No", + "yes": "Yes" + }, + "use_docker": { + "__prompt__": "Do you want to use [bold yellow]Docker[/]?", + "no": "No", + "yes": "Yes" + }, + "ci_pipeline": { + "__prompt__": "What [bold yellow]CI pipeline[/] would you like to use?", + "none": "None", + "gitlab": "GitLab" + }, + "create_cli": { + "__prompt__": "Do you want to create a [bold yellow]CLI[/] for your project?", + "no": "No", + "yes": "Yes" + }, + "config_file": { + "__prompt__": "Which [bold yellow]config file format[/] do you prefer?", + "none": "None", + "hocon": "HOCON", + "yaml": "YAML" + }, + "code_formatter": { + "__prompt__": "What [bold yellow]code formatter[/] would you like to use?", + "none": "None", + "black": "Black" + }, + "editor_settings": { + "__prompt__": "Which [bold yellow]editor settings[/] do you want to include?", + "none": "None", + "pycharm": "PyCharm", + "vscode": "VS Code" + } + } } + + From 85ebfc4f37334fd2c0cfada693c1d9882aac1590 Mon Sep 17 00:00:00 2001 From: Christian Baur Date: Thu, 24 Aug 2023 20:00:52 +0200 Subject: [PATCH 34/39] linter happiness --- cookiecutter.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/cookiecutter.json b/cookiecutter.json index 4e410eb..ede8821 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -65,5 +65,3 @@ } } } - - From 8a1a556f60ea47311f2fda060e04b1c923c9119a Mon Sep 17 00:00:00 2001 From: AlexF1994 Date: Wed, 11 Oct 2023 11:20:53 +0200 Subject: [PATCH 35/39] add poetry option for az devops --- tests/test_options.py | 11 + .../ci/test-pipeline.yml | 359 ++++++++++-------- 2 files changed, 213 insertions(+), 157 deletions(-) diff --git a/tests/test_options.py b/tests/test_options.py index 4c0913f..1573bbb 100644 --- a/tests/test_options.py +++ b/tests/test_options.py @@ -287,6 +287,17 @@ def test_az_devops_conda(az_devops_files): ) +def test_az_devops_poetry(az_devops_files): + check_project( + settings={ + "package_manager": "poetry", + "ci_pipeline": "az-devops", + "use_docker": "yes" + }, + files_existent=az_devops_files + ) + + def test_no_ci_pipeline(az_devops_files): check_project( settings={ diff --git a/{{cookiecutter.project_slug}}/ci/test-pipeline.yml b/{{cookiecutter.project_slug}}/ci/test-pipeline.yml index da809ae..d6b3340 100644 --- a/{{cookiecutter.project_slug}}/ci/test-pipeline.yml +++ b/{{cookiecutter.project_slug}}/ci/test-pipeline.yml @@ -1,157 +1,202 @@ -# trigger on pull requests needs to be set manually in Azure DevOps -trigger: none - -pool: - name: Azure Pipelines - vmImage: ubuntu-latest - -jobs: - -- job: Test - variables: - VENV_FOLDER: $(Pipeline.Workspace)/venv - {%- if cookiecutter.package_manager == 'pip' %} - PIP_CACHE_DIR: $(Pipeline.Workspace)/venv/lib - {%- elif cookiecutter.package_manager == 'conda' %} - CONDA_ENV_NAME: test-env - CONDA_PKGS_DIRS: /usr/share/miniconda/envs/$(CONDA_ENV_NAME) - {%- endif %} - PACKAGE_NAME: '{{ cookiecutter.project_slug }}' - - steps: - - checkout: self - - - task: UsePythonVersion@0 - displayName: Use Python 3.8 - inputs: - versionSpec: 3.8 - addToPath: true - - - task: Bash@3 - displayName: Install system dependencies - inputs: - targetType: inline - script: | - set -uex - sudo apt update - sudo apt install -y build-essential - - # fill in or delete if no Azuzre Keyvault is used - - task: AzureKeyVault@1 - inputs: - azureSubscription: '' - KeyVaultName: '' - SecretsFilter: '' - RunAsPreJob: false - - {%- if cookiecutter.package_manager == 'pip' %} - - task: Cache@2 - inputs: - key: 'pip | venv | $(Agent.OS) | requirements.txt | requirements-dev.txt' - path: $(PIP_CACHE_DIR) - displayName: Cache pip packages - - - task: Bash@3 - displayName: Create venv - inputs: - targetType: inline - script: | - set -uex - python -m venv $(VENV_FOLDER) - source $(VENV_FOLDER)/bin/activate - - - task: Bash@3 - displayName: Setup environment - inputs: - targetType: inline - script: | - set -uex - source $(VENV_FOLDER)/bin/activate - # resolve issues with old cached versions of pip - # Open question: Still needed? - python -m pip install --upgrade pip || (curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python get-pip.py) - python -m pip install wheel - - # install python app dependencies - python -m pip install -r requirements.txt -r requirements-dev.txt -U - - # build and install wheel - python setup.py dist - python -m pip install --force-reinstall dist/*.whl - - - task: Bash@3 - displayName: pytest (with coverage) - # define env variables as needed - env: - PACKAGE_NAME: $(PACKAGE_NAME) - inputs: - targetType: inline - script: | - set -uex - source $(VENV_FOLDER)/bin/activate - - # run pytest - python -m pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=$PACKAGE_NAME --cov-report=xml:coverage-reports/cov.xml --cov-report=html - - {% elif cookiecutter.package_manager == 'conda' -%} - - task: Cache@2 - displayName: Cache conda packages - inputs: - key: 'conda | $(Agent.OS) | environment.yml | environment-dev.yml' - path: $(CONDA_PKGS_DIRS) - cacheHitVar: CONDA_CACHE_RESTORED - - - task: Bash@3 - displayName: Create conda environment - inputs: - targetType: inline - script: | - set -uex - conda env create -n $(CONDA_ENV_NAME) -f environment-dev.yml environment.yml - source /usr/share/miniconda/etc/profile.d/conda.sh - conda activate $(CONDA_ENV_NAME) - python -m pip install wheel - condition: eq(variables.CONDA_CACHE_RESTORED, 'false') - - - task: Bash@3 - displayName: Install project as python package - inputs: - targetType: inline - script: | - set -uex - source /usr/share/miniconda/etc/profile.d/conda.sh - conda activate $(CONDA_ENV_NAME) - python setup.py dist - python -m pip install --force-reinstall dist/*.whl - - # run tests with coverage information - - task: Bash@3 - displayName: pytest (with coverage) - # define env variables as needed - env: - PACKAGE_NAME: $(PACKAGE_NAME) - inputs: - targetType: inline - script: | - set -uex - source /usr/share/miniconda/etc/profile.d/conda.sh - conda activate $(CONDA_ENV_NAME) - python -m pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=$PACKAGE_NAME --cov-report=xml:coverage-reports/cov.xml --cov-report=html - {%- endif %} - - - task: PublishCodeCoverageResults@1 - displayName: Publish code coverage - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: $(System.DefaultWorkingDirectory)/coverage-reports/cov.xml - additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/junit/ - condition: succeededOrFailed() - - # publish pytest results - - task: PublishTestResults@2 - inputs: - testResultsFormat: JUnit - testResultsFiles: $(System.DefaultWorkingDirectory)/**/*-results.xml - testRunTitle: Publish pytest results - failTaskOnFailedTests: false - condition: succeededOrFailed() +# trigger on pull requests needs to be set manually in Azure DevOps +trigger: none + +pool: + name: Azure Pipelines + vmImage: ubuntu-latest + +jobs: + +- job: Test + variables: + VENV_FOLDER: $(Pipeline.Workspace)/venv + {%- if cookiecutter.package_manager == 'pip' %} + PIP_CACHE_DIR: $(Pipeline.Workspace)/venv/lib + {%- elif cookiecutter.package_manager == 'conda' %} + CONDA_ENV_NAME: test-env + CONDA_PKGS_DIRS: /usr/share/miniconda/envs/$(CONDA_ENV_NAME) + {%- elif cookiecutter.package_manager == 'poetry' %} + POETRY_VERSION: 1.0.5 + POETRY_CACHE_DIR: $(Pipeline.Workspace)/venv/lib + {%- endif %} + PACKAGE_NAME: '{{ cookiecutter.project_slug }}' + + steps: + - checkout: self + + - task: UsePythonVersion@0 + displayName: Use Python 3.8 + inputs: + versionSpec: 3.8 + addToPath: true + + - task: Bash@3 + displayName: Install system dependencies + inputs: + targetType: inline + script: | + set -uex + sudo apt update + sudo apt install -y build-essential + + # fill in or delete if no Azuzre Keyvault is used + - task: AzureKeyVault@1 + inputs: + azureSubscription: '' + KeyVaultName: '' + SecretsFilter: '' + RunAsPreJob: false + + {%- if cookiecutter.package_manager == 'pip' %} + - task: Cache@2 + inputs: + key: 'pip | venv | $(Agent.OS) | requirements.txt | requirements-dev.txt' + path: $(PIP_CACHE_DIR) + displayName: Cache pip packages + + - task: Bash@3 + displayName: Create venv + inputs: + targetType: inline + script: | + set -uex + python -m venv $(VENV_FOLDER) + source $(VENV_FOLDER)/bin/activate + + - task: Bash@3 + displayName: Setup environment + inputs: + targetType: inline + script: | + set -uex + source $(VENV_FOLDER)/bin/activate + # resolve issues with old cached versions of pip + # Open question: Still needed? + python -m pip install --upgrade pip || (curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python get-pip.py) + python -m pip install wheel + + # install python app dependencies + python -m pip install -r requirements.txt -r requirements-dev.txt -U + + # build and install wheel + python setup.py dist + python -m pip install --force-reinstall dist/*.whl + + - task: Bash@3 + displayName: pytest (with coverage) + # define env variables as needed + env: + PACKAGE_NAME: $(PACKAGE_NAME) + inputs: + targetType: inline + script: | + set -uex + source $(VENV_FOLDER)/bin/activate + + # run pytest + python -m pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=$PACKAGE_NAME --cov-report=xml:coverage-reports/cov.xml --cov-report=html + + {% elif cookiecutter.package_manager == 'conda' -%} + - task: Cache@2 + displayName: Cache conda packages + inputs: + key: 'conda | $(Agent.OS) | environment.yml | environment-dev.yml' + path: $(CONDA_PKGS_DIRS) + cacheHitVar: CONDA_CACHE_RESTORED + + - task: Bash@3 + displayName: Create conda environment + inputs: + targetType: inline + script: | + set -uex + conda env create -n $(CONDA_ENV_NAME) -f environment-dev.yml environment.yml + source /usr/share/miniconda/etc/profile.d/conda.sh + conda activate $(CONDA_ENV_NAME) + python -m pip install wheel + condition: eq(variables.CONDA_CACHE_RESTORED, 'false') + + - task: Bash@3 + displayName: Install project as python package + inputs: + targetType: inline + script: | + set -uex + source /usr/share/miniconda/etc/profile.d/conda.sh + conda activate $(CONDA_ENV_NAME) + python setup.py dist + python -m pip install --force-reinstall dist/*.whl + + # run tests with coverage information + - task: Bash@3 + displayName: pytest (with coverage) + # define env variables as needed + env: + PACKAGE_NAME: $(PACKAGE_NAME) + inputs: + targetType: inline + script: | + set -uex + source /usr/share/miniconda/etc/profile.d/conda.sh + conda activate $(CONDA_ENV_NAME) + + python -m pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=$PACKAGE_NAME --cov-report=xml:coverage-reports/cov.xml --cov-report=html + + {% elif cookiecutter.package_manager == 'poetry' -%} + - task: Cache@2 + displayName: Cache poetry packages + inputs: + key: 'poetry | $(Agent.OS) | | pyproject.toml ' + path: $(POETRY_CACHE_DIR) + cacheHitVar: POETRY_CACHE_RESTORED + + - task: Bash@3 + displayName: Configure poetry + inputs: + targetType: inline + script: | + set -uex + pip install poetry==$(POETRY_VERSION) + poetry config virtualenvs.in-project false + poetry config virtualenvs.path $(VENV_FOLDER) + + - task: Bash@3 + displayName: Create venv + inputs: + targetType: inline + script: | + set -uex + poetry install + source `poetry env info --path`/bin/activate + condition: eq(variables.CONDA_CACHE_RESTORED, 'false') + + # run tests with coverage information + - task: Bash@3 + displayName: pytest (with coverage) + # define env variables as needed + env: + PACKAGE_NAME: $(PACKAGE_NAME) + inputs: + targetType: inline + script: | + set -uex + source `poetry env info --path`/bin/activate + python -m pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=$PACKAGE_NAME --cov-report=xml:coverage-reports/cov.xml --cov-report=html + {%- endif %} + + - task: PublishCodeCoverageResults@1 + displayName: Publish code coverage + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: $(System.DefaultWorkingDirectory)/coverage-reports/cov.xml + additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/junit/ + condition: succeededOrFailed() + + # publish pytest results + - task: PublishTestResults@2 + inputs: + testResultsFormat: JUnit + testResultsFiles: $(System.DefaultWorkingDirectory)/**/*-results.xml + testRunTitle: Publish pytest results + failTaskOnFailedTests: false + condition: succeededOrFailed() From 82f94ff69ce36f925f999a179c33090c7f680a00 Mon Sep 17 00:00:00 2001 From: AlexF1994 Date: Wed, 18 Oct 2023 16:13:51 +0200 Subject: [PATCH 36/39] make poetry test pipeline work --- .../ci/test-pipeline.yml | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/{{cookiecutter.project_slug}}/ci/test-pipeline.yml b/{{cookiecutter.project_slug}}/ci/test-pipeline.yml index d6b3340..d04b239 100644 --- a/{{cookiecutter.project_slug}}/ci/test-pipeline.yml +++ b/{{cookiecutter.project_slug}}/ci/test-pipeline.yml @@ -16,8 +16,10 @@ jobs: CONDA_ENV_NAME: test-env CONDA_PKGS_DIRS: /usr/share/miniconda/envs/$(CONDA_ENV_NAME) {%- elif cookiecutter.package_manager == 'poetry' %} - POETRY_VERSION: 1.0.5 - POETRY_CACHE_DIR: $(Pipeline.Workspace)/venv/lib + POETRY_VERSION: 1.6 + # we have to cache the whole folder in order to activate the env later on + # otherwise the activate binary isn't restored for a cache hit + POETRY_CACHE_DIR: $(System.DefaultWorkingDirectory)/.venv {%- endif %} PACKAGE_NAME: '{{ cookiecutter.project_slug }}' @@ -48,6 +50,7 @@ jobs: RunAsPreJob: false {%- if cookiecutter.package_manager == 'pip' %} + - task: Cache@2 inputs: key: 'pip | venv | $(Agent.OS) | requirements.txt | requirements-dev.txt' @@ -97,6 +100,7 @@ jobs: python -m pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=$PACKAGE_NAME --cov-report=xml:coverage-reports/cov.xml --cov-report=html {% elif cookiecutter.package_manager == 'conda' -%} + - task: Cache@2 displayName: Cache conda packages inputs: @@ -143,10 +147,11 @@ jobs: python -m pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=$PACKAGE_NAME --cov-report=xml:coverage-reports/cov.xml --cov-report=html {% elif cookiecutter.package_manager == 'poetry' -%} + - task: Cache@2 displayName: Cache poetry packages inputs: - key: 'poetry | $(Agent.OS) | | pyproject.toml ' + key: 'poetry | $(Agent.OS) | pyproject.toml ' path: $(POETRY_CACHE_DIR) cacheHitVar: POETRY_CACHE_RESTORED @@ -157,8 +162,6 @@ jobs: script: | set -uex pip install poetry==$(POETRY_VERSION) - poetry config virtualenvs.in-project false - poetry config virtualenvs.path $(VENV_FOLDER) - task: Bash@3 displayName: Create venv @@ -166,22 +169,21 @@ jobs: targetType: inline script: | set -uex - poetry install + poetry install --no-root source `poetry env info --path`/bin/activate - condition: eq(variables.CONDA_CACHE_RESTORED, 'false') + condition: eq(variables.POETRY_CACHE_RESTORED, 'false') # run tests with coverage information - task: Bash@3 displayName: pytest (with coverage) - # define env variables as needed - env: - PACKAGE_NAME: $(PACKAGE_NAME) inputs: targetType: inline script: | set -uex + # install root which obviously shouldn't be cached + poetry install --only-root source `poetry env info --path`/bin/activate - python -m pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=$PACKAGE_NAME --cov-report=xml:coverage-reports/cov.xml --cov-report=html + python -m pytest tests --doctest-modules --junitxml=$(System.DefaultWorkingDirectory)/junit/test-results.xml --cov=src/'{{ cookiecutter.module_name }}' --cov-report=xml:$(System.DefaultWorkingDirectory)/coverage-reports/cov.xml --cov-report=html:$(System.DefaultWorkingDirectory)/coverage-reports/cov.html {%- endif %} - task: PublishCodeCoverageResults@1 From ac84560d5fc29d4e792812c504490d336a38d203 Mon Sep 17 00:00:00 2001 From: AlexF1994 Date: Wed, 18 Oct 2023 16:59:49 +0200 Subject: [PATCH 37/39] solve new line issue --- {{cookiecutter.project_slug}}/ci/test-pipeline.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/{{cookiecutter.project_slug}}/ci/test-pipeline.yml b/{{cookiecutter.project_slug}}/ci/test-pipeline.yml index d04b239..ba59b27 100644 --- a/{{cookiecutter.project_slug}}/ci/test-pipeline.yml +++ b/{{cookiecutter.project_slug}}/ci/test-pipeline.yml @@ -41,7 +41,7 @@ jobs: sudo apt update sudo apt install -y build-essential - # fill in or delete if no Azuzre Keyvault is used + # fill in or delete if no Azure Keyvault is used - task: AzureKeyVault@1 inputs: azureSubscription: '' @@ -98,8 +98,7 @@ jobs: # run pytest python -m pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=$PACKAGE_NAME --cov-report=xml:coverage-reports/cov.xml --cov-report=html - - {% elif cookiecutter.package_manager == 'conda' -%} + {% elif cookiecutter.package_manager == 'conda' %} - task: Cache@2 displayName: Cache conda packages @@ -143,10 +142,8 @@ jobs: set -uex source /usr/share/miniconda/etc/profile.d/conda.sh conda activate $(CONDA_ENV_NAME) - python -m pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=$PACKAGE_NAME --cov-report=xml:coverage-reports/cov.xml --cov-report=html - - {% elif cookiecutter.package_manager == 'poetry' -%} + {% elif cookiecutter.package_manager == 'poetry' %} - task: Cache@2 displayName: Cache poetry packages @@ -184,8 +181,8 @@ jobs: poetry install --only-root source `poetry env info --path`/bin/activate python -m pytest tests --doctest-modules --junitxml=$(System.DefaultWorkingDirectory)/junit/test-results.xml --cov=src/'{{ cookiecutter.module_name }}' --cov-report=xml:$(System.DefaultWorkingDirectory)/coverage-reports/cov.xml --cov-report=html:$(System.DefaultWorkingDirectory)/coverage-reports/cov.html - {%- endif %} + {%- endif %} - task: PublishCodeCoverageResults@1 displayName: Publish code coverage inputs: From 5fb9e2a73c3dde2db1d40547a94e3a425e5b7fc2 Mon Sep 17 00:00:00 2001 From: Sebastian Straub Date: Wed, 25 Oct 2023 10:35:39 +0200 Subject: [PATCH 38/39] adjust descriptions --- cookiecutter.json | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/cookiecutter.json b/cookiecutter.json index ede8821..a9b3a8f 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -15,16 +15,18 @@ "code_formatter": ["none", "black"], "editor_settings": ["none", "pycharm", "vscode"], "__prompts__": { - "full_name": "What is your [bold yellow]full name[/]?", - "company_name": "Enter your [bold yellow]company name[/]. Leave empty if not applicable.", + "full_name": "What's your [bold yellow]name[/]?", + "company_name": "Enter your [bold yellow]company name[/]; leave empty if not applicable", "email": "What's your [bold yellow]email address[/]?", - "project_name": "Name your [bold yellow]project[/]:", - "project_short_description": "Provide a [bold yellow]short description[/] for the project:", + "project_name": "Please provide the [bold yellow]full name of your project[/], as it would appear in a headline", + "project_slug": "Please provide a [bold yellow]slug[/], which is the project name as it would appear in a URL, or accept this suggestion", + "module_name": "Please provide a [bold yellow]module name[/] for your project. Make it short, if possible, and use underscores instead of whitespace", + "project_short_description": "Provide a [bold yellow]short description[/] for the project in one sentence", "package_manager": { - "__prompt__": "Which [bold yellow]package manager[/] would you like to use?", - "conda": "Conda", - "pip": "Pip", - "poetry": "Poetry" + "__prompt__": "Which [bold yellow]packaging tool[/] would you like to use?", + "conda": "conda (environment.yml)", + "pip": "pip (setup.py)", + "poetry": "poetry (pyproject.toml)" }, "use_notebooks": { "__prompt__": "Do you want to include [bold yellow]Jupyter Notebooks[/] in your project?", @@ -39,7 +41,7 @@ "ci_pipeline": { "__prompt__": "What [bold yellow]CI pipeline[/] would you like to use?", "none": "None", - "gitlab": "GitLab" + "gitlab": "GitLab CI" }, "create_cli": { "__prompt__": "Do you want to create a [bold yellow]CLI[/] for your project?", From 57a1c444aa9f4410bc1723b2ef24b4b36664000f Mon Sep 17 00:00:00 2001 From: AlexF1994 Date: Tue, 7 Nov 2023 14:13:33 +0100 Subject: [PATCH 39/39] fix missing comma in cookiecutter.json --- cookiecutter.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookiecutter.json b/cookiecutter.json index 303427f..8f6b2b7 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -18,7 +18,7 @@ "cd/build-dev.yml", "cd/build.yml", "cd/delete-old-images.yml" - ] + ], "__prompts__": { "full_name": "What's your [bold yellow]name[/]?", "company_name": "Enter your [bold yellow]company name[/]; leave empty if not applicable",