diff --git a/.github/workflows/tests-conda.yml b/.github/workflows/tests-conda.yml index 4ea7e1d..592b102 100644 --- a/.github/workflows/tests-conda.yml +++ b/.github/workflows/tests-conda.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7, 3.11] + python-version: [3.8, 3.11] os: [ubuntu-latest, windows-latest] name: "Test: Python ${{ matrix.python-version }}, conda, ${{ matrix.os }}" steps: diff --git a/.github/workflows/tests-pip.yml b/.github/workflows/tests-pip.yml index 0eb5218..bf28953 100644 --- a/.github/workflows/tests-pip.yml +++ b/.github/workflows/tests-pip.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7, 3.11] + python-version: [3.8, 3.11] os: [ubuntu-latest, windows-latest] name: "Test: Python ${{ matrix.python-version }}, pip, ${{ matrix.os }}" steps: diff --git a/.gitignore b/.gitignore index 432a694..041aa8c 100644 --- a/.gitignore +++ b/.gitignore @@ -55,19 +55,22 @@ target/ # Jupyter Notebook .ipynb_checkpoints +# pyenv +.python-version + # virtualenv .env .venv venv*/ +# OS specific +.DS_Store + # IDE settings .vscode/ .idea/ *.iws -# Pyenv -.python-version - # project-specific stuff serve/ -cookiecutter-server.yml \ No newline at end of file +cookiecutter-server.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 37341c9..968b50d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,6 @@ repos: rev: v4.4.0 hooks: - id: check-added-large-files - - id: check-ast - id: check-merge-conflict - id: end-of-file-fixer - id: mixed-line-ending diff --git a/README.md b/README.md index 1a02ab9..8f578fb 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # AT Python Template [![build](https://img.shields.io/github/actions/workflow/status/at-gmbh/at-python-template/tests-pip.yml?branch=master)](https://github.com/at-gmbh/at-python-template/actions?query=branch%3Amaster+) -![Python Version](https://img.shields.io/badge/python-3.7%20--%203.11-blue) +![Python Version](https://img.shields.io/badge/python-3.8%20--%203.11-blue) [![License](https://img.shields.io/github/license/at-gmbh/at-python-template)](https://github.com/at-gmbh/at-python-template/blob/master/LICENSE) ![GitHub Repo stars](https://img.shields.io/github/stars/at-gmbh/at-python-template?style=social) @@ -13,7 +13,7 @@ This is the official Python Project Template of Alexander Thamm GmbH (AT). It is 2. `cookiecutter https://github.com/at-gmbh/at-python-template` 3. profit! -This will install or update cookiecutter on your system and create a new project in the current folder using the AT Python Template. Please note: Python 3.7 or higher is required. +This will install or update cookiecutter on your system and create a new project in the current folder using the AT Python Template. Please note: Python 3.8 or higher is required. > This template requires `cookiecutter>=1.7.2`. If you experience issues installing it into your default conda environment, we recommend to create a new clean environment with nothing but the `cookiecutter` package installed. @@ -44,6 +44,7 @@ Unfortunately, cookiecutter does not allow us to show any description of the opt * Select your `ci_pipeline` - `none` (default): Don't use any CI/CD pipeline. - `gitlab`: If you plan to use GitLab, this option will add a CI/CD Pipeline definition for [GitLab CI/CD](https://docs.gitlab.com/ee/ci/). The pipeline includes basic steps to build, test and deploy your code. The deployment steps do nothing but echoing a String, as deployment is very project-specific. + - `az-devops`: If you plan to use [Azure DevOps](https://azure.microsoft.com/en-us/products/devops), this option will add a CI pipeline and templates for CD pipelines. For the CD pipelines to work, you need to add project specific information. * `create_cli` (yes or no): if you plan to build an application with a command line interface (CLI), select *yes* here. This will integrate a template for the CLI into your project - minimal boilerplate guaranteed! (We're leveraging the awesome [typer](https://typer.tiangolo.com/) library for this.) * `config_file`: select your preferred config format. It is best practice to store your configuration separate from your code, even for small projects, but because there are a gazillion ways to do this, each project seems to reinvents the wheel. We want to provide a few options to set you up with a working configuration: - `yaml`: use [YAML](https://yaml.org/) as your configuration file format. Easy to read and write, widely adopted, relies on the [PyYAML](https://pyyaml.org/) package. diff --git a/cookiecutter.json b/cookiecutter.json index be82fea..8f6b2b7 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -9,9 +9,66 @@ "package_manager": ["conda", "pip", "poetry"], "use_notebooks": ["no", "yes"], "use_docker": ["no", "yes"], - "ci_pipeline": ["none", "gitlab"], + "ci_pipeline": ["none", "gitlab", "az-devops"], "create_cli": ["no", "yes"], "config_file": ["none", "hocon", "yaml"], "code_formatter": ["none", "black"], - "editor_settings": ["none", "pycharm", "vscode"] + "editor_settings": ["none", "pycharm", "vscode"], + "_copy_without_render": [ + "cd/build-dev.yml", + "cd/build.yml", + "cd/delete-old-images.yml" + ], + "__prompts__": { + "full_name": "What's your [bold yellow]name[/]?", + "company_name": "Enter your [bold yellow]company name[/]; leave empty if not applicable", + "email": "What's your [bold yellow]email address[/]?", + "project_name": "Please provide the [bold yellow]full name of your project[/], as it would appear in a headline", + "project_slug": "Please provide a [bold yellow]slug[/], which is the project name as it would appear in a URL, or accept this suggestion", + "module_name": "Please provide a [bold yellow]module name[/] for your project. Make it short, if possible, and use underscores instead of whitespace", + "project_short_description": "Provide a [bold yellow]short description[/] for the project in one sentence", + "package_manager": { + "__prompt__": "Which [bold yellow]packaging tool[/] would you like to use?", + "conda": "conda (environment.yml)", + "pip": "pip (setup.py)", + "poetry": "poetry (pyproject.toml)" + }, + "use_notebooks": { + "__prompt__": "Do you want to include [bold yellow]Jupyter Notebooks[/] in your project?", + "no": "No", + "yes": "Yes" + }, + "use_docker": { + "__prompt__": "Do you want to use [bold yellow]Docker[/]?", + "no": "No", + "yes": "Yes" + }, + "ci_pipeline": { + "__prompt__": "What [bold yellow]CI pipeline[/] would you like to use?", + "none": "None", + "gitlab": "GitLab CI" + }, + "create_cli": { + "__prompt__": "Do you want to create a [bold yellow]CLI[/] for your project?", + "no": "No", + "yes": "Yes" + }, + "config_file": { + "__prompt__": "Which [bold yellow]config file format[/] do you prefer?", + "none": "None", + "hocon": "HOCON", + "yaml": "YAML" + }, + "code_formatter": { + "__prompt__": "What [bold yellow]code formatter[/] would you like to use?", + "none": "None", + "black": "Black" + }, + "editor_settings": { + "__prompt__": "Which [bold yellow]editor settings[/] do you want to include?", + "none": "None", + "pycharm": "PyCharm", + "vscode": "VS Code" + } + } } diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index 3cf7a48..e7f09ef 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -73,7 +73,18 @@ ".gitlab-ci.yml", } -files_ci_all = files_ci_gitlab +files_ci_devops = { + 'ci/test-pipeline.yml', +} + +files_cd_devops = { + 'cd/build-dev.yml', + 'cd/build.yml', + 'cd/trigger.yml', + 'cd/delete-old-images.yml' +} + +files_ci_all = files_ci_gitlab | files_ci_devops | files_cd_devops folders_editor = [ '.idea__editor', @@ -167,10 +178,20 @@ def handle_editor_settings(): def handle_ci(): ci_pipeline = '{{ cookiecutter.ci_pipeline }}' + use_docker = '{{ cookiecutter.use_docker }}' if ci_pipeline == "gitlab": _delete_files(files_ci_all - files_ci_gitlab) + os.rmdir('ci') + os.rmdir('cd') + elif ci_pipeline == "az-devops": + _delete_files(files_ci_all - files_ci_devops - files_cd_devops) + if use_docker == 'no': + _delete_files(files_cd_devops) + os.rmdir('cd') elif ci_pipeline == 'none': _delete_files(files_ci_all) + os.rmdir('ci') + os.rmdir('cd') def print_success(): diff --git a/hooks/pre_gen_project.py b/hooks/pre_gen_project.py index 71c5b0b..7cf604e 100644 --- a/hooks/pre_gen_project.py +++ b/hooks/pre_gen_project.py @@ -19,9 +19,9 @@ with warnings.catch_warnings(): warnings.simplefilter("ignore", category=DeprecationWarning) - # check Python version (3.7 or higher) - if StrictVersion(platform.python_version()) < StrictVersion("3.7.0"): - print("ERROR: You are using Python {}, but Python 3.7 or higher is required " + # check Python version (3.8 or higher) + if StrictVersion(platform.python_version()) < StrictVersion("3.8.0"): + print("ERROR: You are using Python {}, but Python 3.8 or higher is required " "to use this template".format(platform.python_version())) sys.exit(1) # check cookiecutter version (1.7.2 or higher) diff --git a/tests/test_options.py b/tests/test_options.py index 6783bc5..7cb3246 100644 --- a/tests/test_options.py +++ b/tests/test_options.py @@ -1,6 +1,27 @@ from pathlib import Path from .util import assert_file_contains, check_project +import pytest + + +@pytest.fixture +def az_devops_cd_files(): + return [ + 'cd/build.yml', + 'cd/build-dev.yml', + 'cd/delete-old-images.yml', + 'cd/trigger.yml' + ] + + +@pytest.fixture +def az_devops_ci_files(): + return ['ci/test-pipeline.yml'] + + +@pytest.fixture +def az_devops_files(az_devops_ci_files, az_devops_cd_files): + return az_devops_ci_files + az_devops_cd_files def test_base(): @@ -61,10 +82,26 @@ def test_docker_poetry(): def test_docker_no(): check_project( - settings={'use_docker': 'no'}, + settings={ + 'use_docker': 'no' + }, files_non_existent=['Dockerfile', 'docker-compose.yml', '.dockerignore']) +def test_docker_no_az_devops(az_devops_cd_files): + check_project( + settings={ + 'use_docker': 'no', + 'ci_pipeline': 'az-devops' + }, + files_non_existent=[ + 'Dockerfile', + 'docker-compose.yml', + '.dockerignore', + ].extend(az_devops_cd_files) + ) + + def test_cli_yes(): check_project( settings={'create_cli': 'yes'}, @@ -197,6 +234,7 @@ def test_poetry_regression(): run_pytest=True, ) + def test_gitlab_pip(): check_project( settings={ @@ -206,6 +244,7 @@ def test_gitlab_pip(): files_existent=[".gitlab-ci.yml"] ) + def test_gitlab_conda(): check_project( settings={ @@ -215,6 +254,7 @@ def test_gitlab_conda(): files_existent=[".gitlab-ci.yml"] ) + def test_gitlab_poetry(): check_project( settings={ @@ -224,10 +264,44 @@ def test_gitlab_poetry(): files_existent=[".gitlab-ci.yml"] ) -def test_no_ci_pipeline(): + +def test_az_devops_pip(az_devops_files): + check_project( + settings={ + "package_manager": "pip", + "ci_pipeline": "az-devops", + "use_docker": "yes" + }, + files_existent=az_devops_files + ) + + +def test_az_devops_conda(az_devops_files): + check_project( + settings={ + "package_manager": "conda", + "ci_pipeline": "az-devops", + "use_docker": "yes" + }, + files_existent=az_devops_files + ) + + +def test_az_devops_poetry(az_devops_files): + check_project( + settings={ + "package_manager": "poetry", + "ci_pipeline": "az-devops", + "use_docker": "yes" + }, + files_existent=az_devops_files + ) + + +def test_no_ci_pipeline(az_devops_files): check_project( settings={ "ci_pipeline": "none" }, - files_non_existent=[".gitlab-ci.yml"] + files_non_existent=[".gitlab-ci.yml"] + az_devops_files ) diff --git a/tests/version_check.py b/tests/version_check.py index 81a7c4e..1caecc9 100644 --- a/tests/version_check.py +++ b/tests/version_check.py @@ -23,7 +23,7 @@ shutil.rmtree(temp_dir, ignore_errors=True) # handle possible issues & give proper return codes -if b'Python 3.7 or higher' in stdout or b'successfully created' in stdout: +if b'Python 3.8 or higher' in stdout or b'successfully created' in stdout: if actual_fail == expect_fail: print("Python {} {} as expected".format( platform.python_version(), diff --git a/{{cookiecutter.project_slug}}/.gitignore b/{{cookiecutter.project_slug}}/.gitignore index 781bee0..0a25ded 100644 --- a/{{cookiecutter.project_slug}}/.gitignore +++ b/{{cookiecutter.project_slug}}/.gitignore @@ -55,11 +55,17 @@ target/ # Jupyter Notebook .ipynb_checkpoints +# pyenv +.python-version + # virtualenv .env .venv venv*/ +# OS specific +.DS_Store + # IDE settings .vscode/ .idea/ diff --git a/{{cookiecutter.project_slug}}/.gitlab-ci.yml b/{{cookiecutter.project_slug}}/.gitlab-ci.yml index 0c0e110..316e614 100644 --- a/{{cookiecutter.project_slug}}/.gitlab-ci.yml +++ b/{{cookiecutter.project_slug}}/.gitlab-ci.yml @@ -27,6 +27,7 @@ stages: - deploy-prod # Build and Test + {% if cookiecutter.package_manager == 'poetry' -%} build-wheel: image: python:3-slim @@ -38,8 +39,7 @@ build-wheel: expire_in: 6 mos script: - pip install poetry==$POETRY_VERSION - - poetry build -f wheel -{% else -%} + - poetry build -f wheel {% else -%} build-wheel: image: python:3-slim stage: build @@ -49,8 +49,7 @@ build-wheel: - dist/{{ cookiecutter.module_name }}-*.whl expire_in: 6 mos script: - - python setup.py dist -{%- endif %} + - python setup.py dist {%- endif %} {% if cookiecutter.package_manager == 'poetry' -%} test-unit: @@ -68,8 +67,7 @@ test-unit: - poetry install --no-root - source `poetry env info --path`/bin/activate script: - - pytest tests -{% elif cookiecutter.package_manager == 'conda' -%} + - pytest tests {% elif cookiecutter.package_manager == 'conda' -%} test-unit: stage: test image: continuumio/miniconda3 @@ -88,8 +86,7 @@ test-unit: - source activate .venv - pip install dist/{{ cookiecutter.module_name }}-*.whl script: - - pytest tests -{% elif cookiecutter.package_manager == 'pip' -%} + - pytest tests {% elif cookiecutter.package_manager == 'pip' -%} test-unit: stage: test image: python:3-slim @@ -106,8 +103,7 @@ test-unit: - pip install -r requirements.txt -r requirements-dev.txt - pip install dist/{{ cookiecutter.module_name }}-*.whl script: - - pytest tests -{%- endif %} + - pytest tests {%- endif %} # Nonprod deployments diff --git a/{{cookiecutter.project_slug}}/.pre-commit-config.yaml b/{{cookiecutter.project_slug}}/.pre-commit-config.yaml index 64337cd..32091a9 100644 --- a/{{cookiecutter.project_slug}}/.pre-commit-config.yaml +++ b/{{cookiecutter.project_slug}}/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: rev: stable hooks: - id: black - language_version: python3.7 + language_version: python3.8 exclude: ^notebooks{% else %} - repo: https://github.com/PyCQA/flake8 rev: '5.0.4' diff --git a/{{cookiecutter.project_slug}}/Dockerfile__conda b/{{cookiecutter.project_slug}}/Dockerfile__conda index 118796a..e703509 100644 --- a/{{cookiecutter.project_slug}}/Dockerfile__conda +++ b/{{cookiecutter.project_slug}}/Dockerfile__conda @@ -1,21 +1,20 @@ -ARG PYTHON_IMAGE_TAG=4.8.2 +ARG IMAGE_TAG=22.9.0-2 + +FROM condaforge/mambaforge:${IMAGE_TAG} + +LABEL maintainer="{{ cookiecutter.company_name if cookiecutter.company_name else cookiecutter.full_name }}" -FROM continuumio/miniconda3:${PYTHON_IMAGE_TAG} -{%- if cookiecutter.company_name %} -LABEL maintainer="{{ cookiecutter.company_name }}" -{% else %} -LABEL maintainer="{{ cookiecutter.full_name }}" -{% endif -%} WORKDIR /{{cookiecutter.module_name}} -COPY . . +COPY environment.yml . RUN conda config --set channel_priority strict && \ - conda env create -n {{cookiecutter.module_name}}_env -f environment.yml + mamba env create -n {{cookiecutter.module_name}}_env -f environment.yml # Make RUN commands use the new environment (see: https://pythonspeed.com/articles/activate-conda-dockerfile/) -SHELL ["conda", "run", "-n", "{{cookiecutter.module_name}}_env", "/bin/bash", "-c"] +SHELL ["mamba", "run", "-n", "{{cookiecutter.module_name}}_env", "/bin/bash", "-c"] +COPY . . RUN python setup.py install # ENTRYPOINT doesn't use the same shell as RUN so you need the conda stuff -ENTRYPOINT ["conda", "run", "-n", "{{cookiecutter.module_name}}_env", "python", "-OO", "-m", "{{ cookiecutter.module_name }}"] +ENTRYPOINT ["mamba", "run", "-n", "{{cookiecutter.module_name}}_env", "python", "-OO", "-m", "{{ cookiecutter.module_name }}"] diff --git a/{{cookiecutter.project_slug}}/README.md b/{{cookiecutter.project_slug}}/README.md index a197710..44f892f 100644 --- a/{{cookiecutter.project_slug}}/README.md +++ b/{{cookiecutter.project_slug}}/README.md @@ -30,13 +30,15 @@ To set up your local development environment, please use a fresh virtual environ pip install -r requirements.txt -r requirements-dev.txt pip install -e . -The first command will install all requirements for the application and to execute tests. With the second command, you'll get an editable installation of the module, so that imports work properly. +The first command will install all requirements for the application and to execute tests. +With the second command, you'll get an editable installation of the module, so that imports work properly. {% elif cookiecutter.package_manager == 'poetry' %} To set up your local development environment, please run: poetry install -Behind the scenes, this creates a virtual environment and installs `{{ cookiecutter.module_name }}` along with its dependencies into a new virtualenv. Whenever you run `poetry run `, that `` is actually run inside the virtualenv managed by poetry. +Behind the scenes, this creates a virtual environment and installs `{{ cookiecutter.module_name }}` along with its dependencies into a new virtualenv. +Whenever you run `poetry run `, that `` is actually run inside the virtualenv managed by poetry. {% endif -%} {% if cookiecutter.create_cli == 'yes' %} @@ -85,6 +87,7 @@ This way, you'll always use the latest version of your module code in your noteb Assuming you already have Jupyter installed, you can make your virtual environment available as a separate kernel by running: {{ install_command }} ipykernel + {{ py_command }} -m ipykernel install --user --name="{{ cookiecutter.project_slug }}" Note that we mainly use notebooks for experiments, visualizations and reports. Every piece of functionality that is meant to be reused should go into module code and be imported into notebooks. @@ -102,6 +105,7 @@ this will clean up the build folder and then run the `bdist_wheel` command. Before contributing, please set up the pre-commit hooks to reduce errors and ensure consistency pip install -U pre-commit + pre-commit install If you run into any issues, you can remove the hooks again with `pre-commit uninstall`. diff --git a/{{cookiecutter.project_slug}}/cd/build-dev.yml b/{{cookiecutter.project_slug}}/cd/build-dev.yml new file mode 100644 index 0000000..fbc062f --- /dev/null +++ b/{{cookiecutter.project_slug}}/cd/build-dev.yml @@ -0,0 +1,47 @@ +# Pipeline which builds the docker image for DEV and pushes it to DEV Container Registry +# +# expected parameters: +# ACR: Azure Container Registry ID to push image to +# ACR_SUB: Service Connection to authenticate against the Azure Container Registry +# REPOSITORY: The repository to push to in the Container Registry +# SUBSCRIPTION: Service Connection used for Azure CLI Task +# IMAGE_COUNT: Number of last images to retain, all others are deleted + +parameters: +- name: ACR + type: string +- name: ACR_SUB + type: string +- name: REPOSITORY + type: string +- name: SUBSCRIPTION + type: string +- name: IMAGE_COUNT + type: number + +jobs: +- job: build_push_docker_image + steps: + - checkout: self + + - task: Docker@2 + inputs: + containerRegistry: ${{ parameters.ACR_SUB }} + repository: ${{ parameters.REPOSITORY }} + command: build + Dockerfile: '**/Dockerfile' + + - task: Docker@2 + inputs: + containerRegistry: ${{ parameters.ACR_SUB }} + repository: ${{ parameters.REPOSITORY }} + command: push + Dockerfile: '**/Dockerfile' + tags: $(Build.BuildId) + + - template: delete-old-images.yml + parameters: + ACR: ${{ parameters.ACR }} + SUBSCRIPTION: ${{ parameters.SUBSCRIPTION }} + REPOSITORY: ${{ parameters.REPOSITORY }} + IMAGE_COUNT: ${{ parameters.IMAGE_COUNT}} diff --git a/{{cookiecutter.project_slug}}/cd/build.yml b/{{cookiecutter.project_slug}}/cd/build.yml new file mode 100644 index 0000000..4602395 --- /dev/null +++ b/{{cookiecutter.project_slug}}/cd/build.yml @@ -0,0 +1,53 @@ +# Pipeline which copies docker image from the Container Registry in a (previous) environment to the next environment +# This will be used to deploy the image DEV -> QA -> PROD +# +# expected parameters: +# ACR_PREVIOUS : Azure Container Registry to pull Docker image from +# ACR_NEXT: Azure Container Registry to push Docker image to +# ACR_SUB_PREVIOUS: Service Connection used for authentication with Container Registry OLD +# ACR_SUB_NEXT: Service Connection used for authentication with Container Registry NEW +# REPOSITORY: The repository to push and pull to/from in the Container Registries, should be the same for all environments to avoid complexity +# IMAGE_COUNT: Number of last images to retain, all others are deleted + +parameters: +- name: ACR_PREVIOUS + type: string +- name: ACR_NEXT + type: string +- name: ACR_SUB_PREVIOUS + type: string +- name: ACR_SUB_NEXT + type: string +- name: REPOSITORY + type: string +- name: IMAGE_COUNT + type: number +- name: SUBSCRIPTION + type: string + +steps: +- task: Docker@2 + displayName: Pull image from container repository of previous environment + inputs: + containerRegistry: ${{ parameters.ACR_SUB_PREVIOUS }} + repository: ${{ parameters.REPOSITORY }} + command: pull + arguments: ${{ parameters.ACR_PREVIOUS }}/${{ parameters.REPOSITORY }}:$(Build.BuildId) + +- bash: docker tag ${{ parameters.ACR_PREVIOUS }}/${{ parameters.REPOSITORY }}:$(Build.BuildId) ${{ parameters.ACR_NEXT }}/${{ parameters.REPOSITORY }}:$(Build.BuildId) + displayName: Promote Docker image to registry of next environment + +- task: Docker@2 + displayName: Push Image to container repository of next environment + inputs: + containerRegistry: ${{ parameters.ACR_SUB_NEXT }} + repository: ${{ parameters.REPOSITORY }} + command: push + tags: $(Build.BuildId) + +- template: delete-old-images.yml + parameters: + ACR: ${{ parameters.ACR_NEXT }} + SUBSCRIPTION: ${{ parameters.SUBSCRIPTION }} + REPOSITORY: ${{ parameters.REPOSITORY }} + IMAGE_COUNT: ${{ parameters.IMAGE_COUNT}} diff --git a/{{cookiecutter.project_slug}}/cd/delete-old-images.yml b/{{cookiecutter.project_slug}}/cd/delete-old-images.yml new file mode 100644 index 0000000..0dd7969 --- /dev/null +++ b/{{cookiecutter.project_slug}}/cd/delete-old-images.yml @@ -0,0 +1,39 @@ +# This template encapsulates the functionality to delete old images, +# so only a pre-specified number of images is present in the registry +# +# Expected Parameters +# ACR: Azure Container Registry containing the images to delete +# REPOSITORY: Specific Repo in the Registry +# IMAGE_COUNT: Number of historic images to retain +# ACR_SUB: Azure Service Connection to use for authentication against registry + +parameters: + - name: ACR + type: string + - name: REPOSITORY + type: string + - name: IMAGE_COUNT + type: number + - name: SUBSCRIPTION + type: string + +steps: + - task: AzureCLI@2 + displayName: 'Clean-up old Docker image' + inputs: + azureSubscription: ${{ parameters.SUBSCRIPTION }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + ACR=${{ parameters.ACR }} + REPOSITORY=${{ parameters.REPOSITORY }} + # Number of newest images in the repository that will not be deleted + COUNT=${{ parameters.IMAGE_COUNT }} + + OLD_IMAGES=$(az acr repository show-tags --name $ACR --repository $REPOSITORY --orderby time_asc -o tsv | head -n -$COUNT) + echo "$OLD_IMAGES" + for OLD_IMAGE in $OLD_IMAGES + do + az acr repository delete --name $ACR --image $REPOSITORY:$OLD_IMAGE --yes + done + arguments: '-failOnStandardError false' diff --git a/{{cookiecutter.project_slug}}/cd/trigger.yml b/{{cookiecutter.project_slug}}/cd/trigger.yml new file mode 100644 index 0000000..a0ee95f --- /dev/null +++ b/{{cookiecutter.project_slug}}/cd/trigger.yml @@ -0,0 +1,66 @@ +trigger: + branches: + include: + - master + paths: + include: + - src/{{ cookiecutter.module_name }} + +pool: + vmImage: 'Ubuntu-latest' + +stages: +- stage: dev_build + displayName: Build and Push Docker Image to DEV + jobs: + - template: build-dev.yml + parameters: + ACR: # complete with the identifier of your azure container registry on DEV + ACR_SUB: # complete with name of docker registry service connection on DEV + REPOSITORY: # complete with name of repository + SUBSCRIPTION: # complete with name of service connection + IMAGE_COUNT: 5 + +- stage: qa_build + dependsOn: dev_build + displayName: Copy Docker Image from DEV to QA + jobs: + - deployment: dev_to_qa + displayName: Copy Docker Image from DEV to QA + # environment is only set, so that an approval process can be defined + environment: # complete with Azure Dev Ops environment + strategy: + runOnce: + deploy: + steps: + - template: build.yml + parameters: + ACR_PREVIOUS: # complete with the identifier of your azure container registry on DEV + ACR_NEXT: # complete with the identifier of your azure container registry on QA + ACR_SUB_PREVIOUS: # complete with name of docker registry service connection on DEV + ACR_SUB_NEXT: # complete with name of docker registry service connection on QA + REPOSITORY: # complete with name of repository + SUBSCRIPTION: # complete with name of service connection + IMAGE_COUNT: 5 + +- stage: prod_build + dependsOn: qa_build + displayName: Copy Docker Image from QA to PROD + jobs: + - deployment: qa_to_prod + displayName: Copy Docker Image from QA to PROD + # environment is only set, so that an approval process can be defined + environment: # complete with Azure Dev Ops environment + strategy: + runOnce: + deploy: + steps: + - template: build.yml + parameters: + ACR_PREVIOUS: # complete with the identifier of your azure container registry on QA + ACR_NEXT: # complete with the identifier of your azure container registry on PROD + ACR_SUB_PREVIOUS: # complete with name of docker registry service connection on QA + ACR_SUB_NEXT: # complete with name of docker registry service connection on PROD + REPOSITORY: # complete with name of repository + SUBSCRIPTION: # complete with name of service connection + IMAGE_COUNT: 5 diff --git a/{{cookiecutter.project_slug}}/ci/test-pipeline.yml b/{{cookiecutter.project_slug}}/ci/test-pipeline.yml new file mode 100644 index 0000000..ba59b27 --- /dev/null +++ b/{{cookiecutter.project_slug}}/ci/test-pipeline.yml @@ -0,0 +1,201 @@ +# trigger on pull requests needs to be set manually in Azure DevOps +trigger: none + +pool: + name: Azure Pipelines + vmImage: ubuntu-latest + +jobs: + +- job: Test + variables: + VENV_FOLDER: $(Pipeline.Workspace)/venv + {%- if cookiecutter.package_manager == 'pip' %} + PIP_CACHE_DIR: $(Pipeline.Workspace)/venv/lib + {%- elif cookiecutter.package_manager == 'conda' %} + CONDA_ENV_NAME: test-env + CONDA_PKGS_DIRS: /usr/share/miniconda/envs/$(CONDA_ENV_NAME) + {%- elif cookiecutter.package_manager == 'poetry' %} + POETRY_VERSION: 1.6 + # we have to cache the whole folder in order to activate the env later on + # otherwise the activate binary isn't restored for a cache hit + POETRY_CACHE_DIR: $(System.DefaultWorkingDirectory)/.venv + {%- endif %} + PACKAGE_NAME: '{{ cookiecutter.project_slug }}' + + steps: + - checkout: self + + - task: UsePythonVersion@0 + displayName: Use Python 3.8 + inputs: + versionSpec: 3.8 + addToPath: true + + - task: Bash@3 + displayName: Install system dependencies + inputs: + targetType: inline + script: | + set -uex + sudo apt update + sudo apt install -y build-essential + + # fill in or delete if no Azure Keyvault is used + - task: AzureKeyVault@1 + inputs: + azureSubscription: '' + KeyVaultName: '' + SecretsFilter: '' + RunAsPreJob: false + + {%- if cookiecutter.package_manager == 'pip' %} + + - task: Cache@2 + inputs: + key: 'pip | venv | $(Agent.OS) | requirements.txt | requirements-dev.txt' + path: $(PIP_CACHE_DIR) + displayName: Cache pip packages + + - task: Bash@3 + displayName: Create venv + inputs: + targetType: inline + script: | + set -uex + python -m venv $(VENV_FOLDER) + source $(VENV_FOLDER)/bin/activate + + - task: Bash@3 + displayName: Setup environment + inputs: + targetType: inline + script: | + set -uex + source $(VENV_FOLDER)/bin/activate + # resolve issues with old cached versions of pip + # Open question: Still needed? + python -m pip install --upgrade pip || (curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python get-pip.py) + python -m pip install wheel + + # install python app dependencies + python -m pip install -r requirements.txt -r requirements-dev.txt -U + + # build and install wheel + python setup.py dist + python -m pip install --force-reinstall dist/*.whl + + - task: Bash@3 + displayName: pytest (with coverage) + # define env variables as needed + env: + PACKAGE_NAME: $(PACKAGE_NAME) + inputs: + targetType: inline + script: | + set -uex + source $(VENV_FOLDER)/bin/activate + + # run pytest + python -m pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=$PACKAGE_NAME --cov-report=xml:coverage-reports/cov.xml --cov-report=html + {% elif cookiecutter.package_manager == 'conda' %} + + - task: Cache@2 + displayName: Cache conda packages + inputs: + key: 'conda | $(Agent.OS) | environment.yml | environment-dev.yml' + path: $(CONDA_PKGS_DIRS) + cacheHitVar: CONDA_CACHE_RESTORED + + - task: Bash@3 + displayName: Create conda environment + inputs: + targetType: inline + script: | + set -uex + conda env create -n $(CONDA_ENV_NAME) -f environment-dev.yml environment.yml + source /usr/share/miniconda/etc/profile.d/conda.sh + conda activate $(CONDA_ENV_NAME) + python -m pip install wheel + condition: eq(variables.CONDA_CACHE_RESTORED, 'false') + + - task: Bash@3 + displayName: Install project as python package + inputs: + targetType: inline + script: | + set -uex + source /usr/share/miniconda/etc/profile.d/conda.sh + conda activate $(CONDA_ENV_NAME) + python setup.py dist + python -m pip install --force-reinstall dist/*.whl + + # run tests with coverage information + - task: Bash@3 + displayName: pytest (with coverage) + # define env variables as needed + env: + PACKAGE_NAME: $(PACKAGE_NAME) + inputs: + targetType: inline + script: | + set -uex + source /usr/share/miniconda/etc/profile.d/conda.sh + conda activate $(CONDA_ENV_NAME) + python -m pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=$PACKAGE_NAME --cov-report=xml:coverage-reports/cov.xml --cov-report=html + {% elif cookiecutter.package_manager == 'poetry' %} + + - task: Cache@2 + displayName: Cache poetry packages + inputs: + key: 'poetry | $(Agent.OS) | pyproject.toml ' + path: $(POETRY_CACHE_DIR) + cacheHitVar: POETRY_CACHE_RESTORED + + - task: Bash@3 + displayName: Configure poetry + inputs: + targetType: inline + script: | + set -uex + pip install poetry==$(POETRY_VERSION) + + - task: Bash@3 + displayName: Create venv + inputs: + targetType: inline + script: | + set -uex + poetry install --no-root + source `poetry env info --path`/bin/activate + condition: eq(variables.POETRY_CACHE_RESTORED, 'false') + + # run tests with coverage information + - task: Bash@3 + displayName: pytest (with coverage) + inputs: + targetType: inline + script: | + set -uex + # install root which obviously shouldn't be cached + poetry install --only-root + source `poetry env info --path`/bin/activate + python -m pytest tests --doctest-modules --junitxml=$(System.DefaultWorkingDirectory)/junit/test-results.xml --cov=src/'{{ cookiecutter.module_name }}' --cov-report=xml:$(System.DefaultWorkingDirectory)/coverage-reports/cov.xml --cov-report=html:$(System.DefaultWorkingDirectory)/coverage-reports/cov.html + + {%- endif %} + - task: PublishCodeCoverageResults@1 + displayName: Publish code coverage + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: $(System.DefaultWorkingDirectory)/coverage-reports/cov.xml + additionalCodeCoverageFiles: $(System.DefaultWorkingDirectory)/junit/ + condition: succeededOrFailed() + + # publish pytest results + - task: PublishTestResults@2 + inputs: + testResultsFormat: JUnit + testResultsFiles: $(System.DefaultWorkingDirectory)/**/*-results.xml + testRunTitle: Publish pytest results + failTaskOnFailedTests: false + condition: succeededOrFailed() diff --git a/{{cookiecutter.project_slug}}/docker-compose.yml b/{{cookiecutter.project_slug}}/docker-compose.yml index a55f316..d7c7871 100644 --- a/{{cookiecutter.project_slug}}/docker-compose.yml +++ b/{{cookiecutter.project_slug}}/docker-compose.yml @@ -6,6 +6,6 @@ services: build: context: .{% if cookiecutter.package_manager != 'poetry' %} args: - PYTHON_IMAGE_TAG: {% if cookiecutter.package_manager == 'conda' %}"4.8.2"{% else %}"3.7-stretch"{% endif %}{% endif %} + PYTHON_IMAGE_TAG: {% if cookiecutter.package_manager == 'conda' %}"4.8.2"{% else %}"3.8-stretch"{% endif %}{% endif %} command: '{% if cookiecutter.create_cli == 'yes' %}--help{% endif %}' tty: true diff --git a/{{cookiecutter.project_slug}}/pyproject.toml b/{{cookiecutter.project_slug}}/pyproject.toml index 93582ad..b03fe76 100644 --- a/{{cookiecutter.project_slug}}/pyproject.toml +++ b/{{cookiecutter.project_slug}}/pyproject.toml @@ -4,20 +4,17 @@ version = "0.1.0" description = "{{ cookiecutter.project_short_description }}" authors = ["{{ cookiecutter.full_name }} <{{ cookiecutter.email }}>"] license = "Proprietary" -packages = [ - { include = "{{ cookiecutter.module_name }}", from = "src" }, -] +packages = [{ include = "{{ cookiecutter.module_name }}", from = "src" }, ] include = ["src/{{ cookiecutter.module_name }}/res/*"] {% if cookiecutter.create_cli == "yes" %} [tool.poetry.scripts] -{{ cookiecutter.project_slug }} = "{{ cookiecutter.module_name }}.main:app" -{% endif %} +{{ cookiecutter.project_slug }} = "{{ cookiecutter.module_name }}.main:app" {% endif %} + [tool.poetry.dependencies] python = "^3.10"{% if cookiecutter.config_file == 'hocon' %} pyhocon = "^0.3.59"{% elif cookiecutter.config_file == 'yaml' %} PyYAML = "^6.0"{% endif %}{% if cookiecutter.create_cli == 'yes' %} typer = {extras = ["all"], version = "^0.7.0"}{% endif %} -importlib-metadata = {version = "^1.0", python = "<3.8"} [tool.poetry.dev-dependencies]{% if cookiecutter.code_formatter == 'black' %} black = "^22.10"{% endif %} diff --git a/{{cookiecutter.project_slug}}/setup.py b/{{cookiecutter.project_slug}}/setup.py index ade8190..8ff3637 100644 --- a/{{cookiecutter.project_slug}}/setup.py +++ b/{{cookiecutter.project_slug}}/setup.py @@ -35,5 +35,5 @@ def read(fname): 'pre-commit', ], platforms='any', - python_requires='>=3.7', + python_requires='>=3.8', ) diff --git a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/__main__.py b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/__main__.py index e692541..1828e2a 100644 --- a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/__main__.py +++ b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/__main__.py @@ -1,3 +1,4 @@ {% if cookiecutter.create_cli == 'yes' %}from {{ cookiecutter.module_name }}.main import app + app(){% else %}from {{ cookiecutter.module_name }}.main import main main(){% endif %} diff --git a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/main__cli.py b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/main__cli.py index b184458..755a70e 100644 --- a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/main__cli.py +++ b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/main__cli.py @@ -2,15 +2,14 @@ import typer -from {{ cookiecutter.module_name }} import __title__ -from {{ cookiecutter.module_name }} import __version__{% if cookiecutter.config_file != 'none' %} -from {{ cookiecutter.module_name }} import util{% endif %} +from {{ cookiecutter.module_name }} import __title__ , __version__{% if cookiecutter.config_file != 'none' %}, util{% endif %} logger = logging.getLogger('{{ cookiecutter.module_name }}') app = typer.Typer( name='{{ cookiecutter.module_name }}', - help="{{ cookiecutter.project_short_description }}") + help="{{ cookiecutter.project_short_description }}" +) def version_callback(version: bool): @@ -20,10 +19,22 @@ def version_callback(version: bool): ConfigOption = typer.Option( - {% if cookiecutter.config_file == 'yaml' %}...{% else %}None{% endif %}, '-c', '--config', metavar='PATH', help="path to the program configuration") + {% if cookiecutter.config_file == 'yaml' %}...{% else %}None{% endif %}, + '-c', + '--config', + metavar='PATH', + help="path to the program configuration" +) + + VersionOption = typer.Option( - None, '-v', '--version', callback=version_callback, is_eager=True, - help="print the program version and exit") + None, + '-v', + '--version', + callback=version_callback, + is_eager=True, + help="print the program version and exit" +) @app.command() diff --git a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/util__yaml.py b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/util__yaml.py index ad44e50..78b62e8 100644 --- a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/util__yaml.py +++ b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/util__yaml.py @@ -13,7 +13,7 @@ def get_resource_string(path: str, decode=True) -> Union[str, bytes]: Load a package resource (i.e. a file from within this package) :param path: the path, starting at the root of the current module (e.g. 'res/default.conf'). - must be a string, not a Path object! + must be a string, not a Path object! :param decode: if true, decode the file contents as string (otherwise return bytes) :return: the contents of the resource file (as string or bytes) """ diff --git a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/version.py b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/version.py index c207133..c5441b0 100644 --- a/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/version.py +++ b/{{cookiecutter.project_slug}}/src/{{cookiecutter.module_name}}/version.py @@ -1,7 +1,4 @@ -{% if cookiecutter.package_manager == 'poetry' %}try: - import importlib.metadata as importlib_metadata -except ModuleNotFoundError: - import importlib_metadata +{% if cookiecutter.package_manager == 'poetry' %}from importlib.metadata import version -__version__ = importlib_metadata.version("{{ cookiecutter.module_name }}") +__version__ = version("{{ cookiecutter.project_slug }}") {%- else %}__version__ = '0.1.0'{% endif %}