Create basic repository structure

hakbailey · hakbailey · commit 304b30bd4ed4 · 2022-07-18T15:08:04.000-04:00
Why these changes are being introduced:
Since this is a template repository, we want it to have our standard
Python CLI app structure with all the usual tools already in place for
testing, linting/code quality, exception monitoring, logging,
Github Actions, etc.

How this addresses that need:
* Creates basic app folder with initial cli and config modules.
* Creates tests for initial cli and config functionality.
* Adds .github folder with PR template, dependabot configuration, and
  CI workflow.
* Adds Dockerfile set up for basic image creation.
* Adds Makefile with standard dependency management, testing, and
  linting commands.
* Adds Pipfile with initial package and dev dependencies for testing,
  linting, and exception monitoring.
* Adds setup.cfg with initial pytest and linter configurations.
* Adds README with standard dev instructions and an "App setup" section
  at the top with instructions for creating a new app from this
  template.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -0,0 +1,18 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+
+  # Maintain dependencies for npm
+  - package-ecosystem: "pipenv"
+    directory: "/"
+    schedule:
+      interval: "daily"
diff --git a/.github/pull-request-template.md b/.github/pull-request-template.md
@@ -0,0 +1,39 @@
+### What does this PR do?
+
+Describe the overall purpose of the PR changes. Doesn't need to be as specific as the
+individual commits.
+
+### Helpful background context
+
+Describe any additional context beyond what the PR accomplishes if it is likely to be
+useful to a reviewer.
+
+Delete this section if it isn't applicable to the PR.
+
+### How can a reviewer manually see the effects of these changes?
+
+Explain how to see the proposed changes in the application if possible.
+
+Delete this section if it isn't applicable to the PR.
+
+### Includes new or updated dependencies?
+
+YES | NO
+
+### What are the relevant tickets?
+
+Include links to Jira Software and/or Jira Service Management tickets here.
+
+### Developer
+
+- [ ] All new ENV is documented in README (or there is none)
+- [ ] Stakeholder approval has been confirmed (or is not needed)
+
+### Code Reviewer
+
+- [ ] The commit message is clear and follows our guidelines
+      (not just this pull request message)
+- [ ] There are appropriate tests covering any new functionality
+- [ ] The documentation has been updated or is unnecessary
+- [ ] The changes have been verified
+- [ ] New dependencies are appropriate or there were no changes
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,7 @@
+name: CI
+on: push
+jobs:
+  test:
+    uses: mitlibraries/.github/.github/workflows/python-shared-test.yml@main
+  lint:
+    uses: mitlibraries/.github/.github/workflows/python-shared-lint.yml@main
diff --git a/.python-version b/.python-version
@@ -0,0 +1 @@
+3.10.3
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,12 @@
+FROM python:3.10-slim as build
+WORKDIR /app
+COPY . .
+
+RUN pip install --no-cache-dir --upgrade pip pipenv
+
+RUN apt-get update && apt-get upgrade -y && apt-get install -y git
+
+COPY Pipfile* /
+RUN pipenv install
+
+ENTRYPOINT ["pipenv", "run", "app"]
diff --git a/Makefile b/Makefile
@@ -0,0 +1,40 @@
+SHELL=/bin/bash
+DATETIME:=$(shell date -u +%Y%m%dT%H%M%SZ)
+
+### Dependency commands ###
+
+install: ## Install dependencies and CLI app
+	pipenv install --dev
+
+update: install ## Update all Python dependencies
+	pipenv clean
+	pipenv update --dev
+
+### Test commands ###
+
+test: ## Run tests and print a coverage report
+	pipenv run coverage run --source=app -m pytest -vv
+	pipenv run coverage report -m
+
+coveralls: test
+	pipenv run coverage lcov -o ./coverage/lcov.info
+
+### Code quality and safety commands ###
+
+lint: bandit black mypy pylama safety ## Run linting, code quality, and safety checks
+
+bandit:
+	pipenv run bandit -r app
+
+black:
+	pipenv run black --check --diff .
+
+mypy:
+	pipenv run mypy app
+
+pylama:
+	pipenv run pylama --options setup.cfg
+
+safety:
+	pipenv check
+	pipenv verify
diff --git a/Pipfile b/Pipfile
@@ -0,0 +1,23 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+click = "*"
+sentry-sdk = "*"
+
+[dev-packages]
+bandit = "*"
+black = "*"
+coverage = "*"
+coveralls = "*"
+mypy = "*"
+pylama = {extras = ["all"], version = "*"}
+pytest = "*"
+
+[requires]
+python_version = "3.10"
+
+[scripts]
+app = "python -c \"from app.cli import main; main()\""
diff --git a/README.md b/README.md
@@ -0,0 +1,57 @@
+# python-cli-template
+
+A template repository for creating Python CLI applications.
+
+## App setup (delete this section and above after initial application setup)
+
+1. Rename "app" to the desired app name across the repo. (May be helpful to do a project-wide find-and-replace).
+2. Update Python version if needed.
+3. Install all dependencies to create initial Pipfile.lock with current versions.
+4. Add initial app description to README and update initial required ENV variable documentation as needed.
+5. Update license if needed (check app-specific dependencies for licensing terms).
+6. Check Github repository settings:
+   - Confirm repo branch protection settings are correct (see [dev docs](https://mitlibraries.github.io/guides/basics/github.html) for details)
+   - Confirm that all of the following are enabled in the repo's code security and analysis settings:
+      - Dependabot alerts
+      - Dependabot security updates
+      - Secret scanning
+7. Create a Sentry project for the app if needed (we want this for most apps):
+   - Send initial exceptions to Sentry project for dev, stage, and prod environments to create them.
+   - Create an alert for the prod environment only, with notifications sent to the appropriate team(s).
+   - If *not* using Sentry, delete Sentry configuration from config.py and test_config.py, and remove sentry_sdk from project dependencies.
+
+# app
+
+Description of the app
+
+## Development
+
+To install with dev dependencies:
+
+```
+make install
+```
+
+To run unit tests:
+
+```
+make test
+```
+
+To lint the repo:
+
+```
+make lint
+```
+
+To run the app:
+
+```
+pipenv run app --help
+```
+
+## Required ENV
+
+`SENTRY_DSN` = If set to a valid Sentry DSN, enables Sentry exception monitoring. This is not needed for local development.
+
+`WORKSPACE` = Set to `dev` for local development, this will be set to `stage` and `prod` in those environments by Terraform.
diff --git a/app/__init__.py b/app/__init__.py
@@ -0,0 +1 @@
+"""app package."""
diff --git a/app/cli.py b/app/cli.py
@@ -0,0 +1,28 @@
+import logging
+from datetime import timedelta
+from time import perf_counter
+
+import click
+
+from app.config import configure_logger, configure_sentry
+
+logger = logging.getLogger(__name__)
+
+
+@click.command()
+@click.option(
+    "-v", "--verbose", is_flag=True, help="Pass to log at debug level instead of info"
+)
+def main(verbose: bool) -> None:
+    start_time = perf_counter()
+    root_logger = logging.getLogger()
+    logger.info(configure_logger(root_logger, verbose))
+    logger.info(configure_sentry())
+    logger.info("Running process")
+
+    # Do things here!
+
+    elapsed_time = perf_counter() - start_time
+    logger.info(
+        "Total time to complete process: %s", str(timedelta(seconds=elapsed_time))
+    )
diff --git a/app/config.py b/app/config.py
@@ -0,0 +1,33 @@
+import logging
+import os
+
+import sentry_sdk
+
+
+def configure_logger(logger: logging.Logger, verbose: bool) -> str:
+    if verbose:
+        logging.basicConfig(
+            format="%(asctime)s %(levelname)s %(name)s.%(funcName)s() line %(lineno)d: "
+            "%(message)s"
+        )
+        logger.setLevel(logging.DEBUG)
+        for handler in logging.root.handlers:
+            handler.addFilter(logging.Filter("app"))
+    else:
+        logging.basicConfig(
+            format="%(asctime)s %(levelname)s %(name)s.%(funcName)s(): %(message)s"
+        )
+        logger.setLevel(logging.INFO)
+    return (
+        f"Logger '{logger.name}' configured with level="
+        f"{logging.getLevelName(logger.getEffectiveLevel())}"
+    )
+
+
+def configure_sentry() -> str:
+    env = os.getenv("WORKSPACE")
+    sentry_dsn = os.getenv("SENTRY_DSN")
+    if sentry_dsn and sentry_dsn.lower() != "none":
+        sentry_sdk.init(sentry_dsn, environment=env)
+        return f"Sentry DSN found, exceptions will be sent to Sentry with env={env}"
+    return "No Sentry DSN found, exceptions will not be sent to Sentry"
diff --git a/setup.cfg b/setup.cfg
@@ -0,0 +1,16 @@
+[mypy]
+disallow_incomplete_defs = True
+disallow_untyped_calls = True
+disallow_untyped_defs = True
+
+[mypy-sentry_sdk.*]
+ignore_missing_imports = True
+
+[pylama]
+ignore = C0114,C0116,D100,D103,W0012
+linters = eradicate,isort,mccabe,pycodestyle,pydocstyle,pyflakes,pylint
+max_line_length = 90
+
+[tool:pytest]
+log_level = DEBUG
+
diff --git a/tests/__init__.py b/tests/__init__.py
@@ -0,0 +1 @@
+"""test package."""
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -0,0 +1,15 @@
+import os
+
+import pytest
+from click.testing import CliRunner
+
+
+@pytest.fixture(autouse=True)
+def test_env():
+    os.environ = {"SENTRY_DSN": None, "WORKSPACE": "test"}
+    yield
+
+
+@pytest.fixture()
+def runner():
+    return CliRunner()
diff --git a/tests/test_cli.py b/tests/test_cli.py
@@ -0,0 +1,17 @@
+from app.cli import main
+
+
+def test_cli_no_options(caplog, runner):
+    result = runner.invoke(main)
+    assert result.exit_code == 0
+    assert "Logger 'root' configured with level=INFO" in caplog.text
+    assert "Running process" in caplog.text
+    assert "Total time to complete process" in caplog.text
+
+
+def test_cli_all_options(caplog, runner):
+    result = runner.invoke(main, ["--verbose"])
+    assert result.exit_code == 0
+    assert "Logger 'root' configured with level=DEBUG" in caplog.text
+    assert "Running process" in caplog.text
+    assert "Total time to complete process" in caplog.text
diff --git a/tests/test_config.py b/tests/test_config.py
@@ -0,0 +1,35 @@
+import logging
+
+from app.config import configure_logger, configure_sentry
+
+
+def test_configure_logger_not_verbose():
+    logger = logging.getLogger(__name__)
+    result = configure_logger(logger, verbose=False)
+    assert logger.getEffectiveLevel() == 20
+    assert result == "Logger 'tests.test_config' configured with level=INFO"
+
+
+def test_configure_logger_verbose():
+    logger = logging.getLogger(__name__)
+    result = configure_logger(logger, verbose=True)
+    assert logger.getEffectiveLevel() == 10
+    assert result == "Logger 'tests.test_config' configured with level=DEBUG"
+
+
+def test_configure_sentry_no_env_variable(monkeypatch):
+    monkeypatch.delenv("SENTRY_DSN", raising=False)
+    result = configure_sentry()
+    assert result == "No Sentry DSN found, exceptions will not be sent to Sentry"
+
+
+def test_configure_sentry_env_variable_is_none(monkeypatch):
+    monkeypatch.setenv("SENTRY_DSN", "None")
+    result = configure_sentry()
+    assert result == "No Sentry DSN found, exceptions will not be sent to Sentry"
+
+
+def test_configure_sentry_env_variable_is_dsn(monkeypatch):
+    monkeypatch.setenv("SENTRY_DSN", "https://1234567890@00000.ingest.sentry.io/123456")
+    result = configure_sentry()
+    assert result == "Sentry DSN found, exceptions will be sent to Sentry with env=test"
