diff --git a/.circle/Makefile b/.circle/Makefile index 428105c0..31b41421 100644 --- a/.circle/Makefile +++ b/.circle/Makefile @@ -1,267 +1,33 @@ ROOT_DIR ?= $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) -ifndef FORCE_CHECK_ALL_FILES -CHANGED_FILES := $(shell $(CI_DIR)/utils/git-changes files) -CHANGED_PY := $(shell ${CI_DIR}/utils/git-changes py) -CHANGED_YAML := $(shell $(CI_DIR)/utils/git-changes yaml) -CHANGED_JSON := $(shell $(CI_DIR)/utils/git-changes json) -endif -CHANGED_DIRECTORIES := $(shell $(CI_DIR)/utils/git-changes directories) VIRTUALENV_DIR ?= virtualenv ST2_REPO_PATH ?= /tmp/st2 ST2_REPO_BRANCH ?= master -FORCE_CHECK_ALL_FILES ?= false -FORCE_CHECK_PACK ?= false -export ST2_REPO_PATH ROOT_DIR FORCE_CHECK_ALL_FILES FORCE_CHECK_PACK +export ST2_REPO_PATH ST2_REPO_BRANCH ROOT_DIR -# All components are prefixed by st2 -COMPONENTS := $(wildcard /tmp/st2/st2*) -COMPONENTS_RUNNERS := $(wildcard /tmp/st2/contrib/runners/*) - -.PHONY: all -all: requirements lint packs-resource-register packs-tests - -.PHONY: all-ci -all-ci: compile .license-check .flake8 .pylint .copy-pack-to-subdirectory .configs-check .metadata-check .packs-resource-register .packs-tests - -.PHONY: lint -lint: requirements flake8 pylint configs-check metadata-check - -.PHONY: flake8 -flake8: requirements .flake8 - -.PHONY: pylint -pylint: requirements .clone_st2_repo .pylint - -.PHONY: configs-check -configs-check: requirements .clone_st2_repo .copy-pack-to-subdirectory .configs-check - -.PHONY: metadata-check -metadata-check: requirements .metadata-check - -# Task which copies pack to temporary sub-directory so we can use old-style check scripts which -# # require pack to be in a sub-directory -.PHONY: .copy-pack-to-subdirectory -.copy-pack-to-subdirectory: - rm -rf /tmp/packs/$(PACK_NAME) - mkdir -p /tmp/packs/$(PACK_NAME) - cp -r ./* /tmp/packs/$(PACK_NAME) - -.PHONY: packs-resource-register -packs-resource-register: requirements .clone_st2_repo .copy-pack-to-subdirectory .packs-resource-register - -.PHONY: packs-missing-tests -packs-missing-tests: requirements .packs-missing-tests - -.PHONY: packs-tests -packs-tests: requirements .clone_st2_repo .packs-tests - -.PHONY: compile -compile: - @echo "======================= compile ========================" - @echo "------- Compile all .py files (syntax check test) ------" - if python -c 'import compileall,re; compileall.compile_dir(".", rx=re.compile(r"/virtualenv|virtualenv-osx|virtualenv-py3|.tox|.git|.venv-st2devbox"), quiet=True)' | grep .; then exit 1; else exit 0; fi - -.PHONY: .flake8 -.flake8: - @echo - @echo "==================== flake8 ====================" - @echo - . $(VIRTUALENV_DIR)/bin/activate; \ - if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \ - find ./* -name "*.py" | while read py_file; do \ - flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$py_file || exit 1; \ - done; \ - elif [ -n "${CHANGED_PY}" ]; then \ - for file in ${CHANGED_PY}; do \ - if [ -n "$$file" ]; then \ - flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$file || exit 1; \ - fi; \ - done; \ - else \ - echo "No files have changed, skipping run..."; \ - fi; - -.PHONY: .pylint -.pylint: - @echo - @echo "==================== pylint ====================" - @echo - . $(VIRTUALENV_DIR)/bin/activate; \ - if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ -n "${CHANGED_PY}" ]; then \ - REQUIREMENTS_DIR=$(CI_DIR)/.circle/ \ - CONFIG_DIR=$(CI_DIR)/lint-configs/ \ - st2-check-pylint-pack $(ROOT_DIR) || exit 1; \ - else \ - echo "No files have changed, skipping run..."; \ - fi; - -.PHONY: .configs-check -.configs-check: - @echo - @echo "==================== configs-check ====================" - @echo - @# The number of changed files in the AWS pack exceeds the limits of Bash, - @# leading to CI failures like this: - @# https://circleci.com/gh/StackStorm-Exchange/stackstorm-aws/320 - @# Instead of passing the entire list into a Bash for loop, we convert the - @# make variable to a Bash string, convert that to a Bash array, and then - @# iterate through each element of the array - . $(VIRTUALENV_DIR)/bin/activate; \ - if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \ - find $(CI_DIR)/* -name "*.yaml" -o -name "*.yml" | while read yaml_file; do \ - st2-check-validate-yaml-file "$$yaml_file" || exit 1 ; \ - done; \ - elif [ -n "${CHANGED_YAML}" ]; then \ - for file in $(CHANGED_YAML); do \ - if [ -n "$$file" ]; then \ - st2-check-validate-yaml-file $$file || exit 1 ; \ - fi; \ - done; \ - else \ - echo "No files have changed, skipping run..."; \ - fi - @# - . $(VIRTUALENV_DIR)/bin/activate; \ - if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \ - find $(CI_DIR)/* -name "*.json" | while read json_file; do \ - st2-check-validate-json-file "$$json_file" || exit 1 ; \ - done; \ - elif [ -n "${CHANGED_JSON}" ]; then \ - for file in $(CHANGED_JSON); do \ - if [ -n "$$file" ]; then \ - echo "file: $$file"; \ - st2-check-validate-json-file $$file || exit 1 ; \ - fi; \ - done; \ - else \ - echo "No files have changed, skipping run..."; \ - fi - @# - @echo - @echo "==================== example config check ====================" - @echo - . $(VIRTUALENV_DIR)/bin/activate; \ - if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ -n "${CHANGED_FILES}" ]; then \ - st2-check-validate-pack-example-config /tmp/packs/$(PACK_NAME) || exit 1; \ - else \ - echo "No files have changed, skipping run..."; \ - fi; - -.PHONY: .metadata-check -.metadata-check: - @echo - @echo "==================== metadata-check ====================" - @echo - . $(VIRTUALENV_DIR)/bin/activate; \ - if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ -n "${CHANGED_YAML}" ]; then \ - st2-check-validate-pack-metadata-exists $(ROOT_DIR) || exit 1; \ - else \ - echo "No files have changed, skipping run..."; \ - fi; - -.PHONY: .packs-resource-register -.packs-resource-register: - @echo - @echo "==================== packs-resource-register ====================" - @echo - . $(VIRTUALENV_DIR)/bin/activate; \ - if [ -z "${CHANGED_FILES}" ]; then \ - echo "No files have changed, skipping run..."; \ - else \ - st2-check-register-pack-resources /tmp/packs/$(PACK_NAME) || exit 1; \ - fi; - -.PHONY: .packs-tests -.packs-tests: - @echo - @echo "==================== packs-tests ====================" - @echo - . $(VIRTUALENV_DIR)/bin/activate; \ - if [ -z "${CHANGED_FILES}" ]; then \ - echo "No files have changed, skipping run..."; \ - else \ - $(ST2_REPO_PATH)/st2common/bin/st2-run-pack-tests -c -t -x -j -p $(ROOT_DIR) || exit 1; \ - fi; - -.PHONY: .packs-missing-tests -.packs-missing-tests: - @echo - @echo "==================== pack-missing-tests ====================" - @echo - if [ -z "${CHANGED_FILES}" ]; then \ - echo "No files have changed, skipping run..."; \ - else \ - st2-check-print-pack-tests-coverage $(ROOT_DIR) || exit 1; \ - fi; - -# Target which veries repo root contains LICENSE file with ASF 2.0 content -.PHONY: .license-check -.license-check: - @echo - @echo "==================== license-check ====================" - @echo - if [ -z "${CHANGED_FILES}" ] && [ "$${FORCE_CHECK_ALL_FILES}" = "false" ]; then \ - echo "No files have changed, skipping run..."; \ - else \ - if [ ! -f "$(ROOT_DIR)/LICENSE" ]; then \ - echo "Missing LICENSE file in $(ROOT_DIR)"; \ - exit 2;\ - fi;\ - cat $(ROOT_DIR)/LICENSE | grep -q "Apache License" || (echo "LICENSE file doesn't contain Apache 2.0 license text" ; exit 2); \ - cat $(ROOT_DIR)/LICENSE | grep -q "Version 2.0" || (echo "LICENSE file doesn't contain Apache 2.0 license text" ; exit 2); \ - cat $(ROOT_DIR)/LICENSE | grep -q "www.apache.org/licenses/LICENSE-2.0" || (echo "LICENSE file doesn't contain Apache 2.0 license text" ; exit 2); \ - fi; - -.PHONY: .clone_st2_repo -.clone_st2_repo: /tmp/st2 -/tmp/st2: - @echo - @echo "==================== cloning st2 repo ====================" - @echo - @rm -rf /tmp/st2 - @git clone https://github.com/StackStorm/st2.git --depth 1 --single-branch --branch $(ST2_REPO_BRANCH) /tmp/st2 - -.PHONY: .install-runners -.install-runners: - @echo "" - @echo "================== install runners ====================" - @echo "" - @for component in $(COMPONENTS_RUNNERS); do \ - echo "==========================================================="; \ - echo "Installing runner:" $$component; \ - echo "==========================================================="; \ - (. $(VIRTUALENV_DIR)/bin/activate; cd $$component; python setup.py develop); \ - done - @echo "" - @echo "================== register metrics drivers ======================" - @echo "" - - # Install st2common to register metrics drivers - (. $(VIRTUALENV_DIR)/bin/activate; cd $(ST2_REPO_PATH)/st2common; python setup.py develop) - -.PHONY: requirements -requirements: virtualenv .clone_st2_repo .install-runners - @echo - @echo "==================== requirements ====================" - @echo - . $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --upgrade "pip>=9.0,<9.1" - . $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(CI_DIR)/.circle/requirements-dev.txt - . $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(CI_DIR)/.circle/requirements-pack-tests.txt - -.PHONY: requirements-ci -requirements-ci: - @echo - @echo "==================== requirements-ci ====================" - @echo - . $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --upgrade "pip>=9.0,<9.1" - . $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(CI_DIR)/.circle/requirements-dev.txt - . $(VIRTUALENV_DIR)/bin/activate && $(VIRTUALENV_DIR)/bin/pip install --cache-dir $(HOME)/.pip-cache -q -r $(CI_DIR)/.circle/requirements-pack-tests.txt - -.PHONY: virtualenv virtualenv: $(VIRTUALENV_DIR)/bin/activate $(VIRTUALENV_DIR)/bin/activate: @echo @echo "==================== virtualenv ====================" @echo test -d $(VIRTUALENV_DIR) || virtualenv --no-site-packages $(VIRTUALENV_DIR) + +$(VIRTUALENV_DIR)/bin/invoke: $(VIRTUALENV_DIR) + . $(VIRTUALENV_DIR)/bin/activate && pip install invoke + +.PHONY: invoke +invoke: virtualenv $(VIRTUALENV_DIR)/bin/invoke + $(VIRTUALENV_DIR)/bin/pip install invoke + +# https://stackoverflow.com/a/33018558 +# Workaround to support all previous make targets +# This default target simply passes all targets on to invoke +# We can't add invoke as a make dependency for the .DEFAULT target since the +# dependency will get overridden by whatever target is passed in +.DEFAULT: + @# Manually make virtualenv target + if [ ! -d $(VIRTUALENV_DIR) ]; then make virtualenv; fi + @# Manually make invoke target + if [ ! -e $(VIRTUALENV_DIR)/bin/invoke ]; then $(VIRTUALENV_DIR)/bin/pip install invoke; fi + . $(VIRTUALENV_DIR)/bin/activate && invoke --search-root=$(CI_DIR) $@ + @#. $(VIRTUALENV_DIR)/bin/activate && echo $$PYTHONPATH diff --git a/.circle/dependencies b/.circle/dependencies index 2b1fee90..ab9db1a8 100755 --- a/.circle/dependencies +++ b/.circle/dependencies @@ -49,14 +49,14 @@ if [ ! -z "${ROOT_DIR}" ]; then echo "Copying Makefile to ${ROOT_DIR}" cp ~/ci/.circle/Makefile ${ROOT_DIR} - make -C requirements-ci .install-runners + make -C requirements install-runners else PACK_REQUIREMENTS_FILE="$(pwd)/requirements.txt" PACK_TESTS_REQUIREMENTS_FILE="$(pwd)/requirements-tests.txt" echo "Copying Makefile to $(pwd)" cp ~/ci/.circle/Makefile . - make requirements-ci .install-runners + make requirements install-runners fi # Install pack requirements diff --git a/tasks/__init__.py b/tasks/__init__.py new file mode 100644 index 00000000..f060f582 --- /dev/null +++ b/tasks/__init__.py @@ -0,0 +1,97 @@ +from invoke import Collection, task, run + +from . import check +from . import copy +from . import git_tasks +from . import lint +from . import requirements +from . import tests + + +# All tasks are implemented in submodules of this package +# All tasks in this module are only for reverse compatibility with the original +# Makefile + +# This task aliases a Python built-in +@task(requirements.install, lint.lint, tests.packs_resource_register, tests.packs_tests) +def all_(ctx): + pass + + +@task(check.compile_, check.license, lint.flake8, lint.pylint, + copy.copy_pack_to_subdirectory, check.configs, check.metadata, + tests.packs_resource_register, tests.packs_tests) +def all_ci(ctx): + pass + + +@task(lint.lint) +def lint_(ctx): + pass + + +@task(lint.flake8) +def flake8(ctx): + pass + + +@task(lint.pylint) +def pylint(ctx): + pass + + +@task(check.configs) +def configs_check(ctx): + pass + + +@task(check.metadata) +def metadata_check(ctx): + pass + + +@task(tests.packs_resource_register) +def packs_resource_register(ctx): + pass + + +@task(tests.packs_missing_tests) +def packs_missing_tests(ctx): + pass + + +@task(tests.packs_tests) +def packs_tests(ctx): + pass + + +@task(check.compile_) +def compile_(ctx): + pass + + +@task(requirements.runners) +def install_runners(ctx): + pass + + +@task(requirements.install) +def requirements(ctx): + pass + + +namespace = Collection() + +namespace.add_task(all_, name='all') +namespace.add_task(all_ci, name='all-ci') +namespace.add_task(lint_, name='lint') +namespace.add_task(flake8, name='flake8') +namespace.add_task(pylint, name='pylint') +namespace.add_task(configs_check, name='configs-check') +namespace.add_task(metadata_check, name='metadata-check') +namespace.add_task(packs_resource_register, name='packs-resource-register') +namespace.add_task(packs_missing_tests, name='packs-missing-tests') +namespace.add_task(packs_tests, name='packs-tests') +namespace.add_task(compile_, name='compile') +namespace.add_task(install_runners, name='install-runners') +namespace.add_task(requirements, name='requirements') diff --git a/tasks/check.py b/tasks/check.py new file mode 100644 index 00000000..b2575748 --- /dev/null +++ b/tasks/check.py @@ -0,0 +1,132 @@ +import compileall +import glob +import os +import re + +from invoke import run, task + +from . import copy + + +@task +def compile_(ctx): + print("======================= compile ========================") + print("------- Compile all .py files (syntax check test) ------") + rgx = re.compile(r"/virtualenv|virtualenv-osx|virtualenv-py3|.tox|.git|.venv-st2devbox") + if not compileall.compile_dir(".", rx=rgx, quiet=True): + raise Exception("Could not compile all files") + + +@task +def license(ctx): + # Verifies repo contains LICENSE file with ASF 2.0 content + print("") + print("==================== license-check ====================") + print("") + apache_license_rgx = re.compile(r'.*Apache License.*') + license_version_rgx = re.compile(r'.*Version 2\.0.*') + apache_url_rgx = re.compile(r'.*www\.apache\.org/licenses/LICENSE-2\.0.*') + files = [] + top_level_git_dir = ctx.run('git rev-parse --show-toplevel').stdout.splitlines()[0] + base_branch = os.environ.get('BASE_BRANCH', 'origin/master') + license_file = os.path.join(top_level_git_dir, 'LICENSE') + with ctx.cd(top_level_git_dir): + files = ctx.run("git diff --relative --diff-filter=ACMRTUXB " + "--name-only {}".format(base_branch)).stdout.splitlines() + if os.environ.get('FORCE_CHECK_ALL_FILES', False) == 'true' or files: + # Check for the existence of a LICENSE file + if not os.path.exists(license_file): + raise Exception("Missing LICENSE file in {root_dir}".format(root_dir=top_level_git_dir)) + + with open(license_file) as f: + lines = f.read().splitlines() + + found_apache_license = False + found_license_version = False + found_apache_url = False + for line in lines: + if apache_license_rgx.match(line): + found_apache_license = True + if license_version_rgx.match(line): + found_license_version = True + if apache_url_rgx.match(line): + found_apache_url = True + if found_apache_license and found_license_version and found_apache_url: + break + else: + raise Exception("LICENSE file doesn't contain Apache 2.0 license text") + else: + print("No files have changed, skipping run...") + + +@task(copy.copy_pack_to_subdirectory) +def configs(ctx): + print("") + print("==================== configs-check ====================") + print("") + + # + # YAML FILES + # + yaml_files = [] + top_level_git_dir = ctx.run('git rev-parse --show-toplevel').stdout.splitlines()[0] + base_branch = os.environ.get('BASE_BRANCH', 'origin/master') + with ctx.cd(top_level_git_dir): + yaml_files = ctx.run("git diff --relative --diff-filter=ACMRTUXB " + "--name-only {} -- '*.yaml' '*.yml'".format(base_branch)).stdout.splitlines() + if os.environ.get('FORCE_CHECK_ALL_FILES', False) == 'true': + for yaml_file in glob.glob("**.yaml", recursive=True) + glob.glob("**.yml", recursive=True): + ctx.run("st2-check-validate-yaml-file {yaml_file}".format(yaml_file=yaml_file)) + elif yaml_files: + for yaml_file in yaml_files: + ctx.run("st2-check-validate-yaml-file {yaml_file}".format(yaml_file=yaml_file)) + else: + print("No files have changed, skipping run...") + + # + # JSON FILES + # + json_files = [] + with ctx.cd(top_level_git_dir): + json_files = ctx.run("git diff --relative --diff-filter=ACMRTUXB " + "--name-only {} -- '*.json'".format(base_branch)).stdout.splitlines() + if os.environ.get('FORCE_CHECK_ALL_FILES', False) == 'true': + for json_file in glob.glob("**.json", recursive=True): + ctx.run("st2-check-validate-json-file {json_file}".format(json_file=json_file)) + elif json_files: + for json_file in json_files: + ctx.run("st2-check-validate-json-file {json_file}".format(json_file=json_file)) + else: + print("No files have changed, skipping run...") + + # + # EXAMPLE CONFIG CHECK + # + print("") + print("==================== example config check ====================") + print("") + files = [] + with ctx.cd(top_level_git_dir): + files = ctx.run("git diff --relative --diff-filter=ACMRTUXB " + "--name-only {}".format(base_branch)).stdout.splitlines() + if os.environ.get('FORCE_CHECK_ALL_FILES', False) == 'true' or files: + ctx.run("st2-check-validate-pack-example-config " + "/tmp/packs/{pack_name}".format(pack_name=os.environ['PACK_NAME'])) + else: + print("No files have changed, skipping run...") + + +@task +def metadata(ctx): + print("") + print("==================== metadata-check ====================") + print("") + top_level_git_dir = ctx.run('git rev-parse --show-toplevel').stdout.splitlines()[0] + base_branch = os.environ.get('BASE_BRANCH', 'origin/master') + files = [] + files = ctx.run("git diff --relative --diff-filter=ACMRTUXB " + "--name-only {}".format(base_branch)).stdout.splitlines() + if os.environ.get('FORCE_CHECK_ALL_FILES', False) == 'true' or files: + ctx.run("st2-check-validate-pack-metadata-exists {pack_dir}".format(pack_dir=top_level_git_dir)) + else: + print("No files have changed, skipping run...") diff --git a/tasks/copy.py b/tasks/copy.py new file mode 100644 index 00000000..d59b4022 --- /dev/null +++ b/tasks/copy.py @@ -0,0 +1,13 @@ +import os +import shutil + +from invoke import run, task + + +@task +def copy_pack_to_subdirectory(ctx): + pack_name = os.environ['PACK_NAME'] + shutil.rmtree('/tmp/packs/{pack_name}'.format(pack_name=pack_name), ignore_errors=True) + if not os.path.exists('/tmp/packs'): + os.mkdir('/tmp/packs') + shutil.copytree('.', '/tmp/packs/{pack_name}'.format(pack_name=pack_name)) diff --git a/tasks/git_tasks.py b/tasks/git_tasks.py new file mode 100644 index 00000000..049ef7dc --- /dev/null +++ b/tasks/git_tasks.py @@ -0,0 +1,25 @@ +import os + +from invoke import run, task + + +@task +def clone_st2_repo(recursive=False): + if not os.path.exists('/tmp/st2'): + try: + from git import Repo + except ImportError: + run("git clone https://github.com/StackStorm/st2.git " + "--depth 1 " + "--single-branch " + "--branch {st2_repo_branch} " + "{st2_repo_path}".format( + st2_repo_path=os.environ.get('ST2_REPO_PATH'), + st2_repo_branch=os.environ.get('ST2_REPO_BRANCH'))) + else: + Repo.clone_from( + "https://github.com/StackStorm/st2.git", + os.environ.get('ST2_REPO_PATH', '/tmp/st2'), + depth=1, + single_branch=True, + branch=os.environ.get('ST2_REPO_BRANCH')) diff --git a/tasks/lint.py b/tasks/lint.py new file mode 100644 index 00000000..b9be315a --- /dev/null +++ b/tasks/lint.py @@ -0,0 +1,61 @@ +import glob +import os + +from invoke import task, run + +from . import check +from . import git_tasks +from . import requirements + + +CI_DIR = os.environ.get('CI_DIR', '/home/circleci/ci') + + +@task(requirements.install) +def flake8(ctx): + print("") + print("==================== flake8 ====================") + print("") + top_level_git_dir = ctx.run('git rev-parse --show-toplevel').stdout.splitlines()[0] + base_branch = os.environ.get('BASE_BRANCH', 'origin/master') + files = [] + with ctx.cd(top_level_git_dir): + files = ctx.run("git diff --relative --diff-filter=ACMRTUXB " + "--name-only {} -- '*.py'".format(base_branch)).stdout.splitlines() + if os.environ.get('FORCE_CHECK_ALL_FILES', False) == 'true': + for file_ in glob.glob("**.py", recursive=True): + ctx.run("flake8 --config={flake8_config} {flake8_file}".format( + flake8_config=os.path.join(CI_DIR, 'lint-configs', 'python', '.flake8'), + flake8_file=file_)) + elif files: + for file_ in files: + ctx.run("flake8 --config={flake8_config} {flake8_file}".format( + flake8_config=os.path.join(CI_DIR, 'lint-configs', 'python', '.flake8'), + flake8_file=file_)) + else: + print("No files have changed, skipping run...") + + +@task(requirements.install, git_tasks.clone_st2_repo) +def pylint(ctx): + print("") + print("==================== pylint ====================") + print("") + top_level_git_dir = ctx.run('git rev-parse --show-toplevel').stdout.splitlines()[0] + base_branch = os.environ.get('BASE_BRANCH', 'origin/master') + files = [] + with ctx.cd(top_level_git_dir): + files = ctx.run("git diff --relative --diff-filter=ACMRTUXB " + "--name-only {} -- '*.py'".format(base_branch)).stdout.splitlines() + if os.environ.get('FORCE_CHECK_ALL_FILES', False) == 'true' or files: + ctx.run('st2-check-pylint-pack {pack_dir}'.format(pack_dir=top_level_git_dir), env={ + 'REQUIREMENTS_DIR': os.path.join(CI_DIR, '.circle'), + 'CONFIG_DIR': os.path.join(CI_DIR, 'lint-configs'), + }) + else: + print("No files have changed, skipping run...") + + +@task(requirements.install, flake8, pylint, check.configs, check.metadata) +def lint(ctx): + pass diff --git a/tasks/requirements.py b/tasks/requirements.py new file mode 100644 index 00000000..5117566a --- /dev/null +++ b/tasks/requirements.py @@ -0,0 +1,39 @@ +import glob +import os + +from invoke import task, run + +from . import git_tasks + + +@task +def runners(ctx): + print("") + print("================== install runners ====================") + print("") + for component in glob.glob("{}/contrib/runners/*".format(os.environ.get('ST2_REPO_PATH', '/tmp/st2'))): + print("===========================================================") + print("Installing runner: {component}".format(component=component)) + print("===========================================================") + with ctx.cd(component): + ctx.run("python setup.py develop") + print("") + print("================== register metrics drivers ======================") + print("") + + with ctx.cd(os.path.join(os.environ.get('ST2_REPO_PATH', '/tmp/st2'), 'st2common')): + ctx.run("python setup.py develop") + + +@task(git_tasks.clone_st2_repo, runners) +def install(ctx): + print("") + print("==================== requirements ====================") + print("") + ctx.run("pip install --upgrade \"pip>=9.0,<9.1\"") + home = os.path.expanduser("~") + ci_dir = os.path.join(home, 'ci') + ctx.run("pip install --cache-dir {home}/.pip-cache -q " + "-r {ci_dir}/.circle/requirements-dev.txt".format(home=home, ci_dir=ci_dir)) + ctx.run("pip install --cache-dir {home}/.pip-cache -q " + "-r {ci_dir}/.circle/requirements-pack-tests.txt".format(home=home, ci_dir=ci_dir)) diff --git a/tasks/tests.py b/tasks/tests.py new file mode 100644 index 00000000..08ee353d --- /dev/null +++ b/tasks/tests.py @@ -0,0 +1,61 @@ +import os + +from invoke import run, task + +from . import copy +from . import requirements +from . import git_tasks + + +@task(requirements.install, git_tasks.clone_st2_repo, copy.copy_pack_to_subdirectory) +def packs_resource_register(ctx): + print("") + print("==================== packs-resource-register ====================") + print("") + base_branch = os.environ.get('BASE_BRANCH', 'origin/master') + files = [] + top_level_git_dir = ctx.run('git rev-parse --show-toplevel').stdout.splitlines()[0] + with ctx.cd(top_level_git_dir): + files = ctx.run("git diff --relative --diff-filter=ACMRTUXB " + "--name-only {}".format(base_branch)).stdout.splitlines() + if files: + ctx.run("st2-check-register-pack-resources /tmp/packs/{pack_name}".format(pack_name=os.environ['PACK_NAME'])) + else: + print("No files have changed, skipping run...") + + +@task(requirements.install) +def packs_tests(ctx): + print("") + print("==================== packs-tests ====================") + print("") + base_branch = os.environ.get('BASE_BRANCH', 'origin/master') + files = [] + top_level_git_dir = ctx.run('git rev-parse --show-toplevel').stdout.splitlines()[0] + with ctx.cd(top_level_git_dir): + files = ctx.run("git diff --relative --diff-filter=ACMRTUXB " + "--name-only {}".format(base_branch)).stdout.splitlines() + if files: + ctx.run("{st2_repo_path}/st2common/bin/st2-run-pack-tests -c -t -x -j -p " + "{pack_dir}".format( + st2_repo_path=os.environ.get('ST2_REPO_PATH', '/tmp/st2'), + pack_dir=top_level_git_dir)) + else: + print("No files have changed, skipping run...") + + +@task(requirements.install, git_tasks.clone_st2_repo) +def packs_missing_tests(ctx): + print("") + print("==================== pack-missing-tests ====================") + print("") + base_branch = os.environ.get('BASE_BRANCH', 'origin/master') + files = [] + top_level_git_dir = ctx.run('git rev-parse --show-toplevel').stdout.splitlines()[0] + with ctx.cd(top_level_git_dir): + files = ctx.run("git diff --relative --diff-filter=ACMRTUXB " + "--name-only {}".format(base_branch)).stdout.splitlines() + if files: + ctx.run("st2-check-print-pack-tests-coverage {pack_dir}".format(pack_dir=top_level_git_dir)) + else: + print("No files have changed, skipping run...") diff --git a/tasks/utils.py b/tasks/utils.py new file mode 100644 index 00000000..37e2564d --- /dev/null +++ b/tasks/utils.py @@ -0,0 +1,15 @@ +import git + + +def git_changes(repo_path, from_branch, to_branch='origin/master'): + repo = git.Repo(repo_path) + from_branch_commit = repo.commit(from_branch) + to_branch_commit = repo.commit(to_branch) + diff_index = to_branch_commit.diff(from_branch_commit) + + return list(set( + [x.a_blob.path for x in diff_index.iter_change_type('A') if x.a_blob] + + [x.a_blob.path for x in diff_index.iter_change_type('R') if x.a_blob] + + [x.a_blob.path for x in diff_index.iter_change_type('M') if x.a_blob] + + [x.a_blob.path for x in diff_index.iter_change_type('T') if x.a_blob] + ))