Merge branch 'release' into konflux

Author: ligangty

.github/workflows/linters.yaml

@@ -29,17 +29,6 @@ jobs:
       - name: Run flake8 on python${{ matrix.python-version }}
         run: python -m tox -e flake8
 
-  # markdownlint:
-  #  name: Markdownlint
-  #  runs-on: ubuntu-latest
-
-  #  steps:
-  #    - name: Check out repo
-  #      uses: actions/checkout@v2
-
-  #    - name: Run markdownlint
-  #      uses: containerbuildsystem/actions/markdownlint@master
-
   pylint:
     name: Pylint analyzer for Python ${{ matrix.python-version }}
     runs-on: ubuntu-latest
@@ -71,25 +60,25 @@ jobs:
       - name: Run ShellCheck
         uses: containerbuildsystem/actions/shellcheck@master
 
-#  mypy:
-#    name: mypy type checker for Python ${{ matrix.python-version }}
-#    runs-on: ubuntu-latest
-#
-#    strategy:
-#      matrix:
-#        python-version: [ "3.8" ]
-#
-#    steps:
-#      - uses: actions/checkout@v3
-#      - uses: actions/setup-python@v4
-#        with:
-#          python-version: ${{ matrix.python-version }}
-#      - name: Install dependencies
-#        run: |
-#          python -m pip install --upgrade pip setuptools tox
-#
-#      - name: Run mypy on python${{ matrix.python-version }}
-#        run: python -m tox -e mypy
+  mypy:
+    name: mypy type checker for Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: [ "3.9" ]
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools tox
+
+      - name: Run mypy on python${{ matrix.python-version }}
+        run: python -m tox -e mypy
 
   # bandit:
   #   name: Bandit analyzer for Python ${{ matrix.python-version }}

.github/workflows/unittests.yaml

@@ -31,7 +31,7 @@ jobs:
         run: python -m tox -e test
 
       - name: Upload pytest html report
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           path: __pytest_reports/charon-unit-tests.html

.pylintrc

@@ -24,8 +24,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-enable=all,
-       python3
+enable=all
 disable=I,
         R,  # refactoring checks
         abstract-method,  # nice to have
@@ -34,14 +33,14 @@ disable=I,
         bad-whitespace,  # pep8, nice to have
         broad-except,
         comprehension-escape, # throws false positives on 1.9.0 (Fedora 29)
-        deprecated-lambda,  # nice to have
+        ; deprecated-lambda,  # nice to have
         import-outside-toplevel,
         import-error,  # requires to having all modules installed
         invalid-name,
         len-as-condition,  # nice to have
         line-too-long,  # pep8, should be enabled
         missing-docstring,
-        misplaced-comparison-constant,
+        ; misplaced-comparison-constant,
         multiple-statements,  # nice to have
         no-absolute-import,  # it was required for py2
         no-init,
@@ -51,7 +50,7 @@ disable=I,
         old-division,  # it was required for py2
         protected-access,
         redefined-outer-name,  # nice to have
-        relative-import,  # nice to have
+        ; relative-import,  # nice to have
         super-init-not-called,  # nice to have
         superfluous-parens,  # nice to have
         too-many-lines,

Makefile

@@ -0,0 +1,38 @@
+init-venv:
+		@pip install virtualenv
+		@python -m virtualenv venv
+		@source venv/bin/activate
+		@pip install -r requirements.txt
+		@pip install tox
+.PHONY: init, init-venv
+
+lint:
+		@python -m tox -e flake8
+		@python -m tox -e pylint
+		@python -m tox -e mypy
+.PHONY: lint
+
+test-only:
+		@python -m tox -e test
+.PHONY: test-only
+
+test: lint test-only
+.PHONY: test
+
+clean:
+		rm -rf .coverage .tox .mypy_cache __pytest_reports htmlcov
+		rm -rf build charon.egg-info dist local package
+.PHONY: clean
+
+build:
+		@pip install -r ./requirements.txt
+		@pip install .
+.PHONY: build
+
+sdist:
+		@python3 setup.py sdist
+.PHONY: sdist
+
+image-latest:
+		@podman build . -f ./image/Containerfile -t localhost/charon:latest
+.PHONY: image-latest

charon.spec

@@ -1,7 +1,7 @@
 %global owner Commonjava
 %global modulename charon
 
-%global charon_version 1.2.2
+%global charon_version 1.3.3
 %global sdist_tar_name %{modulename}-%{charon_version}
 
 %global python3_pkgversion 3
@@ -23,33 +23,17 @@ BuildArch: x86_64
 BuildRequires: python%{python3_pkgversion}-setuptools
 BuildRequires: python%{python3_pkgversion}-devel
 
+Requires: python%{python3_pkgversion}-jinja2
 Requires: python%{python3_pkgversion}-boto3
 Requires: python%{python3_pkgversion}-botocore
-Requires: python%{python3_pkgversion}-jinja2
-Requires: python%{python3_pkgversion}-markupsafe
-Requires: python%{python3_pkgversion}-dateutil
-Requires: python%{python3_pkgversion}-six
-Requires: python%{python3_pkgversion}-jmespath
-Requires: python%{python3_pkgversion}-urllib3
-Requires: python%{python3_pkgversion}-s3transfer
 Requires: python%{python3_pkgversion}-click
 Requires: python%{python3_pkgversion}-requests
-Requires: python%{python3_pkgversion}-idna
-Requires: python%{python3_pkgversion}-chardet
-Requires: python%{python3_pkgversion}-cryptography
-Requires: python%{python3_pkgversion}-cffi
-Requires: python%{python3_pkgversion}-pycparser
-Requires: python%{python3_pkgversion}-certifi
-Requires: python%{python3_pkgversion}-pyOpenSSL
-Requires: python%{python3_pkgversion}-ruamel-yaml
+Requires: python%{python3_pkgversion}-pyyaml
 Requires: python%{python3_pkgversion}-defusedxml
-Requires: python%{python3_pkgversion}-semantic-version
 Requires: python%{python3_pkgversion}-subresource-integrity
 Requires: python%{python3_pkgversion}-jsonschema
-Requires: python%{python3_pkgversion}-importlib-metadata
-Requires: python%{python3_pkgversion}-zipp
-Requires: python%{python3_pkgversion}-attrs
-Requires: python%{python3_pkgversion}-pyrsistent
+Requires: python%{python3_pkgversion}-urllib3
+Requires: python%{python3_pkgversion}-semantic-version
 
 %description
 Simple Python tool with command line interface for charon init,

charon/cache.py

@@ -1,6 +1,6 @@
 from boto3 import session
 from botocore.exceptions import ClientError
-from typing import Dict, List
+from typing import Dict, List, Optional
 import os
 import logging
 import uuid
@@ -57,7 +57,7 @@ def __init_aws_client(
             endpoint_url=endpoint_url
         )
 
-    def __get_endpoint(self, extra_conf) -> str:
+    def __get_endpoint(self, extra_conf) -> Optional[str]:
         endpoint_url = os.getenv(ENDPOINT_ENV)
         if not endpoint_url or not endpoint_url.strip():
             if isinstance(extra_conf, Dict):
@@ -97,14 +97,14 @@ def invalidate_paths(
                     " will take more than %d seconds",
                     len(real_paths), total_time_approx)
         results = []
-        current_invalidation = {}
+        current_invalidation: Dict[str, str] = {}
         processed_count = 0
         for batch_paths in real_paths:
             while (current_invalidation and
                     INVALIDATION_STATUS_INPROGRESS == current_invalidation.get('Status', '')):
                 time.sleep(INPRO_W_SECS)
                 try:
-                    result = self.check_invalidation(distr_id, current_invalidation.get('Id'))
+                    result = self.check_invalidation(distr_id, current_invalidation.get('Id', ''))
                     if result:
                         current_invalidation = {
                             'Id': result.get('Id', None),
@@ -159,7 +159,7 @@ def invalidate_paths(
             results.append(current_invalidation)
         return results
 
-    def check_invalidation(self, distr_id: str, invalidation_id: str) -> dict:
+    def check_invalidation(self, distr_id: str, invalidation_id: str) -> Optional[dict]:
         try:
             response = self.__client.get_invalidation(
                 DistributionId=distr_id,
@@ -177,8 +177,9 @@ def check_invalidation(self, distr_id: str, invalidation_id: str) -> dict:
                 "[CloudFront] Error occurred while check invalidation of id %s, "
                 "error: %s", invalidation_id, err
             )
+        return None
 
-    def get_dist_id_by_domain(self, domain: str) -> str:
+    def get_dist_id_by_domain(self, domain: str) -> Optional[str]:
         """Get distribution id by a domain name. The id can be used to send invalidating
         request through #invalidate_paths function
            * Domain are Ronda domains, like "maven.repository.redhat.com"
@@ -200,5 +201,5 @@ def get_dist_id_by_domain(self, domain: str) -> str:
             )
         return None
 
-    def get_domain_by_bucket(self, bucket: str) -> str:
+    def get_domain_by_bucket(self, bucket: str) -> Optional[str]:
         return DEFAULT_BUCKET_TO_DOMAIN.get(bucket, None)

charon/cmd/cmd_cache.py

@@ -15,11 +15,12 @@
 """
 
 from charon.config import get_config
-from charon.cmd.internal import _decide_mode, _get_buckets
+from charon.cmd.internal import _decide_mode, _get_targets
 from charon.cache import CFClient
 from charon.pkgs.pkg_utils import invalidate_cf_paths
+from charon.types import TARGET_TYPE
 from click import command, option, argument, group
-from typing import List, Tuple
+from typing import List, Tuple, Optional
 
 import traceback
 import logging
@@ -58,6 +59,14 @@
     format of CF defining too, and each path should be in a single line.
     """
 )
+@option(
+    "--config",
+    "-c",
+    help="""
+    The charon configuration yaml file path. Default is
+    $HOME/.charon/charon.yaml
+    """
+)
 @option(
     "--debug",
     "-D",
@@ -79,6 +88,7 @@ def invalidate(
     target: str,
     paths: List[str],
     path_file: str,
+    config: str = "",
     quiet: bool = False,
     debug: bool = False
 ):
@@ -110,20 +120,20 @@ def invalidate(
             break
 
     try:
-        (buckets, aws_profile) = _init_cmd(target)
+        (targets, aws_profile) = _init_cmd(target, config)
 
-        for b in buckets:
+        for t in targets:
             cf_client = CFClient(aws_profile=aws_profile)
             # Per aws official doc, if the paths contains wildcard, it is
             # limited to 15 as max items in one request. Otherwise it could
             # be 3000
             if use_wildcard:
                 invalidate_cf_paths(
-                    cf_client, b, work_paths
+                    cf_client, t, work_paths
                 )
             else:
                 invalidate_cf_paths(
-                    cf_client, b, work_paths, batch_size=3000
+                    cf_client, t, work_paths, batch_size=3000
                 )
     except Exception:
         print(traceback.format_exc())
@@ -144,6 +154,14 @@ def invalidate(
     """,
     required=True
 )
+@option(
+    "--config",
+    "-c",
+    help="""
+    The charon configuration yaml file path. Default is
+    $HOME/.charon/charon.yaml
+    """
+)
 @option(
     "--debug",
     "-D",
@@ -164,6 +182,7 @@ def invalidate(
 def check(
     invalidation_id: str,
     target: str,
+    config: str = "",
     quiet: bool = False,
     debug: bool = False
 ):
@@ -175,14 +194,14 @@ def check(
         is_quiet=quiet, is_debug=debug, use_log_file=False
     )
     try:
-        (buckets, aws_profile) = _init_cmd(target)
-        if not buckets:
+        (targets, aws_profile) = _init_cmd(target, config)
+        if not targets:
             sys.exit(1)
 
-        for b in buckets:
+        for t in targets:
             cf_client = CFClient(aws_profile=aws_profile)
-            bucket_name = b[1]
-            domain = b[4]
+            bucket_name = t[1]
+            domain: Optional[str] = t[4]
             if not domain:
                 domain = cf_client.get_domain_by_bucket(bucket_name)
             if domain:
@@ -203,8 +222,8 @@ def check(
         sys.exit(2)
 
 
-def _init_cmd(target: str) -> Tuple[List[Tuple[str, str, str, str, str]], str]:
-    conf = get_config()
+def _init_cmd(target: str, config: str) -> Tuple[List[TARGET_TYPE], str]:
+    conf = get_config(config)
     if not conf:
         sys.exit(1)
 
@@ -213,7 +232,7 @@ def _init_cmd(target: str) -> Tuple[List[Tuple[str, str, str, str, str]], str]:
         logger.error("No AWS profile specified!")
         sys.exit(1)
 
-    return (_get_buckets([target], conf), aws_profile)
+    return (_get_targets([target], conf), aws_profile)
 
 
 @group()