Merge branch 'main' into konflux

Author: ligangty

charon.spec

@@ -1,7 +1,7 @@
 %global owner Commonjava
 %global modulename charon
 
-%global charon_version 1.3.3
+%global charon_version 1.4.0
 %global sdist_tar_name %{modulename}-%{charon_version}
 
 %global python3_pkgversion 3

charon/cmd/__init__.py

@@ -13,16 +13,19 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-from click import group
+from click import group, version_option, pass_context
 from charon.cmd.cmd_upload import upload
 from charon.cmd.cmd_delete import delete
 from charon.cmd.cmd_index import index
 from charon.cmd.cmd_checksum import init_checksum, checksum
 from charon.cmd.cmd_cache import init_cf, cf
+from charon.cmd.cmd_sign import sign
 
 
 @group()
-def cli():
+@version_option()
+@pass_context
+def cli(ctx):
     """Charon is a tool to synchronize several types of
        artifacts repository data to Red Hat Ronda
        service (maven.repository.redhat.com).
@@ -41,3 +44,6 @@ def cli():
 # init checksum command
 init_checksum()
 cli.add_command(checksum)
+
+# radas sign cmd
+cli.add_command(sign)

charon/cmd/cmd_sign.py

@@ -0,0 +1,138 @@
+"""
+Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon)
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from typing import List
+
+from charon.config import get_config
+from charon.pkgs.radas_sign import sign_in_radas
+from charon.cmd.internal import _decide_mode
+from charon.constants import DEFAULT_RADAS_SIGN_IGNORES
+
+from click import command, option, argument
+
+import traceback
+import logging
+import sys
+import datetime
+
+logger = logging.getLogger(__name__)
+
+
+@argument(
+    "repo_url",
+    type=str
+)
+@option(
+    "--requester",
+    "-r",
+    help="""
+    The requester who sends the signing request.
+    """,
+    required=True
+)
+@option(
+    "--result_path",
+    "-p",
+    help="""
+    The path which will save the sign result file.
+    """,
+    required=True
+)
+@option(
+    "--ignore_patterns",
+    "-i",
+    multiple=True,
+    help="""
+    The regex patterns list to filter out the files which should
+    not be allowed to upload to S3. Can accept more than one pattern.
+    """
+)
+@option(
+    "--config",
+    "-c",
+    help="""
+    The charon configuration yaml file path. Default is
+    $HOME/.charon/charon.yaml
+    """
+)
+@option(
+    "--sign_key",
+    "-k",
+    help="""
+    rpm-sign key to be used, will replace {{ key }} in default configuration for signature.
+    Does noting if detach_signature_command does not contain {{ key }} field.
+    """,
+    required=True
+)
+@option(
+    "--debug",
+    "-D",
+    help="Debug mode, will print all debug logs for problem tracking.",
+    is_flag=True,
+    default=False
+)
+@option(
+    "--quiet",
+    "-q",
+    help="Quiet mode, will shrink most of the logs except warning and errors.",
+    is_flag=True,
+    default=False
+)
+@command()
+def sign(
+    repo_url: str,
+    requester: str,
+    result_path: str,
+    sign_key: str,
+    ignore_patterns: List[str] = None,
+    config: str = None,
+    debug=False,
+    quiet=False
+):
+    """Do signing against files in the repo zip in repo_url through
+    radas service. The repo_url points to the maven zip repository
+    in quay.io, which will be sent as the source of the signing.
+    """
+    logger.debug("%s", ignore_patterns)
+    try:
+        current = datetime.datetime.now().strftime("%Y%m%d%I%M")
+        _decide_mode("radas_sign", current, is_quiet=quiet, is_debug=debug)
+        conf = get_config(config)
+        if not conf:
+            logger.error("The charon configuration is not valid!")
+            sys.exit(1)
+        radas_conf = conf.get_radas_config()
+        if not radas_conf or not radas_conf.validate():
+            logger.error("The configuration for radas is not valid!")
+            sys.exit(1)
+        # All ignore files in global config should also be ignored in signing.
+        ig_patterns = conf.get_ignore_patterns()
+        ig_patterns.extend(DEFAULT_RADAS_SIGN_IGNORES)
+        if ignore_patterns:
+            ig_patterns.extend(ignore_patterns)
+        ig_patterns = list(set(ig_patterns))
+        args = {
+            "repo_url": repo_url,
+            "requester": requester,
+            "sign_key": sign_key,
+            "result_path": result_path,
+            "ignore_patterns": ig_patterns,
+            "radas_config": radas_conf
+        }
+        logger.debug("params: %s", args)
+        sign_in_radas(**args)  # type: ignore
+    except Exception:
+        print(traceback.format_exc())
+        sys.exit(2)

charon/cmd/cmd_upload.py

@@ -136,6 +136,16 @@
     default=False
 )
 @option("--dryrun", "-n", is_flag=True, default=False)
+@option(
+    "--sign_result_loc",
+    "-l",
+    default="/tmp/sign",
+    help="""
+    The local save path for oras to pull the radas signature result.
+    Sign request will use this path to download the signature result,
+    Upload will use the file on this path to generate the corresponding .asc files
+    """,
+)
 @command()
 def upload(
     repo: str,
@@ -150,7 +160,8 @@ def upload(
     sign_key: str = "redhatdevel",
     debug=False,
     quiet=False,
-    dryrun=False
+    dryrun=False,
+    sign_result_file=None,
 ):
     """Upload all files from a released product REPO to Ronda
     Service. The REPO points to a product released tarball which
@@ -221,7 +232,8 @@ def upload(
                 key=sign_key,
                 dry_run=dryrun,
                 manifest_bucket_name=manifest_bucket_name,
-                config=config
+                config=config,
+                sign_result_file=sign_result_file
             )
             if not succeeded:
                 sys.exit(1)

charon/config.py

@@ -13,6 +13,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
 import logging
 import os
 from typing import Dict, List, Optional
@@ -24,6 +25,101 @@
 logger = logging.getLogger(__name__)
 
 
+class RadasConfig(object):
+    def __init__(self, data: Dict):
+        self.__umb_host: str = data.get("umb_host", None)
+        self.__umb_host_port: str = data.get("umb_host_port", "5671")
+        self.__result_queue: str = data.get("result_queue", None)
+        self.__request_chan: str = data.get("request_channel", None)
+        self.__client_ca: str = data.get("client_ca", None)
+        self.__client_key: str = data.get("client_key", None)
+        self.__client_key_pass_file: str = data.get("client_key_pass_file", None)
+        self.__root_ca: str = data.get("root_ca", "/etc/pki/tls/certs/ca-bundle.crt")
+        self.__quay_radas_registry_config: Optional[str] = data.get(
+            "quay_radas_registry_config", None
+        )
+        self.__radas_sign_timeout_retry_count: int = data.get("radas_sign_timeout_retry_count", 10)
+        self.__radas_sign_timeout_retry_interval: int = data.get(
+            "radas_sign_timeout_retry_interval", 60
+        )
+        self.__radas_receiver_timeout: int = int(data.get("radas_receiver_timeout", 1800))
+
+    def validate(self) -> bool:
+        if not self.__umb_host:
+            logger.error("Missing host name setting for UMB!")
+            return False
+        if not self.__result_queue:
+            logger.error("Missing the queue setting to receive signing result in UMB!")
+            return False
+        if not self.__request_chan:
+            logger.error("Missing the queue setting to send signing request in UMB!")
+            return False
+        if self.__client_ca and not os.access(self.__client_ca, os.R_OK):
+            logger.error("The client CA file is not valid!")
+            return False
+        if self.__client_key and not os.access(self.__client_key, os.R_OK):
+            logger.error("The client key file is not valid!")
+            return False
+        if self.__client_key_pass_file and not os.access(self.__client_key_pass_file, os.R_OK):
+            logger.error("The client key password file is not valid!")
+            return False
+        if self.__root_ca and not os.access(self.__root_ca, os.R_OK):
+            logger.error("The root ca file is not valid!")
+            return False
+        if self.__quay_radas_registry_config and not os.access(
+            self.__quay_radas_registry_config, os.R_OK
+        ):
+            self.__quay_radas_registry_config = None
+            logger.warning(
+                "The quay registry config for oras is not valid, will ignore the registry config!"
+            )
+        return True
+
+    def umb_target(self) -> str:
+        return f"amqps://{self.__umb_host.strip()}:{self.__umb_host_port}"
+
+    def result_queue(self) -> str:
+        return self.__result_queue.strip()
+
+    def request_channel(self) -> str:
+        return self.__request_chan.strip()
+
+    def client_ca(self) -> str:
+        return self.__client_ca.strip()
+
+    def client_key(self) -> str:
+        return self.__client_key.strip()
+
+    def client_key_password(self) -> str:
+        pass_file = self.__client_key_pass_file
+        if os.access(pass_file, os.R_OK):
+            with open(pass_file, "r") as f:
+                return f.read().strip()
+        elif pass_file:
+            logger.warning("The key password file is not accessible. Will ignore the password.")
+        return ""
+
+    def root_ca(self) -> str:
+        return self.__root_ca.strip()
+
+    def ssl_enabled(self) -> bool:
+        return bool(self.__client_ca and self.__client_key and self.__root_ca)
+
+    def quay_radas_registry_config(self) -> Optional[str]:
+        if self.__quay_radas_registry_config:
+            return self.__quay_radas_registry_config.strip()
+        return None
+
+    def radas_sign_timeout_retry_count(self) -> int:
+        return self.__radas_sign_timeout_retry_count
+
+    def radas_sign_timeout_retry_interval(self) -> int:
+        return self.__radas_sign_timeout_retry_interval
+
+    def receiver_timeout(self) -> int:
+        return self.__radas_receiver_timeout
+
+
 class CharonConfig(object):
     """CharonConfig is used to store all configurations for charon
     tools.
@@ -39,6 +135,13 @@ def __init__(self, data: Dict):
         self.__ignore_signature_suffix: Dict = data.get("ignore_signature_suffix", None)
         self.__signature_command: str = data.get("detach_signature_command", None)
         self.__aws_cf_enable: bool = data.get("aws_cf_enable", False)
+        radas_config: Dict = data.get("radas", None)
+        self.__radas_config: Optional[RadasConfig] = None
+        if radas_config:
+            self.__radas_config = RadasConfig(radas_config)
+            self.__radas_enabled = bool(self.__radas_config and self.__radas_config.validate())
+        else:
+            self.__radas_enabled = False
 
     def get_ignore_patterns(self) -> List[str]:
         return self.__ignore_patterns
@@ -67,19 +170,23 @@ def get_detach_signature_command(self) -> str:
     def is_aws_cf_enable(self) -> bool:
         return self.__aws_cf_enable
 
+    def is_radas_enabled(self) -> bool:
+        return self.__radas_enabled
+
+    def get_radas_config(self) -> Optional[RadasConfig]:
+        return self.__radas_config
+
 
 def get_config(cfgPath=None) -> CharonConfig:
     config_file_path = cfgPath
     if not config_file_path or not os.path.isfile(config_file_path):
         config_file_path = os.path.join(os.getenv("HOME", ""), ".charon", CONFIG_FILE)
-    data = read_yaml_from_file_path(config_file_path, 'schemas/charon.json')
+    data = read_yaml_from_file_path(config_file_path, "schemas/charon.json")
     return CharonConfig(data)
 
 
 def get_template(template_file: str) -> str:
-    template = os.path.join(
-        os.getenv("HOME", ''), ".charon/template", template_file
-    )
+    template = os.path.join(os.getenv("HOME", ""), ".charon/template", template_file)
     if os.path.isfile(template):
         with open(template, encoding="utf-8") as file_:
             return file_.read()

charon/constants.py

@@ -175,3 +175,10 @@
 DEFAULT_ERRORS_LOG = "errors.log"
 
 DEFAULT_REGISTRY = "localhost"
+DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT = 10
+DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL = 60
+
+DEFAULT_RADAS_SIGN_IGNORES = [
+    r".*\.md5$", r".*\.sha1$", r".*\.sha128$", r".*\.sha256$",
+    r".*\.sha512$", r".*\.asc$"
+]