move subprocess command to configuration, make subprocess to run async (#172)

shokakucarrier · web-flow · commit 71ca509b449a · 2023-05-26T16:28:10.000+08:00
* move subprocess command to configuration, make subprocess to run async

* simplify template rendering of sign command

* Remove unwanted file
diff --git a/README.md b/README.md
@@ -14,10 +14,9 @@ future. And Ronda service will be hosted in AWS S3.
 
 See [AWS CLi V2 installation](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html#cliv2-linux-install)
 
-### [Optional] GnuPG CLI tool or rpm-sign
+### [Optional] rpm-sign or GnuPG CLI tool
 
-For artifact signing using keys already in GPG keystore, not required when using exported secret key file.
-Can be configured to use rpm-sign for release.
+Can be configured to use rpm-sign or any command to generate .asc file.
 
 ## Installation
 
@@ -54,7 +53,7 @@ to configure AWS access credentials.
 ### charon-upload: upload a repo to S3
 
 ```bash
-usage: charon upload $tarball --product/-p ${prod} --version/-v ${ver} [--root_path] [--ignore_patterns] [--debug] [--key_id] [--key_file] [--passphrase]
+usage: charon upload $tarball --product/-p ${prod} --version/-v ${ver} [--root_path] [--ignore_patterns] [--debug] [--contain_signature] [--key]
 ```
 
 This command will upload the repo in tarball to S3.
diff --git a/charon/cmd/command.py b/charon/cmd/command.py
@@ -97,33 +97,19 @@
     """,
 )
 @option(
-    "--sign_keyid",
-    "-k",
-    help="""
-    GPG key fingerprint to sign artifacts, cannot work with --sign_keyfile
-    requires key already imported with gpg command --passphrase is needed.
-    """,
-)
-@option(
-    "--sign_keyfile",
-    "-K",
-    help="""
-    GPG key file path to sign artifacts, --passphrase is needed.
-    Will be ignored when using sign_method rpm-sign
-    """,
-)
-@option(
-    "--sign_method",
+    "--contain_signature",
+    "-s",
+    is_flag=True,
     help="""
-    Choose 'gpg' or 'rpm-sign' to warp the actuall command, default to use gpg.
-    key_id will be used to set key for signature.
-    """,
+    Toggle signature generation and upload feature in charon.
+    """
 )
 @option(
-    "--passphrase",
+    "--sign_key",
+    "-k",
     help="""
-    The passphrase for GPG key, Necessary when using gpg as sign method.
-    Will require input when it's not set
+    rpm-sign key to be used, will replace {{ key }} in default configuration for signature.
+    Does noting if detach_signature_command does not contain {{ key }} field.
     """,
 )
 @option(
@@ -150,10 +136,8 @@ def upload(
     root_path="maven-repository",
     ignore_patterns: List[str] = None,
     work_dir: str = None,
-    sign_keyid: str = None,
-    sign_keyfile: str = None,
-    sign_method: str = "gpg",
-    passphrase: str = None,
+    contain_signature: bool = False,
+    sign_key: str = "redhatdevel",
     debug=False,
     quiet=False,
     dryrun=False
@@ -179,10 +163,6 @@ def upload(
             logger.error("No AWS profile specified!")
             sys.exit(1)
 
-        if sign_method != 'gpg' and sign_method != 'rpm-sign':
-            logger.error("Signature method must be gpg or rpm-sign")
-            sys.exit(1)
-
         archive_path = __get_local_repo(repo)
         npm_archive_type = detect_npm_archive(archive_path)
         product_key = f"{product}-{version}"
@@ -196,10 +176,8 @@ def upload(
                 buckets=buckets,
                 aws_profile=aws_profile,
                 dir_=work_dir,
-                key_id=sign_keyid,
-                key_file=sign_keyfile,
-                sign_method=sign_method,
-                passphrase=passphrase,
+                gen_sign=contain_signature,
+                key=sign_key,
                 dry_run=dryrun,
                 manifest_bucket_name=manifest_bucket_name
             )
@@ -220,10 +198,8 @@ def upload(
                 buckets=buckets,
                 aws_profile=aws_profile,
                 dir_=work_dir,
-                key_id=sign_keyid,
-                key_file=sign_keyfile,
-                sign_method=sign_method,
-                passphrase=passphrase,
+                gen_sign=contain_signature,
+                key=sign_key,
                 dry_run=dryrun,
                 manifest_bucket_name=manifest_bucket_name
             )
diff --git a/charon/config.py b/charon/config.py
@@ -37,6 +37,7 @@ def __init__(self, data: Dict):
         self.__targets: Dict = data.get("targets", None)
         self.__manifest_bucket: str = data.get("manifest_bucket", None)
         self.__ignore_signature_suffix: Dict = data.get("ignore_signature_suffix", None)
+        self.__signature_command: str = data.get("detach_signature_command", None)
 
     def get_ignore_patterns(self) -> List[str]:
         return self.__ignore_patterns
@@ -59,6 +60,9 @@ def get_ignore_signature_suffix(self, package_type: str) -> List[str]:
             logger.error("package type %s does not have ignore artifact config.", package_type)
         return xartifact_list
 
+    def get_detach_signature_command(self) -> str:
+        return self.__signature_command
+
 
 def get_config() -> Optional[CharonConfig]:
     config_file_path = os.path.join(os.getenv("HOME"), ".charon", CONFIG_FILE)
diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py
@@ -261,10 +261,8 @@ def handle_maven_uploading(
     aws_profile=None,
     dir_=None,
     do_index=True,
-    key_id=None,
-    key_file=None,
-    sign_method=None,
-    passphrase=None,
+    gen_sign=False,
+    key=None,
     dry_run=False,
     manifest_bucket_name=None
 ) -> Tuple[str, bool]:
@@ -389,19 +387,20 @@ def handle_maven_uploading(
                 failed_metas.extend(_failed_metas)
                 logger.info("archetype-catalog.xml updating done in bucket %s\n", bucket_name)
 
-        # 10. Generate signature file if gpg ket is provided
-        if key_id is not None or key_file is not None:
+        # 10. Generate signature file if contain_signature is set to True
+        if gen_sign:
             conf = get_config()
             if not conf:
                 sys.exit(1)
             suffix_list = __get_suffix(PACKAGE_TYPE_MAVEN, conf)
+            command = conf.get_detach_signature_command()
             artifacts = [s for s in valid_mvn_paths if not s.endswith(tuple(suffix_list))]
             logger.info("Start generating signature for s3 bucket %s\n", bucket_name)
             (_failed_metas, _generated_signs) = signature.generate_sign(
                 PACKAGE_TYPE_MAVEN, artifacts,
                 top_level, prefix,
                 s3_client, bucket_name,
-                key_id, key_file, sign_method, passphrase
+                key, command
             )
             failed_metas.extend(_failed_metas)
             generated_signs.extend(_generated_signs)
diff --git a/charon/pkgs/npm.py b/charon/pkgs/npm.py
@@ -72,10 +72,8 @@ def handle_npm_uploading(
         aws_profile=None,
         dir_=None,
         do_index=True,
-        key_id=None,
-        key_file=None,
-        sign_method=None,
-        passphrase=None,
+        gen_sign=False,
+        key=None,
         dry_run=False,
         manifest_bucket_name=None
 ) -> Tuple[str, bool]:
@@ -167,11 +165,12 @@ def handle_npm_uploading(
             failed_metas.extend(_failed_metas)
             logger.info("package.json uploading done")
 
-        if key_id is not None or key_file is not None:
+        if gen_sign:
             conf = get_config()
             if not conf:
                 sys.exit(1)
             suffix_list = __get_suffix(PACKAGE_TYPE_NPM, conf)
+            command = conf.get_detach_signature_command()
             artifacts = [s for s in valid_paths if not s.endswith(tuple(suffix_list))]
             if META_FILE_GEN_KEY in meta_files:
                 artifacts.extend(meta_files[META_FILE_GEN_KEY])
@@ -180,7 +179,7 @@ def handle_npm_uploading(
                 PACKAGE_TYPE_NPM, artifacts,
                 target_dir, prefix,
                 client, bucket_name,
-                key_id, key_file, sign_method, passphrase
+                key, command
             )
             failed_metas.extend(_failed_metas)
             generated_signs.extend(_generated_signs)
diff --git a/charon/pkgs/signature.py b/charon/pkgs/signature.py
@@ -18,10 +18,9 @@
 import subprocess
 import asyncio
 import logging
-import gnupg
+from jinja2 import Template
 from typing import Awaitable, Callable, List, Tuple
 from charon.storage import S3Client
-from getpass import getpass
 
 logger = logging.getLogger(__name__)
 
@@ -33,34 +32,22 @@ def generate_sign(
     prefix: str,
     s3_client: S3Client,
     bucket: str,
-    key_id: str = None,
-    key_file: str = None,
-    sign_method: str = None,
-    passphrase: str = None
+    key: str = None,
+    command: str = None
 ) -> Tuple[List[str], List[str]]:
     """ This Python function generates a digital signature for a list of metadata files using
     the GPG library for uploads to an Amazon S3 bucket.
 
         * Does not regenerate the existing metadata files when existing
         * Returning all failed to generate signature files due to exceptions
-        * key_id: A string representing the ID of the RSA key to use for signing,
-        GPG command line tool is required when this parameter is not None.
-        * key_file: A string representing the location of the private key file.
-        * passphrase: A string containing the passphrase for the RSA key for key_id or key_file.
+        * key: name of the sign key, using inside template to render correct command,
+        replace {{ key }} field in command string.
+        * command: A string representing the subprocess command to run.
 
     It returns a tuple containing two lists: one with the successfully generated files
     and another with the failed to generate files due to exceptions.
     """
 
-    gpg = None
-    if key_file is not None:
-        gnupg_home_path = os.path.join(os.getenv("HOME"), ".charon", ".gnupg")
-        gpg = gnupg.GPG(gnupghome=gnupg_home_path)
-        gpg.import_keys_file(key_file)
-
-    if sign_method == 'gpg' and passphrase is None:
-        passphrase = getpass('Passphrase for your gpg key:')
-
     async def sign_file(
         filename: str, failed_paths: List[str], generated_signs: List[str]
     ):
@@ -93,18 +80,12 @@ async def sign_file(
                 logger.debug(".asc file %s existed, skipping", remote)
                 return
 
-            if sign_method == "rpm-sign":
-                result = detach_rpm_sign_files(key_id, artifact)
-            elif sign_method == "gpg":
-                result = gpg_sign_files(
-                    gpg=gpg,
-                    artifact=artifact,
-                    key_id=key_id, key_file=key_file,
-                    passphrase=passphrase
-                )
+            run_command = Template(command)
+            result = await __run_cmd_async(run_command.render(key=key, file=artifact).split())
 
-            if result == 0:
+            if result.returncode == 0:
                 generated_signs.append(local)
+                logger.debug("Generated signature file: %s", local)
             else:
                 failed_paths.append(local)
 
@@ -115,82 +96,6 @@ async def sign_file(
         )
 
 
-def detach_rpm_sign_files(key: str, artifact: str) -> int:
-    # Usage: detach_sign_files KEYNAME FILE ...
-
-    # let's make sure we can actually sign with the given key
-    command = [
-        'rpm-sign',
-        '--list-keys',
-        '|',
-        'grep',
-        key
-    ]
-    result = subprocess.run(command, capture_output=True, text=True, check=True)
-    if result.returncode != 0:
-        logger.error("Key %s is not in list of allowed keys", key)
-        return result.returncode
-
-    # okay, now let's actually sign this thing
-    command = [
-        'rpm-sign',
-        '--detachsign',
-        '--key',
-        key,
-        artifact
-    ]
-    try:
-        # result = await __run_cmd_async(command)
-        result = subprocess.run(command, capture_output=True, text=True, check=True)
-    except subprocess.CalledProcessError as e:
-        logger.error(
-                "Error: signature generation failed due to error: %s", e
-            )
-        return result.returncode
-
-    return 1
-
-
-def gpg_sign_files(
-    artifact: str,
-    gpg=None,
-    key_id: str = None,
-    key_file: str = None,
-    passphrase: str = None
-) -> int:
-    command = [
-        'gpg',
-        '--batch',
-        '--armor',
-        '-u', key_id,
-        '--passphrase', passphrase,
-        '--sign', artifact
-    ]
-
-    if key_file is None:
-        # use GPG command line tool to sign artifact if key_id is passed
-        try:
-            # result = await __run_cmd_async(command)
-            result = subprocess.run(command, capture_output=True, text=True, check=True)
-        except subprocess.CalledProcessError as e:
-            logger.error(
-                    "Error: signature generation failed due to error: %s", e
-                )
-        return result.returncode
-    else:
-        try:
-            with open(artifact, "rb") as f:
-                local = artifact + '.asc'
-                gpg.sign_file(f, passphrase=passphrase, output=local, detach=True)
-                return 0
-        except ValueError as e:
-            logger.error(
-                    "Error: signature generation failed due to error: %s", e
-                )
-            return 1
-    return 1
-
-
 def __do_path_cut_and(
     file_paths: List[str],
     path_handler: Callable[[str, List[str], List[str], asyncio.Semaphore], Awaitable[bool]],
diff --git a/charon/schemas/charon.json b/charon/schemas/charon.json
@@ -26,6 +26,10 @@
         }
       }
     },
+    "detach_signature_command": {
+      "type": "string",
+      "description": "signature command to be used for signature"
+    },
     "targets": {
       "type": "object",
       "patternProperties": {
diff --git a/config/charon.yaml b/config/charon.yaml
@@ -12,6 +12,8 @@ ignore_signature_suffix:
   npm:
     - "package.json"
 
+detach_signature_command: "rpm-sign --detach-sign --key {{ key }} {{ file }}"
+
 targets:
   stage-ga:
   - bucket: "stage-maven-ga"
diff --git a/requirements.txt b/requirements.txt
@@ -8,4 +8,3 @@ PyYAML==6.0
 defusedxml==0.7.1
 subresource-integrity==0.2
 jsonschema==3.2.0
-python-gnupg==0.5.0

Original file line number	Diff line number	Diff line change
`@@ -26,6 +26,10 @@`
`26`	`26`	`}`
`27`	`27`	`}`
`28`	`28`	`},`
	`29`	`+ "detach_signature_command": {`
	`30`	`+ "type": "string",`
	`31`	`+ "description": "signature command to be used for signature"`
	`32`	`+ },`
`29`	`33`	`"targets": {`
`30`	`34`	`"type": "object",`
`31`	`35`	`"patternProperties": {`