Merge pull request #150 from ligangty/release

Release

Author: ligangty

charon/__init__.py

@@ -13,13 +13,9 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-import logging
 
 from charon.cmd.command import cli, upload, delete
-from charon.utils.logs import set_logging
 
 # init group command
 cli.add_command(upload)
 cli.add_command(delete)
-
-set_logging(level=logging.INFO)  # override this however you want

charon/cmd/command.py

@@ -13,7 +13,8 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-from typing import List
+from typing import List, Tuple
+
 from charon.config import CharonConfig, get_config
 from charon.utils.logs import set_logging
 from charon.utils.archive import detect_npm_archive, download_archive, NpmArchiveType
@@ -62,9 +63,10 @@
     help="""
     The target to do the uploading, which will decide which s3 bucket
     and what root path where all files will be uploaded to.
+    Can accept more than one target.
     """,
     required=True,
-    multiple=False,
+    multiple=True,
 )
 @option(
     "--root_path",
@@ -112,7 +114,7 @@ def upload(
     repo: str,
     product: str,
     version: str,
-    target: str,
+    target: List[str],
     root_path="maven-repository",
     ignore_patterns: List[str] = None,
     work_dir: str = None,
@@ -126,10 +128,10 @@ def upload(
     """
     tmp_dir = work_dir
     try:
+        __decide_mode(product, version, is_quiet=quiet, is_debug=debug)
         if dryrun:
             logger.info("Running in dry-run mode,"
                         "no files will be uploaded.")
-        __decide_mode(is_quiet=quiet, is_debug=debug)
         if not __validate_prod_key(product, version):
             return
         conf = get_config()
@@ -138,45 +140,47 @@ def upload(
 
         aws_profile = os.getenv("AWS_PROFILE") or conf.get_aws_profile()
         if not aws_profile:
-            logger.warning("No AWS profile specified!")
-
-        aws_bucket = conf.get_aws_bucket(target)
-        if not aws_bucket:
+            logger.error("No AWS profile specified!")
             sys.exit(1)
 
         archive_path = __get_local_repo(repo)
         npm_archive_type = detect_npm_archive(archive_path)
         product_key = f"{product}-{version}"
-        prefix_ = conf.get_bucket_prefix(target)
+        manifest_bucket_name = conf.get_manifest_bucket()
+        targets_ = __get_targets(target, conf)
         if npm_archive_type != NpmArchiveType.NOT_NPM:
             logger.info("This is a npm archive")
-            tmp_dir = handle_npm_uploading(
+            tmp_dir, succeeded = handle_npm_uploading(
                 archive_path,
                 product_key,
-                bucket_name=aws_bucket,
-                prefix=prefix_,
+                targets=targets_,
                 aws_profile=aws_profile,
                 dir_=work_dir,
-                dry_run=dryrun
+                dry_run=dryrun,
+                manifest_bucket_name=manifest_bucket_name
             )
+            if not succeeded:
+                sys.exit(1)
         else:
             ignore_patterns_list = None
             if ignore_patterns:
                 ignore_patterns_list = ignore_patterns
             else:
                 ignore_patterns_list = __get_ignore_patterns(conf)
             logger.info("This is a maven archive")
-            tmp_dir = handle_maven_uploading(
+            tmp_dir, succeeded = handle_maven_uploading(
                 archive_path,
                 product_key,
                 ignore_patterns_list,
                 root=root_path,
-                bucket_name=aws_bucket,
+                targets=targets_,
                 aws_profile=aws_profile,
-                prefix=prefix_,
                 dir_=work_dir,
-                dry_run=dryrun
+                dry_run=dryrun,
+                manifest_bucket_name=manifest_bucket_name
             )
+            if not succeeded:
+                sys.exit(1)
     except Exception:
         print(traceback.format_exc())
         sys.exit(2)  # distinguish between exception and bad config or bad state
@@ -216,9 +220,10 @@ def upload(
     help="""
     The target to do the deletion, which will decide which s3 bucket
     and what root path where all files will be deleted from.
+    Can accept more than one target.
     """,
     required=True,
-    multiple=False,
+    multiple=True,
 )
 @option(
     "--root_path",
@@ -265,7 +270,7 @@ def delete(
     repo: str,
     product: str,
     version: str,
-    target: str,
+    target: List[str],
     root_path="maven-repository",
     ignore_patterns: List[str] = None,
     work_dir: str = None,
@@ -279,10 +284,10 @@ def delete(
     """
     tmp_dir = work_dir
     try:
+        __decide_mode(product, version, is_quiet=quiet, is_debug=debug)
         if dryrun:
             logger.info("Running in dry-run mode,"
                         "no files will be deleted.")
-        __decide_mode(is_quiet=quiet, is_debug=debug)
         if not __validate_prod_key(product, version):
             return
         conf = get_config()
@@ -291,45 +296,47 @@ def delete(
 
         aws_profile = os.getenv("AWS_PROFILE") or conf.get_aws_profile()
         if not aws_profile:
-            logger.warning("No AWS profile specified!")
-
-        aws_bucket = conf.get_aws_bucket(target)
-        if not aws_bucket:
+            logger.error("No AWS profile specified!")
             sys.exit(1)
 
         archive_path = __get_local_repo(repo)
         npm_archive_type = detect_npm_archive(archive_path)
         product_key = f"{product}-{version}"
-        prefix_ = conf.get_bucket_prefix(target)
+        manifest_bucket_name = conf.get_manifest_bucket()
+        targets_ = __get_targets(target, conf)
         if npm_archive_type != NpmArchiveType.NOT_NPM:
             logger.info("This is a npm archive")
-            tmp_dir = handle_npm_del(
+            tmp_dir, succeeded = handle_npm_del(
                 archive_path,
                 product_key,
-                bucket_name=aws_bucket,
-                prefix=prefix_,
+                targets=targets_,
                 aws_profile=aws_profile,
                 dir_=work_dir,
-                dry_run=dryrun
+                dry_run=dryrun,
+                manifest_bucket_name=manifest_bucket_name
             )
+            if not succeeded:
+                sys.exit(1)
         else:
             ignore_patterns_list = None
             if ignore_patterns:
                 ignore_patterns_list = ignore_patterns
             else:
                 ignore_patterns_list = __get_ignore_patterns(conf)
             logger.info("This is a maven archive")
-            tmp_dir = handle_maven_del(
+            tmp_dir, succeeded = handle_maven_del(
                 archive_path,
                 product_key,
                 ignore_patterns_list,
                 root=root_path,
-                bucket_name=aws_bucket,
+                targets=targets_,
                 aws_profile=aws_profile,
-                prefix=prefix_,
                 dir_=work_dir,
-                dry_run=dryrun
+                dry_run=dryrun,
+                manifest_bucket_name=manifest_bucket_name
             )
+            if not succeeded:
+                sys.exit(1)
     except Exception:
         print(traceback.format_exc())
         sys.exit(2)  # distinguish between exception and bad config or bad state
@@ -338,6 +345,23 @@ def delete(
             __safe_delete(tmp_dir)
 
 
+def __get_targets(target: List[str], conf: CharonConfig) -> List[Tuple[str, str, str]]:
+    targets_ = []
+    for tgt in target:
+        aws_bucket = conf.get_aws_bucket(tgt)
+        if not aws_bucket:
+            continue
+        prefix = conf.get_bucket_prefix(tgt)
+        targets_.append([tgt, aws_bucket, prefix])
+    if len(targets_) == 0:
+        logger.error(
+            "All targets are not valid or configured, "
+            "please check your charon configurations."
+        )
+        sys.exit(1)
+    return targets_
+
+
 def __safe_delete(tmp_dir: str):
     if tmp_dir and os.path.exists(tmp_dir):
         logger.info("Cleaning up work directory: %s", tmp_dir)
@@ -386,15 +410,17 @@ def __validate_prod_key(product: str, version: str) -> bool:
     return True
 
 
-def __decide_mode(is_quiet: bool, is_debug: bool):
+def __decide_mode(product: str, version: str, is_quiet: bool, is_debug: bool):
     if is_quiet:
         logger.info("Quiet mode enabled, "
                     "will only give warning and error logs.")
-        set_logging(level=logging.WARNING)
+        set_logging(product, version, level=logging.WARNING)
     elif is_debug:
         logger.info("Debug mode enabled, "
                     "will give all debug logs for tracing.")
-        set_logging(level=logging.DEBUG)
+        set_logging(product, version, level=logging.DEBUG)
+    else:
+        set_logging(product, version, level=logging.INFO)
 
 
 @group()

charon/config.py

@@ -38,6 +38,7 @@ def __init__(self, data: Dict):
         self.__targets: Dict = data.get("targets", None)
         if not self.__targets or not isinstance(self.__targets, Dict):
             raise TypeError("Charon configuration is not correct: targets is invalid.")
+        self.__manifest_bucket: str = data.get("manifest_bucket", None)
 
     def get_ignore_patterns(self) -> List[str]:
         return self.__ignore_patterns
@@ -48,7 +49,7 @@ def get_aws_profile(self) -> str:
     def get_aws_bucket(self, target: str) -> str:
         target_: Dict = self.__targets.get(target, None)
         if not target_ or not isinstance(target_, Dict):
-            logger.error("The target %s is not found in charon configuration.")
+            logger.error("The target %s is not found in charon configuration.", target)
             return None
         bucket = target_.get("bucket", None)
         if not bucket:
@@ -72,6 +73,9 @@ def get_bucket_prefix(self, target: str) -> str:
         prefix = remove_prefix(prefix, "/")
         return prefix
 
+    def get_manifest_bucket(self) -> str:
+        return self.__manifest_bucket
+
 
 def get_config() -> CharonConfig:
     config_file = os.path.join(os.getenv("HOME"), ".charon", CONFIG_FILE)

charon/constants.py

@@ -133,10 +133,8 @@
   </header>
   <hr/>
   <main>
-    <ul style="list-style: none outside;" id="contents">
-    {% for item in index.items %}
-        <li><a href="{{ item }}" title="{{ item }}">{{ item }}</a></li>
-    {% endfor%}
+    <ul style="list-style: none outside;" id="contents">{% for item in index.items %}
+        <li><a href="{{ item }}" title="{{ item }}">{{ item }}</a></li>{% endfor%}
     </ul>
   </main>
   <hr/>
@@ -162,11 +160,9 @@
   <hr/>
   <main>
     <ul style="list-style: none outside;" id="contents">
-    {% for item in index.items %}{% if item.startswith("@") or item.startswith("..") %}
-        <li><a href="{{ item }}index.html" title="{{ item }}">{{ item }}</a></li>
-    {% else %}
-        <li><a href="{{ item }}" title="{{ item }}">{{ item }}</a></li>
-    {% endif %}{% endfor%}
+        {% for item in index.items %}{% if item.startswith("@") or item.startswith("..") %}
+        <li><a href="{{ item }}index.html" title="{{ item }}">{{ item }}</a></li>{% else %}
+        <li><a href="{{ item }}" title="{{ item }}">{{ item }}</a></li>{% endif %}{% endfor%}
     </ul>
   </main>
   <hr/>
@@ -175,3 +171,5 @@
 '''
 
 PROD_INFO_SUFFIX = ".prodinfo"
+MANIFEST_SUFFIX = ".txt"
+DEFAULT_ERRORS_LOG = "errors.log"

charon/pkgs/indexing.py

@@ -127,10 +127,9 @@ def __generate_index_html(
             removed_index = os.path.join(top_level, folder_, "index.html")
         s3_client.delete_files(
             file_paths=[removed_index],
-            bucket_name=bucket,
+            target=(bucket, prefix),
             product=None,
-            root=top_level,
-            key_prefix=prefix
+            root=top_level
         )
     elif len(contents) >= 1:
         real_contents = []