Merge pull request #184 from ligangty/release

Merge from main branch for release 1.2.1

Author: ligangty

.github/workflows/unittests.yaml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9"]
+        python-version: ["3.8", "3.9"]
 
     steps:
       - uses: actions/checkout@v3

charon/pkgs/npm.py

@@ -16,7 +16,7 @@
 import logging
 import os
 import sys
-from json import load, loads, dump, JSONDecodeError
+from json import load, loads, dump, JSONDecodeError, JSONEncoder
 import tarfile
 from tempfile import mkdtemp
 from typing import List, Set, Tuple
@@ -32,7 +32,7 @@
 from charon.pkgs.pkg_utils import upload_post_process, rollback_post_process
 from charon.utils.strings import remove_prefix
 from charon.utils.files import write_manifest
-from charon.utils.map import del_none
+from charon.utils.map import del_none, replace_field
 
 logger = logging.getLogger(__name__)
 
@@ -61,10 +61,15 @@ def __init__(self, metadata, is_version):
             self.dist_tags = {'latest': metadata.get('version')}
             self.versions = {metadata.get('version'): metadata}
         else:
-            self.dist_tags = metadata.get('dist_tags', None)
+            self.dist_tags = metadata.get('dist-tags', None)
             self.versions = metadata.get('versions', None)
 
 
+class NPMPackageMetadataEncoder(JSONEncoder):
+    def default(self, o):
+        return replace_field(del_none(o.__dict__.copy()), "dist_tags", "dist-tags")
+
+
 def handle_npm_uploading(
         tarball_path: str,
         product: str,
@@ -549,7 +554,7 @@ def _write_package_metadata_to_file(package_metadata: NPMPackageMetadata, root='
     final_package_metadata_path = os.path.join(root, package_metadata.name, PACKAGE_JSON)
     try:
         with open(final_package_metadata_path, mode='w', encoding='utf-8') as f:
-            dump(del_none(package_metadata.__dict__.copy()), f)
+            dump(obj=package_metadata, cls=NPMPackageMetadataEncoder, fp=f)
         return final_package_metadata_path
     except FileNotFoundError:
         logger.error(

charon/storage.py

@@ -698,32 +698,36 @@ def list_folder_content(self, bucket_name: str, folder: str) -> List[str]:
            not in its subfolders.
         """
         bucket = self.__get_bucket(bucket_name)
+
         try:
+            paginator = bucket.meta.client.get_paginator('list_objects_v2')
             if not folder or folder.strip() == "/" or folder.strip() == "":
-                result = bucket.meta.client.list_objects(
+                pages = paginator.paginate(
                     Bucket=bucket.name,
                     Delimiter='/'
                 )
             else:
                 prefix = folder if folder.endswith("/") else folder+"/"
-                result = bucket.meta.client.list_objects(
+                pages = paginator.paginate(
                     Bucket=bucket.name,
                     Prefix=prefix,
                     Delimiter='/'
                 )
+
         except (ClientError, HTTPClientError) as e:
             logger.error("ERROR: Can not get contents of %s from bucket"
                          " %s due to error: %s ", folder,
                          bucket_name, e)
             return []
 
         contents = []
-        folders = result.get("CommonPrefixes")
-        if folders:
-            contents.extend([f.get("Prefix") for f in folders])
-        files = result.get("Contents")
-        if files:
-            contents.extend([f.get("Key") for f in files])
+        for page in pages:
+            folders = page.get("CommonPrefixes")
+            if folders:
+                contents.extend([f.get("Prefix") for f in folders])
+            files = page.get("Contents")
+            if files:
+                contents.extend([f.get("Key") for f in files])
         return contents
 
     def file_exists_in_bucket(

charon/utils/map.py

@@ -5,3 +5,10 @@ def del_none(d):
         elif isinstance(value, dict):
             del_none(value)
     return d
+
+
+def replace_field(d, field, new_field):
+    if d[field]:
+        d[new_field] = d[field]
+        del d[field]
+    return d

requirements.txt

@@ -1,10 +1,10 @@
-setuptools-rust==1.1.2
-jinja2==3.0.3
-boto3==1.20.45
-botocore==1.23.45
-click==8.0.3
-requests==2.27.1
-PyYAML==6.0
+setuptools-rust==1.7.0
+Jinja2==3.1.2
+boto3==1.28.46
+botocore==1.31.46
+click==8.1.7
+requests==2.31.0
+PyYAML==6.0.1
 defusedxml==0.7.1
 subresource-integrity==0.2
-jsonschema==3.2.0
+jsonschema==4.19.0

setup.py

@@ -13,14 +13,9 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-
-# !/usr/bin/env python
-
-import re
-
 from setuptools import setup, find_packages
 
-version = "1.1.2"
+version = "1.2.1"
 
 # f = open('README.md')
 # long_description = f.read().strip()
@@ -33,18 +28,6 @@
 in future. And Ronda service will be hosted in AWS S3.
 """
 
-
-def _get_requirements(path):
-    try:
-        with open(path, encoding="utf-8") as f:
-            packages = f.read().splitlines()
-    except (IOError, OSError) as ex:
-        raise RuntimeError(f"Can't open file with requirements: {ex}") from ex
-    packages = (p.strip() for p in packages if not re.match(r'^\s*#', p))
-    packages = list(filter(None, packages))
-    return packages
-
-
 setup(
     zip_safe=True,
     name="charon",
@@ -62,7 +45,6 @@ def _get_requirements(path):
     author="RedHat EXD SPMM",
     license="APLv2",
     packages=find_packages(exclude=["ez_setup", "examples", "tests"]),
-    install_requires=_get_requirements('requirements.txt'),
     package_data={'charon': ['schemas/*.json']},
     test_suite="tests",
     entry_points={

tests/test_npm_del.py

@@ -83,7 +83,7 @@ def __test_prefix(self, prefix: str = None):
         self.assertIn("\"versions\": {\"7.15.8\":", meta_content_client)
         self.assertNotIn("\"7.14.5\": {\"name\":", meta_content_client)
         self.assertIn("\"license\": \"MIT\"", meta_content_client)
-        self.assertIn("\"dist_tags\": {\"latest\": \"7.15.8\"}", meta_content_client)
+        self.assertIn("\"dist-tags\": {\"latest\": \"7.15.8\"}", meta_content_client)
 
         test_tgz = os.path.join(INPUTS, "code-frame-7.15.8.tgz")
         handle_npm_del(

tests/test_npm_del_multi_tgts.py

@@ -117,7 +117,7 @@ def __test_prefix(self, prefix: str = None):
                 "\"license\": \"MIT\"", meta_content_client, msg=f'{bucket_name}'
             )
             self.assertIn(
-                "\"dist_tags\": {\"latest\": \"7.15.8\"}",
+                "\"dist-tags\": {\"latest\": \"7.15.8\"}",
                 meta_content_client, msg=f'{bucket_name}'
             )
 

tests/test_npm_meta.py

@@ -48,7 +48,7 @@ def test_handle_npm_uploading_for_old_version(self):
         bucket = self.mock_s3.Bucket(MY_BUCKET)
         original_version_0_5_8_package_json = """
         {"name": "@redhat/kogito-tooling-workspace",
-        "dist_tags": {"latest": "0.5.8"}, "versions": {"0.5.8": {"name":
+        "dist-tags": {"latest": "0.5.8"}, "versions": {"0.5.8": {"name":
         "@redhat/kogito-tooling-workspace", "version": "0.5.8", "title": "0.5.8title",
         "description": "0.5.8description", "keywords": ["0.5.8"], "maintainers": [
         "0.5.8maintainer"], "repository": {"type": "git", "url": "https://github.com/0.5.8.git"},
@@ -98,7 +98,7 @@ def test_handle_npm_uploading_for_old_version(self):
     def test_handle_npm_uploading_for_new_version(self):
         bucket = self.mock_s3.Bucket(MY_BUCKET)
         original_version_1_0_1_package_json = """
-        {"name": "@redhat/kogito-tooling-workspace", "dist_tags": {"latest": "1.0.1"},
+        {"name": "@redhat/kogito-tooling-workspace", "dist-tags": {"latest": "1.0.1"},
         "versions": {"1.0.1": {"name": "@redhat/kogito-tooling-workspace", "version": "1.0.1",
         "title": "1.0.1title", "description": "1.0.1description", "keywords": ["1.0.1"],
         "maintainers": ["1.0.1maintainer"], "repository": {"type": "git",

tests/test_npm_upload.py

@@ -86,7 +86,7 @@ def test_double_uploads(self):
         self.assertIn("\"7.15.8\": {\"name\":", meta_content_client)
         self.assertIn("\"7.14.5\": {\"name\":", meta_content_client)
         self.assertIn("\"license\": \"MIT\"", meta_content_client)
-        self.assertIn("\"dist_tags\": {\"latest\": \"7.15.8\"}", meta_content_client)
+        self.assertIn("\"dist-tags\": {\"latest\": \"7.15.8\"}", meta_content_client)
 
     def __test_prefix(self, prefix: str = None):
         test_tgz = os.path.join(INPUTS, "code-frame-7.14.5.tgz")
@@ -131,4 +131,4 @@ def __test_prefix(self, prefix: str = None):
         self.assertIn("\"version\": \"7.14.5\"", meta_content_client)
         self.assertIn("\"versions\": {\"7.14.5\":", meta_content_client)
         self.assertIn("\"license\": \"MIT\"", meta_content_client)
-        self.assertIn("\"dist_tags\": {\"latest\": \"7.14.5\"}", meta_content_client)
+        self.assertIn("\"dist-tags\": {\"latest\": \"7.14.5\"}", meta_content_client)