(Re)add support to work with S3 buckets

Signed-off-by: Tobias Wolf <wolf@b1-systems.de>

Author: NotTheEvilOne

.github/actions/features_parse/action.yml

@@ -11,7 +11,7 @@ outputs:
 runs:
   using: composite
   steps:
-    - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.7.3
+    - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.8.0
     - id: result
       shell: bash
       run: |

.github/actions/flavors_parse/action.yml

@@ -13,7 +13,7 @@ outputs:
 runs:
   using: composite
   steps:
-    - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.7.3
+    - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.8.0
     - id: matrix
       shell: bash
       run: |

.github/actions/setup/action.yml

@@ -3,7 +3,7 @@ description: Installs the given GardenLinux Python library
 inputs:
     version:
         description: GardenLinux Python library version
-        default: "0.7.3"
+        default: "0.8.0"
 runs:
     using: composite
     steps:

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gardenlinux"
-version = "0.7.3"
+version = "0.8.0"
 description = "Contains tools to work with the features directory of gardenlinux, for example deducting dependencies from feature sets or validating cnames"
 authors = ["Garden Linux Maintainers <contact@gardenlinux.io>"]
 license = "Apache-2.0"
@@ -34,6 +34,7 @@ gl-cname = "gardenlinux.features.cname_main:main"
 gl-features-parse = "gardenlinux.features.__main__:main"
 gl-flavors-parse = "gardenlinux.flavors.__main__:main"
 gl-oci = "gardenlinux.oci.__main__:main"
+gl-s3 = "gardenlinux.s3.__main__:main"
 
 [tool.pytest.ini_options]
 pythonpath = ["src"]

src/gardenlinux/features/cname.py

@@ -128,6 +128,16 @@ def feature_set(self) -> str:
 
         return Parser().filter_as_string(self.flavor)
 
+    @property
+    def platform(self) -> str:
+        """
+        Returns the platform for the cname parsed.
+
+        :return: (str) Flavor
+        """
+
+        return re.split("[_-]", self._flavor, 1)[0]
+
     @property
     def version(self) -> Optional[str]:
         """

src/gardenlinux/s3/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+
+"""
+S3 module
+"""
+
+from .bucket import Bucket
+from .s3_artifacts import S3Artifacts
+
+__all__ = ["Bucket", "S3Artifacts"]

src/gardenlinux/s3/__main__.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+gl-s3 main entrypoint
+"""
+
+import argparse
+import os
+import re
+import sys
+from functools import reduce
+from pathlib import Path
+from typing import Any, List, Set
+
+from .s3_artifacts import S3Artifacts
+
+
+_ARGS_ACTION_ALLOWED = [
+    "push-artifacts-to-bucket",
+]
+
+def main() -> None:
+    """
+    gl-s3 main()
+
+    :since: 0.7.0
+    """
+
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--bucket", dest="bucket")
+    parser.add_argument("--cname", required=False, dest="cname")
+    parser.add_argument("--path", required=False, dest="path")
+
+    parser.add_argument("action", nargs="?", choices=_ARGS_ACTION_ALLOWED)
+
+    args = parser.parse_args()
+
+    if args.action == "push-artifacts-to-bucket":
+        S3Artifacts(args.bucket, "http://127.0.0.1:8000").push_from_directory(args.cname, args.path)

src/gardenlinux/s3/bucket.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+
+"""
+S3 bucket
+"""
+
+import boto3
+import logging
+from typing import Optional
+
+from ..logger import LoggerSetup
+
+
+class Bucket(object):
+    """
+    S3 bucket class
+
+    :author:     Garden Linux Maintainers
+    :copyright:  Copyright 2024 SAP SE
+    :package:    gardenlinux
+    :subpackage: s3
+    :since:      0.8.0
+    :license:    https://www.apache.org/licenses/LICENSE-2.0
+                 Apache License, Version 2.0
+    """
+
+    def __init__(
+        self,
+        bucket_name: str,
+        endpoint_url: Optional[str] = None,
+        logger: Optional[logging.Logger] = None,
+    ):
+        """
+        Constructor __init__(Bucket)
+
+        :param bucket_name: S3 bucket name
+        :param logger: Logger instance
+
+        :since: 0.8.0
+        """
+
+        if logger is None or not logger.hasHandlers():
+            logger = LoggerSetup.get_logger("gardenlinux.s3")
+
+        config={}
+
+        if endpoint_url is not None:
+            config["endpoint_url"] = endpoint_url
+
+        self._s3_resource = boto3.resource("s3", aws_access_key_id="default", aws_secret_access_key="default", **config)
+        self._bucket = self._s3_resource.Bucket(bucket_name)
+        self._logger = logger
+
+    @property
+    def objects(self):
+        """
+        Returns a list of all objects in a bucket.
+
+        :return: (list) S3 bucket objects
+        :since:  0.8.0
+        """
+
+        return self._bucket.objects.all()
+
+    def __getattr__(self, name):
+        """
+python.org: Called when an attribute lookup has not found the attribute in
+the usual places (i.e. it is not an instance attribute nor is it found in the
+class tree for self).
+
+:param name: Attribute name
+
+:return: (mixed) Attribute
+:since:  v1.0.0
+        """
+
+        return getattr(self._bucket, name)

src/gardenlinux/s3/s3_artifacts.py

@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+
+"""
+S3 GardenLinux artifacts
+"""
+
+import logging
+from configparser import ConfigParser, UNNAMED_SECTION
+from datetime import datetime
+from hashlib import file_digest
+from os import PathLike, stat
+from pathlib import Path
+from tempfile import TemporaryFile
+from typing import Optional
+from urllib.parse import urlencode
+
+from .bucket import Bucket
+from ..features.cname import CName
+from ..logger import LoggerSetup
+
+
+class S3Artifacts(object):
+    """
+    OCI container instance to provide methods for interaction.
+
+    :author:     Garden Linux Maintainers
+    :copyright:  Copyright 2024 SAP SE
+    :package:    gardenlinux
+    :subpackage: s3
+    :since:      0.8.0
+    :license:    https://www.apache.org/licenses/LICENSE-2.0
+                 Apache License, Version 2.0
+    """
+
+    def __init__(
+        self,
+        bucket_name: str,
+        endpoint_url: Optional[str] = None,
+        logger: Optional[logging.Logger] = None,
+    ):
+        """
+        Constructor __init__(Container)
+
+        :param bucket_name: S3 bucket name
+        :param endpoint_url: S3 endpoint URL
+        :param logger: Logger instance
+
+        :since: 0.7.0
+        """
+
+        """Get or create an S3 client."""
+
+        self._bucket = Bucket(bucket_name, endpoint_url)
+
+    def push_from_directory(
+        self,
+        cname,
+        artifacts_dir,
+    ):
+        """
+        Pushes S3 artifacts to the underlying bucket.
+
+        Args:
+            S3Artifacts_name (str): Name of the S3 S3Artifacts
+            prefix (str): Prefix for S3 objects
+            cache_file (str): Path to cache file
+            cache_ttl (int): Cache time-to-live in seconds
+            logger (logging.Logger): Logger instance to use
+
+        Returns:
+            dict: Dictionary containing 'index' and 'artifacts' keys
+        """
+
+        if not isinstance(artifacts_dir, PathLike):
+            artifacts_dir = Path(artifacts_dir)
+
+        cname_object = CName(cname)
+
+        if cname_object.arch is None:
+            raise RuntimeError(
+                "Architecture could not be determined from cname"
+            )
+
+        if not artifacts_dir.is_dir():
+            raise RuntimeError(f"Artifacts directory given is invalid: {artifacts_dir}")
+
+        release_file = artifacts_dir.joinpath(f"{cname}.release")
+        release_timestamp = datetime.fromtimestamp(stat(release_file).st_ctime).isoformat()
+
+        release_config = ConfigParser(allow_unnamed_section=True)
+        release_config.read(release_file)
+
+        if cname_object.version != release_config.get(UNNAMED_SECTION, "GARDENLINUX_VERSION"):
+            raise RuntimeError(
+                f"Release file data and given cname conflict detected: Version {cname_object.version}"
+            )
+
+        if cname_object.commit_id != release_config.get(UNNAMED_SECTION, "GARDENLINUX_COMMIT_ID"):
+            raise RuntimeError(
+                f"Release file data and given cname conflict detected: Commit ID {cname_object.commit_id}"
+            )
+
+        commit_hash = release_config.get(UNNAMED_SECTION, "GARDENLINUX_COMMIT_ID_LONG")
+
+        feature_set = release_config.get(UNNAMED_SECTION, "GARDENLINUX_FEATURES")
+        feature_list = feature_set.split(",")
+
+        metadata = {
+            "platform": cname_object.platform,
+            "architecture": cname_object.arch,
+            "base_image": None,
+            "build_committish": commit_hash,
+            "build_timestamp": release_timestamp,
+            "gardenlinux_epoch": cname_object.version.split(".", 1)[0],
+            "logs": None,
+            "modifiers": feature_set,
+            "require_uefi": "_usi" in feature_list,
+            "secureboot": "_trustedboot" in feature_list,
+            "published_image_metadata": None,
+            "s3_bucket": self._bucket.name,
+            "s3_key": f"meta/singles/{cname}",
+            "test_result": None,
+            "version": cname_object.version,
+            "paths": []
+        }
+
+        for artifact in artifacts_dir.iterdir():
+            if not artifact.match(f"{cname}*"):
+                continue
+
+            s3_key = f"objects/{cname}/{artifact.name}"
+
+            with artifact.open("rb") as fp:
+                md5sum = file_digest(fp, "md5").hexdigest()
+                sha256sum = file_digest(fp, "sha256").hexdigest()
+
+            artifact_metadata = {
+                "name": artifact.name,
+                "s3_bucket_name": self._bucket.name,
+                "s3_key": s3_key,
+                "suffix": "".join(artifact.suffixes),
+                "md5sum": md5sum,
+                "sha256sum": sha256sum,
+            }
+
+            s3_tags = {
+                "architecture": cname_object.arch,
+                "platform": cname_object.platform,
+                "version": cname_object.version,
+                "committish": commit_hash,
+                "md5sum": md5sum,
+                "sha256sum": sha256sum,
+            }
+
+            self._bucket.upload_file(artifact, s3_key, ExtraArgs={'Tagging': urlencode(s3_tags)})
+
+            metadata["paths"].append(artifact_metadata)
+
+        with TemporaryFile(mode='w+') as fp:
+            fp.write(yaml(metadata))
+            self._bucket.upload_fileobj(fp, f"meta/singles/{cname}", ExtraArgs={'ContentType': "text/yaml"})