From b3e49cb06f83f8a330a8efc957f79d812966240d Mon Sep 17 00:00:00 2001 From: Eike Waldt Date: Tue, 1 Jul 2025 16:06:44 +0200 Subject: [PATCH 1/7] add Github Class --- src/gardenlinux/github/__init__.py | 9 ++ src/gardenlinux/github/github.py | 172 +++++++++++++++++++++++++++++ 2 files changed, 181 insertions(+) create mode 100644 src/gardenlinux/github/__init__.py create mode 100644 src/gardenlinux/github/github.py diff --git a/src/gardenlinux/github/__init__.py b/src/gardenlinux/github/__init__.py new file mode 100644 index 00000000..8319f72a --- /dev/null +++ b/src/gardenlinux/github/__init__.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- + +""" +GitHub module +""" + +from .github import GitHub + +__all__ = ["GitHub"] diff --git a/src/gardenlinux/github/github.py b/src/gardenlinux/github/github.py new file mode 100644 index 00000000..bd790f22 --- /dev/null +++ b/src/gardenlinux/github/github.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- + +import subprocess +import json +import base64 + +from ..logger import LoggerSetup + + +class GitHub(object): + """ + GitHub operations handler using GitHub CLI. + + :author: Garden Linux Maintainers + :copyright: Copyright 2024 SAP SE + :package: gardenlinux + :subpackage: github + :since: 0.9.0 + :license: https://www.apache.org/licenses/LICENSE-2.0 + Apache License, Version 2.0 + """ + + def __init__(self, owner="gardenlinux", repo="gardenlinux", logger=None): + """ + Constructor __init__(GitHub) + + :param owner: GitHub repository owner + :param repo: GitHub repository name + :param logger: Logger instance + + :since: 0.9.0 + """ + + if logger is None or not logger.hasHandlers(): + logger = LoggerSetup.get_logger("gardenlinux.github") + + self._owner = owner + self._repo = repo + self._logger = logger + + self._logger.debug(f"GitHub initialized for {owner}/{repo}") + + def api(self, endpoint, **kwargs): + """ + Execute a GitHub API call using gh cli. + + :param endpoint: GitHub API endpoint (e.g. "/repos/owner/repo/contents/file.yaml") + :param kwargs: Additional parameters for the API call + + :return: (dict) Parsed JSON response + :since: 0.9.0 + """ + + command = ["gh", "api", endpoint] + + # Add any additional parameters to the command + for key, value in kwargs.items(): + if key.startswith("--"): + command.extend([key, str(value)]) + else: + command.extend([f"--{key}", str(value)]) + + self._logger.debug(f"Executing GitHub API call: {' '.join(command)}") + + try: + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True, + ) + + return json.loads(result.stdout) + + except subprocess.CalledProcessError as e: + self._logger.error(f"GitHub API call failed: {e.stderr}") + raise RuntimeError( + f"GitHub API call failed for endpoint {endpoint}: {e.stderr}" + ) + except json.JSONDecodeError as e: + self._logger.error(f"Failed to parse GitHub API response: {e}") + raise RuntimeError(f"Failed to parse GitHub API response: {e}") + + def get_file_content(self, file_path, ref=None): + """ + Get file content from GitHub repository. + + :param file_path: Path to file in repository (e.g. "flavors.yaml") + :param ref: Git reference (commit, branch, tag). If None, uses default branch + + :return: (str) File content + :since: 0.9.0 + """ + + endpoint = f"/repos/{self._owner}/{self._repo}/contents/{file_path}" + + # Add ref parameter if specified + if ref is not None: + endpoint = f"{endpoint}?ref={ref}" + + self._logger.debug( + f"Fetching file content: {file_path} (ref: {ref or 'default'})" + ) + + try: + response = self.api(endpoint) + + # Decode base64 content + content = base64.b64decode(response["content"]).decode("utf-8") + + self._logger.debug( + f"Successfully fetched {len(content)} characters from {file_path}" + ) + + return content + + except Exception as e: + self._logger.error(f"Failed to fetch file content for {file_path}: {e}") + raise RuntimeError(f"Failed to fetch file content for {file_path}: {e}") + + def get_flavors_yaml(self, commit="latest"): + """ + Get flavors.yaml content from the repository. + + :param commit: Commit hash or "latest" for default branch + + :return: (str) flavors.yaml content + :since: 0.9.0 + """ + + ref = None if commit == "latest" else commit + commit_short = commit if commit == "latest" else commit[:8] + + self._logger.debug(f"Fetching flavors.yaml for commit {commit_short}") + + try: + content = self.get_file_content("flavors.yaml", ref=ref) + self._logger.debug( + f"Successfully fetched flavors.yaml for commit {commit_short}" + ) + return content + + except Exception as e: + self._logger.error( + f"Failed to fetch flavors.yaml for commit {commit_short}: {e}" + ) + raise RuntimeError( + f"Failed to fetch flavors.yaml for commit {commit_short}: {e}" + ) + + @property + def repository_url(self): + """ + Returns the GitHub repository URL. + + :return: (str) GitHub repository URL + :since: 0.9.0 + """ + + return f"https://github.com/{self._owner}/{self._repo}" + + @property + def api_url(self): + """ + Returns the GitHub API base URL for this repository. + + :return: (str) GitHub API base URL + :since: 0.9.0 + """ + + return f"https://api.github.com/repos/{self._owner}/{self._repo}" From 0670cd5c6b025160f5f3e6fa3f3a21b45f32777f Mon Sep 17 00:00:00 2001 From: Eike Waldt Date: Tue, 1 Jul 2025 16:08:01 +0200 Subject: [PATCH 2/7] optionally fetch flavors.yaml from GitHub --- src/gardenlinux/flavors/__main__.py | 63 ++++++++++++++++++++--------- src/gardenlinux/flavors/parser.py | 1 + 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/src/gardenlinux/flavors/__main__.py b/src/gardenlinux/flavors/__main__.py index 9d7997ac..46c05938 100644 --- a/src/gardenlinux/flavors/__main__.py +++ b/src/gardenlinux/flavors/__main__.py @@ -11,6 +11,7 @@ import sys from ..git import Git +from ..github import GitHub from .parser import Parser @@ -48,6 +49,11 @@ def parse_args(): parser = ArgumentParser(description="Parse flavors.yaml and generate combinations.") + parser.add_argument( + "--commit", + default=None, + help="Commit hash to fetch flavors.yaml from GitHub (if not specified, uses local file).", + ) parser.add_argument( "--no-arch", action="store_true", @@ -120,25 +126,44 @@ def main(): args = parse_args() - flavors_file = os.path.join(Git().root, "flavors.yaml") - - if not os.path.isfile(flavors_file): - sys.exit(f"Error: {flavors_file} does not exist.") - - # Load and validate the flavors.yaml - with open(flavors_file, "r") as file: - flavors_data = file.read() - - combinations = Parser(flavors_data).filter( - include_only_patterns=args.include_only, - wildcard_excludes=args.exclude, - only_build=args.build, - only_test=args.test, - only_test_platform=args.test_platform, - only_publish=args.publish, - filter_categories=args.category, - exclude_categories=args.exclude_category, - ) + if args.commit: + # Use GitHub API to fetch flavors.yaml + github = GitHub() + flavors_content = github.get_flavors_yaml(commit=args.commit) + + parser = Parser(data=flavors_content) + + combinations = parser.filter( + include_only_patterns=args.include_only, + wildcard_excludes=args.exclude, + only_build=args.build, + only_test=args.test, + only_test_platform=args.test_platform, + only_publish=args.publish, + filter_categories=args.category, + exclude_categories=args.exclude_category, + ) + else: + # Use local file + flavors_file = os.path.join(Git().root, "flavors.yaml") + + if not os.path.isfile(flavors_file): + sys.exit(f"Error: {flavors_file} does not exist.") + + # Load and validate the flavors.yaml + with open(flavors_file, "r") as file: + flavors_data = file.read() + + combinations = Parser(flavors_data).filter( + include_only_patterns=args.include_only, + wildcard_excludes=args.exclude, + only_build=args.build, + only_test=args.test, + only_test_platform=args.test_platform, + only_publish=args.publish, + filter_categories=args.category, + exclude_categories=args.exclude_category, + ) if args.json_by_arch: grouped_combinations = Parser.group_by_arch(combinations) diff --git a/src/gardenlinux/flavors/parser.py b/src/gardenlinux/flavors/parser.py index 5ac69c8b..74f5e5d3 100644 --- a/src/gardenlinux/flavors/parser.py +++ b/src/gardenlinux/flavors/parser.py @@ -10,6 +10,7 @@ from ..constants import GL_FLAVORS_SCHEMA from ..logger import LoggerSetup +from ..github import GitHub class Parser(object): From 1a2d58fe74d5729cfc69355b17823ded3fdbb044 Mon Sep 17 00:00:00 2001 From: Eike Waldt Date: Tue, 1 Jul 2025 16:08:42 +0200 Subject: [PATCH 3/7] add S3ObjectIndex Class --- src/gardenlinux/s3/__init__.py | 3 +- src/gardenlinux/s3/s3_object_index.py | 140 ++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 src/gardenlinux/s3/s3_object_index.py diff --git a/src/gardenlinux/s3/__init__.py b/src/gardenlinux/s3/__init__.py index ec76b3b7..0c6b9166 100644 --- a/src/gardenlinux/s3/__init__.py +++ b/src/gardenlinux/s3/__init__.py @@ -6,5 +6,6 @@ from .bucket import Bucket from .s3_artifacts import S3Artifacts +from .s3_object_index import S3ObjectIndex -__all__ = ["Bucket", "S3Artifacts"] +__all__ = ["Bucket", "S3Artifacts", "S3ObjectIndex"] diff --git a/src/gardenlinux/s3/s3_object_index.py b/src/gardenlinux/s3/s3_object_index.py new file mode 100644 index 00000000..fa80d401 --- /dev/null +++ b/src/gardenlinux/s3/s3_object_index.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- + +""" +S3 object index with flavors filtering +""" + +import base64 +import json +import logging +import os +import subprocess +import time +import yaml +from typing import Any, Optional + +from ..flavors.parser import Parser +from ..logger import LoggerSetup +from .bucket import Bucket + + +class S3ObjectIndex(object): + """ + S3 object index class with flavors filtering capabilities + + :author: Garden Linux Maintainers + :copyright: Copyright 2024 SAP SE + :package: gardenlinux + :subpackage: s3 + :since: 0.9.0 + :license: https://www.apache.org/licenses/LICENSE-2.0 + Apache License, Version 2.0 + """ + + def __init__( + self, + bucket_name: str, + endpoint_url: Optional[str] = None, + s3_resource_config: Optional[dict[str, Any]] = None, + logger: Optional[logging.Logger] = None, + ): + """ + Constructor __init__(S3ObjectIndex) + + :param bucket_name: S3 bucket name + :param endpoint_url: S3 endpoint URL + :param s3_resource_config: Additional boto3 S3 config values + :param logger: Logger instance + + :since: 0.9.0 + """ + + if logger is None or not logger.hasHandlers(): + logger = LoggerSetup.get_logger("gardenlinux.s3") + + self._bucket = Bucket(bucket_name, endpoint_url, s3_resource_config) + self._logger = logger + + def get_index( + self, + prefix: str, + cache_file: Optional[str] = None, + cache_ttl: int = 3600, + ) -> dict[str, Any]: + """ + Get and cache S3 objects with an indexed list of objects. + + :param prefix: Prefix for S3 objects + :param cache_file: Path to cache file (optional, enables caching when provided) + :param cache_ttl: Cache time-to-live in seconds + + :returns: Dictionary containing 'index' and 'artifacts' keys + + :since: 0.9.0 + """ + + self._logger.debug(f"Getting object index for prefix: {prefix}") + + # Fetch directly if no caching + if cache_file is None: + artifacts = [ + s3_object.key + for s3_object in self._bucket.objects.filter(Prefix=prefix).all() + ] + self._logger.debug(f"Fetched {len(artifacts)} artifacts without caching") + return {"index": self._build_index(artifacts), "artifacts": artifacts} + + # Check cache + index_file = cache_file + ".index.json" + if ( + os.path.exists(cache_file) + and os.path.exists(index_file) + and time.time() - os.path.getmtime(cache_file) < cache_ttl + ): + try: + with open(cache_file, "r") as f: + artifacts = json.load(f) + with open(index_file, "r") as f: + index = json.load(f) + self._logger.debug("Using cached object index") + return {"index": index, "artifacts": artifacts} + except (json.JSONDecodeError, IOError): + self._logger.warning("Cache files corrupted, fetching fresh data") + + # Fetch from S3 and cache + artifacts = [ + s3_object.key + for s3_object in self._bucket.objects.filter(Prefix=prefix).all() + ] + index = self._build_index(artifacts) + + self._logger.info(f"Fetched {len(artifacts)} artifacts from S3") + + # Save cache + try: + with open(cache_file, "w") as f: + json.dump(artifacts, f) + with open(index_file, "w") as f: + json.dump(index, f) + self._logger.debug("Saved object index to cache") + except IOError: + self._logger.warning("Failed to save cache files") + + return {"index": index, "artifacts": artifacts} + + def _build_index(self, objects: list[str]) -> dict[str, list[str]]: + """ + Build an index of objects for faster searching. + + :param objects: List of object keys + :returns: Dictionary index with simple objects list + :since: 0.9.0 + """ + + cnames = { + obj.split("/")[1] + for obj in objects + if obj.startswith("objects/") and len(obj.split("/")) >= 3 + } + self._logger.debug(f"Built index with {len(cnames)} unique objects") + return {"objects": sorted(cnames)} From 4d425ae6d1be8a334ccb3a3d0178c0e106d82675 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Mon, 11 Aug 2025 17:00:58 +0200 Subject: [PATCH 4/7] Enhance handling of `flavors.yaml` checked out from GitHub Signed-off-by: Tobias Wolf --- src/gardenlinux/constants.py | 2 + src/gardenlinux/flavors/__main__.py | 84 +++++++++++++++-------------- src/gardenlinux/flavors/parser.py | 1 - src/gardenlinux/git/git.py | 5 +- tests/flavors/test_main.py | 69 +++++++++++++----------- 5 files changed, 87 insertions(+), 74 deletions(-) diff --git a/src/gardenlinux/constants.py b/src/gardenlinux/constants.py index 9d054913..3c7090b2 100644 --- a/src/gardenlinux/constants.py +++ b/src/gardenlinux/constants.py @@ -140,6 +140,8 @@ "secureboot.aws-efivars": "application/io.gardenlinux.cert.secureboot.aws-efivars", } +GL_REPOSITORY_URL = "https://github.com/gardenlinux/gardenlinux" + OCI_ANNOTATION_SIGNATURE_KEY = "io.gardenlinux.oci.signature" OCI_ANNOTATION_SIGNED_STRING_KEY = "io.gardenlinux.oci.signed-string" OCI_IMAGE_INDEX_MEDIA_TYPE = "application/vnd.oci.image.index.v1+json" diff --git a/src/gardenlinux/flavors/__main__.py b/src/gardenlinux/flavors/__main__.py index 46c05938..d037d5d3 100644 --- a/src/gardenlinux/flavors/__main__.py +++ b/src/gardenlinux/flavors/__main__.py @@ -6,14 +6,27 @@ """ from argparse import ArgumentParser +from pathlib import Path +from tempfile import TemporaryDirectory import json import os import sys -from ..git import Git -from ..github import GitHub +from git import Repo +from git.exc import GitError from .parser import Parser +from ..constants import GL_REPOSITORY_URL +from ..git import Git + + +def _get_flavors_file_data(flavors_file): + if not flavors_file.exists(): + raise RuntimeError(f"Error: {flavors_file} does not exist.") + + # Load and validate the flavors.yaml + with flavors_file.open("r") as fp: + return fp.read() def generate_markdown_table(combinations, no_arch): @@ -52,7 +65,7 @@ def parse_args(): parser.add_argument( "--commit", default=None, - help="Commit hash to fetch flavors.yaml from GitHub (if not specified, uses local file).", + help="Commit hash to fetch flavors.yaml from GitHub. An existing 'flavors.yaml' file will be preferred.", ) parser.add_argument( "--no-arch", @@ -126,44 +139,33 @@ def main(): args = parse_args() - if args.commit: - # Use GitHub API to fetch flavors.yaml - github = GitHub() - flavors_content = github.get_flavors_yaml(commit=args.commit) - - parser = Parser(data=flavors_content) - - combinations = parser.filter( - include_only_patterns=args.include_only, - wildcard_excludes=args.exclude, - only_build=args.build, - only_test=args.test, - only_test_platform=args.test_platform, - only_publish=args.publish, - filter_categories=args.category, - exclude_categories=args.exclude_category, - ) - else: - # Use local file - flavors_file = os.path.join(Git().root, "flavors.yaml") - - if not os.path.isfile(flavors_file): - sys.exit(f"Error: {flavors_file} does not exist.") - - # Load and validate the flavors.yaml - with open(flavors_file, "r") as file: - flavors_data = file.read() - - combinations = Parser(flavors_data).filter( - include_only_patterns=args.include_only, - wildcard_excludes=args.exclude, - only_build=args.build, - only_test=args.test, - only_test_platform=args.test_platform, - only_publish=args.publish, - filter_categories=args.category, - exclude_categories=args.exclude_category, - ) + try: + flavors_data = _get_flavors_file_data(Path(Git().root, "flavors.yaml")) + except (GitError, RuntimeError): + with TemporaryDirectory() as git_directory: + repo = Repo.clone_from( + GL_REPOSITORY_URL, git_directory, no_origin=True, sparse=True + ) + + ref = repo.heads.main + + if args.commit is not None: + ref = ref.set_commit(args.commit) + + flavors_data = _get_flavors_file_data( + Path(repo.working_dir, "flavors.yaml") + ) + + combinations = Parser(flavors_data).filter( + include_only_patterns=args.include_only, + wildcard_excludes=args.exclude, + only_build=args.build, + only_test=args.test, + only_test_platform=args.test_platform, + only_publish=args.publish, + filter_categories=args.category, + exclude_categories=args.exclude_category, + ) if args.json_by_arch: grouped_combinations = Parser.group_by_arch(combinations) diff --git a/src/gardenlinux/flavors/parser.py b/src/gardenlinux/flavors/parser.py index 74f5e5d3..5ac69c8b 100644 --- a/src/gardenlinux/flavors/parser.py +++ b/src/gardenlinux/flavors/parser.py @@ -10,7 +10,6 @@ from ..constants import GL_FLAVORS_SCHEMA from ..logger import LoggerSetup -from ..github import GitHub class Parser(object): diff --git a/src/gardenlinux/git/git.py b/src/gardenlinux/git/git.py index 8bf44a9e..64524ad2 100755 --- a/src/gardenlinux/git/git.py +++ b/src/gardenlinux/git/git.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- +from os import PathLike +from pathlib import Path import sys + from git import Repo from git import Git as _Git -from os import PathLike -from pathlib import Path from ..logger import LoggerSetup diff --git a/tests/flavors/test_main.py b/tests/flavors/test_main.py index 274f22a4..4df43a51 100644 --- a/tests/flavors/test_main.py +++ b/tests/flavors/test_main.py @@ -2,7 +2,8 @@ import sys import pytest -import gardenlinux.flavors.__main__ as fm +from gardenlinux.flavors import __main__ as fm +from gardenlinux.git import Git def test_generate_markdown_table(): @@ -98,22 +99,17 @@ def remove_arch(combinations): return DummyParser -def test_main_exits_when_flavors_missing(tmp_path, monkeypatch): - # Arrange - # make Git().root point to a tmp dir that does NOT contain flavors.yaml +def _make_git_class(tmp_path): + """ + Factory to create a fake Parser class + Instances ignore the favors_data passed to __init__. + """ + class DummyGit: def __init__(self): - self.root = str(tmp_path) + self.root = tmp_path - monkeypatch.setattr(fm, "Git", DummyGit) - - # ensure no flavors.yaml - monkeypatch.setattr(sys, "argv", ["prog"]) - - # Act / Assert - with pytest.raises(SystemExit) as excinfo: - fm.main() - assert "does not exist" in str(excinfo.value) + return DummyGit def test_main_json_by_arch_prints_json(tmp_path, monkeypatch, capsys): @@ -122,14 +118,11 @@ def test_main_json_by_arch_prints_json(tmp_path, monkeypatch, capsys): flavors_file = tmp_path / "flavors.yaml" flavors_file.write_text("dummy: content") - class DummyGit: - def __init__(self): - self.root = str(tmp_path) - # define combinations and expected grouped mapping combinations = [("x86", "linux-x86"), ("arm", "android-arm")] grouped = {"x86": ["linux-x86"], "arm": ["android-arm"]} + DummyGit = _make_git_class(str(tmp_path)) DummyParser = _make_parser_class(filter_result=combinations, group_result=grouped) monkeypatch.setattr(fm, "Git", DummyGit) monkeypatch.setattr(fm, "Parser", DummyParser) @@ -151,13 +144,11 @@ def test_main_json_by_arch_with_no_arch_strips_arch_suffix( flavors_file = tmp_path / "flavors.yaml" flavors_file.write_text("dummy: content") - class DummyGit: - def __init__(self): - self.root = str(tmp_path) - combinations = [("x86", "linux-x86"), ("arm", "android-arm")] # group_by_arch returns items that include architecture suffixes grouped = {"x86": ["linux-x86"], "arm": ["android-arm"]} + + DummyGit = _make_git_class(str(tmp_path)) DummyParser = _make_parser_class(filter_result=combinations, group_result=grouped) monkeypatch.setattr(fm, "Git", DummyGit) @@ -179,11 +170,9 @@ def test_main_markdown_table_branch(tmp_path, monkeypatch, capsys): flavors_file = tmp_path / "flavors.yaml" flavors_file.write_text("dummy: content") - class DummyGit: - def __init__(self): - self.root = str(tmp_path) - combinations = [("x86_64", "linux-x86_64"), ("armv7", "android-armv7")] + + DummyGit = _make_git_class(str(tmp_path)) DummyParser = _make_parser_class(filter_result=combinations) monkeypatch.setattr(fm, "Git", DummyGit) @@ -205,12 +194,10 @@ def test_main_default_prints_flavors_list(tmp_path, monkeypatch, capsys): flavors_file = tmp_path / "flavors.yaml" flavors_file.write_text("dummy: content") - class DummyGit: - def __init__(self): - self.root = str(tmp_path) - # filter returns tuples; main's default branch prints comb[1] values, sorted unique combinations = [("x86", "linux-x86"), ("arm", "android-arm")] + + DummyGit = _make_git_class(str(tmp_path)) DummyParser = _make_parser_class(filter_result=combinations) monkeypatch.setattr(fm, "Git", DummyGit) @@ -224,3 +211,25 @@ def __init__(self): # Assert assert sorted(lines) == sorted(["linux-x86", "android-arm"]) + + +def test_main_default_prints_git_flavors_list(tmp_path, monkeypatch, capsys): + # Arrange + flavors_file = tmp_path / "flavors.yaml" + flavors_file.write_text("dummy: content") + + # filter returns tuples; main's default branch prints comb[1] values, sorted unique + combinations = [("x86", "linux-x86"), ("arm", "android-arm")] + + DummyParser = _make_parser_class(filter_result=combinations) + + monkeypatch.setattr(fm, "Parser", DummyParser) + monkeypatch.setattr(sys, "argv", ["prog"]) + + # Act + fm.main() + out = capsys.readouterr().out + lines = out.strip().splitlines() + + # Assert + assert sorted(lines) == sorted(["linux-x86", "android-arm"]) From f606f071f70340eb4372cb533c7ca3e31a6152dd Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Tue, 12 Aug 2025 09:25:16 +0200 Subject: [PATCH 5/7] Integrate `S3ObjectIndex` functionality into `gardenlinux.s3.Bucket` Signed-off-by: Tobias Wolf --- src/gardenlinux/s3/__init__.py | 3 +- src/gardenlinux/s3/bucket.py | 36 ++++++- src/gardenlinux/s3/s3_artifacts.py | 14 ++- src/gardenlinux/s3/s3_object_index.py | 140 -------------------------- tests/s3/test_bucket.py | 23 +++++ 5 files changed, 71 insertions(+), 145 deletions(-) delete mode 100644 src/gardenlinux/s3/s3_object_index.py diff --git a/src/gardenlinux/s3/__init__.py b/src/gardenlinux/s3/__init__.py index 0c6b9166..ec76b3b7 100644 --- a/src/gardenlinux/s3/__init__.py +++ b/src/gardenlinux/s3/__init__.py @@ -6,6 +6,5 @@ from .bucket import Bucket from .s3_artifacts import S3Artifacts -from .s3_object_index import S3ObjectIndex -__all__ = ["Bucket", "S3Artifacts", "S3ObjectIndex"] +__all__ = ["Bucket", "S3Artifacts"] diff --git a/src/gardenlinux/s3/bucket.py b/src/gardenlinux/s3/bucket.py index d50d4784..e383e791 100644 --- a/src/gardenlinux/s3/bucket.py +++ b/src/gardenlinux/s3/bucket.py @@ -4,10 +4,15 @@ S3 bucket """ -import boto3 +import json import logging +from os import PathLike +from pathlib import Path +from time import time from typing import Any, Optional +import boto3 + from ..logger import LoggerSetup @@ -111,6 +116,35 @@ def download_fileobj(self, key, fp, *args, **kwargs): self._logger.info(f"Downloaded {key} from S3 as binary data") + def read_cache_file_or_filter(self, cache_file, cache_ttl: int = 3600, **kwargs): + """ + Read S3 object keys from cache if valid or filter for S3 object keys. + + :param cache_file: Path to cache file + :param cache_ttl: Cache time-to-live in seconds + + :returns: S3 object keys read or filtered + + :since: 0.9.0 + """ + + if not isinstance(cache_file, PathLike): + cache_file = Path(cache_file) + + if cache_file.exists() and (time() - cache_file.stat().st_mtime) < cache_ttl: + with cache_file.open("r") as fp: + return json.loads(fp.read()) + + artifacts = [ + s3_object.key for s3_object in self._bucket.objects.filter(**kwargs).all() + ] + + if cache_file is not None: + with cache_file.open("w") as fp: + fp.write(json.dumps(artifacts)) + + return artifacts + def upload_file(self, file_name, key, *args, **kwargs): """ boto3: Upload a file to an S3 object. diff --git a/src/gardenlinux/s3/s3_artifacts.py b/src/gardenlinux/s3/s3_artifacts.py index 0bccf557..7938277a 100644 --- a/src/gardenlinux/s3/s3_artifacts.py +++ b/src/gardenlinux/s3/s3_artifacts.py @@ -16,11 +16,10 @@ from typing import Any, Optional from urllib.parse import urlencode +from .bucket import Bucket from ..features.cname import CName from ..logger import LoggerSetup -from .bucket import Bucket - class S3Artifacts(object): """ @@ -55,6 +54,17 @@ def __init__( self._bucket = Bucket(bucket_name, endpoint_url, s3_resource_config) + @property + def bucket(self): + """ + Returns the underlying S3 bucket. + + :return: (boto3.Bucket) S3 bucket + :since: 0.9.0 + """ + + return self._bucket + def download_to_directory( self, cname, diff --git a/src/gardenlinux/s3/s3_object_index.py b/src/gardenlinux/s3/s3_object_index.py deleted file mode 100644 index fa80d401..00000000 --- a/src/gardenlinux/s3/s3_object_index.py +++ /dev/null @@ -1,140 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -S3 object index with flavors filtering -""" - -import base64 -import json -import logging -import os -import subprocess -import time -import yaml -from typing import Any, Optional - -from ..flavors.parser import Parser -from ..logger import LoggerSetup -from .bucket import Bucket - - -class S3ObjectIndex(object): - """ - S3 object index class with flavors filtering capabilities - - :author: Garden Linux Maintainers - :copyright: Copyright 2024 SAP SE - :package: gardenlinux - :subpackage: s3 - :since: 0.9.0 - :license: https://www.apache.org/licenses/LICENSE-2.0 - Apache License, Version 2.0 - """ - - def __init__( - self, - bucket_name: str, - endpoint_url: Optional[str] = None, - s3_resource_config: Optional[dict[str, Any]] = None, - logger: Optional[logging.Logger] = None, - ): - """ - Constructor __init__(S3ObjectIndex) - - :param bucket_name: S3 bucket name - :param endpoint_url: S3 endpoint URL - :param s3_resource_config: Additional boto3 S3 config values - :param logger: Logger instance - - :since: 0.9.0 - """ - - if logger is None or not logger.hasHandlers(): - logger = LoggerSetup.get_logger("gardenlinux.s3") - - self._bucket = Bucket(bucket_name, endpoint_url, s3_resource_config) - self._logger = logger - - def get_index( - self, - prefix: str, - cache_file: Optional[str] = None, - cache_ttl: int = 3600, - ) -> dict[str, Any]: - """ - Get and cache S3 objects with an indexed list of objects. - - :param prefix: Prefix for S3 objects - :param cache_file: Path to cache file (optional, enables caching when provided) - :param cache_ttl: Cache time-to-live in seconds - - :returns: Dictionary containing 'index' and 'artifacts' keys - - :since: 0.9.0 - """ - - self._logger.debug(f"Getting object index for prefix: {prefix}") - - # Fetch directly if no caching - if cache_file is None: - artifacts = [ - s3_object.key - for s3_object in self._bucket.objects.filter(Prefix=prefix).all() - ] - self._logger.debug(f"Fetched {len(artifacts)} artifacts without caching") - return {"index": self._build_index(artifacts), "artifacts": artifacts} - - # Check cache - index_file = cache_file + ".index.json" - if ( - os.path.exists(cache_file) - and os.path.exists(index_file) - and time.time() - os.path.getmtime(cache_file) < cache_ttl - ): - try: - with open(cache_file, "r") as f: - artifacts = json.load(f) - with open(index_file, "r") as f: - index = json.load(f) - self._logger.debug("Using cached object index") - return {"index": index, "artifacts": artifacts} - except (json.JSONDecodeError, IOError): - self._logger.warning("Cache files corrupted, fetching fresh data") - - # Fetch from S3 and cache - artifacts = [ - s3_object.key - for s3_object in self._bucket.objects.filter(Prefix=prefix).all() - ] - index = self._build_index(artifacts) - - self._logger.info(f"Fetched {len(artifacts)} artifacts from S3") - - # Save cache - try: - with open(cache_file, "w") as f: - json.dump(artifacts, f) - with open(index_file, "w") as f: - json.dump(index, f) - self._logger.debug("Saved object index to cache") - except IOError: - self._logger.warning("Failed to save cache files") - - return {"index": index, "artifacts": artifacts} - - def _build_index(self, objects: list[str]) -> dict[str, list[str]]: - """ - Build an index of objects for faster searching. - - :param objects: List of object keys - :returns: Dictionary index with simple objects list - :since: 0.9.0 - """ - - cnames = { - obj.split("/")[1] - for obj in objects - if obj.startswith("objects/") and len(obj.split("/")) >= 3 - } - self._logger.debug(f"Built index with {len(cnames)} unique objects") - return {"objects": sorted(cnames)} diff --git a/tests/s3/test_bucket.py b/tests/s3/test_bucket.py index 94b8118d..903b7b0f 100644 --- a/tests/s3/test_bucket.py +++ b/tests/s3/test_bucket.py @@ -69,6 +69,29 @@ def test_download_file(s3_setup): assert target_path.read_text() == "some data" +def test_read_cache_file_or_filter(s3_setup): + """ + Try to read with cache + """ + + env = s3_setup + env.s3.Object(env.bucket_name, "file.txt").put(Body=b"some data") + + bucket = Bucket(env.bucket_name, s3_resource_config={"region_name": REGION}) + cache_file = env.tmp_path / "s3.cache.json" + + result = bucket.read_cache_file_or_filter(cache_file, 1, Prefix="file") + assert result == ["file.txt"] + + env.s3.Object(env.bucket_name, "file2.txt").put(Body=b"some data") + + result = bucket.read_cache_file_or_filter(cache_file, 3600, Prefix="file") + assert result == ["file.txt"] + + result = bucket.read_cache_file_or_filter(cache_file, 0, Prefix="file") + assert result == ["file.txt", "file2.txt"] + + def test_upload_fileobj(s3_setup): """ Upload a file-like in-memory object to the bucket From cf131eb93e1f5bef9fd3e5c89a10d62ce186df18 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Tue, 12 Aug 2025 10:03:49 +0200 Subject: [PATCH 6/7] Remove `gardenlinux.github` as it's functionality is not used currently Signed-off-by: Tobias Wolf --- .github/actions/features_parse/action.yml | 2 +- .github/actions/flavors_parse/action.yml | 2 +- .github/actions/setup/action.yml | 2 +- pyproject.toml | 2 +- src/gardenlinux/github/__init__.py | 9 -- src/gardenlinux/github/github.py | 172 ---------------------- 6 files changed, 4 insertions(+), 185 deletions(-) delete mode 100644 src/gardenlinux/github/__init__.py delete mode 100644 src/gardenlinux/github/github.py diff --git a/.github/actions/features_parse/action.yml b/.github/actions/features_parse/action.yml index 71797cf0..306eea80 100644 --- a/.github/actions/features_parse/action.yml +++ b/.github/actions/features_parse/action.yml @@ -11,7 +11,7 @@ outputs: runs: using: composite steps: - - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.8.9 + - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.9.0 - id: result shell: bash run: | diff --git a/.github/actions/flavors_parse/action.yml b/.github/actions/flavors_parse/action.yml index f30aa571..71e478fe 100644 --- a/.github/actions/flavors_parse/action.yml +++ b/.github/actions/flavors_parse/action.yml @@ -13,7 +13,7 @@ outputs: runs: using: composite steps: - - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.8.9 + - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.9.0 - id: matrix shell: bash run: | diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 47a974f0..0f9a19ba 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -4,7 +4,7 @@ description: Installs the given GardenLinux Python library inputs: version: description: GardenLinux Python library version - default: "0.8.9" + default: "0.9.0" python_version: description: Python version to setup default: "3.13" diff --git a/pyproject.toml b/pyproject.toml index da452fe0..c5f1c205 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gardenlinux" -version = "0.8.9" +version = "0.9.0" description = "Contains tools to work with the features directory of gardenlinux, for example deducting dependencies from feature sets or validating cnames" authors = ["Garden Linux Maintainers "] license = "Apache-2.0" diff --git a/src/gardenlinux/github/__init__.py b/src/gardenlinux/github/__init__.py deleted file mode 100644 index 8319f72a..00000000 --- a/src/gardenlinux/github/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -GitHub module -""" - -from .github import GitHub - -__all__ = ["GitHub"] diff --git a/src/gardenlinux/github/github.py b/src/gardenlinux/github/github.py deleted file mode 100644 index bd790f22..00000000 --- a/src/gardenlinux/github/github.py +++ /dev/null @@ -1,172 +0,0 @@ -# -*- coding: utf-8 -*- - -import subprocess -import json -import base64 - -from ..logger import LoggerSetup - - -class GitHub(object): - """ - GitHub operations handler using GitHub CLI. - - :author: Garden Linux Maintainers - :copyright: Copyright 2024 SAP SE - :package: gardenlinux - :subpackage: github - :since: 0.9.0 - :license: https://www.apache.org/licenses/LICENSE-2.0 - Apache License, Version 2.0 - """ - - def __init__(self, owner="gardenlinux", repo="gardenlinux", logger=None): - """ - Constructor __init__(GitHub) - - :param owner: GitHub repository owner - :param repo: GitHub repository name - :param logger: Logger instance - - :since: 0.9.0 - """ - - if logger is None or not logger.hasHandlers(): - logger = LoggerSetup.get_logger("gardenlinux.github") - - self._owner = owner - self._repo = repo - self._logger = logger - - self._logger.debug(f"GitHub initialized for {owner}/{repo}") - - def api(self, endpoint, **kwargs): - """ - Execute a GitHub API call using gh cli. - - :param endpoint: GitHub API endpoint (e.g. "/repos/owner/repo/contents/file.yaml") - :param kwargs: Additional parameters for the API call - - :return: (dict) Parsed JSON response - :since: 0.9.0 - """ - - command = ["gh", "api", endpoint] - - # Add any additional parameters to the command - for key, value in kwargs.items(): - if key.startswith("--"): - command.extend([key, str(value)]) - else: - command.extend([f"--{key}", str(value)]) - - self._logger.debug(f"Executing GitHub API call: {' '.join(command)}") - - try: - result = subprocess.run( - command, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - check=True, - ) - - return json.loads(result.stdout) - - except subprocess.CalledProcessError as e: - self._logger.error(f"GitHub API call failed: {e.stderr}") - raise RuntimeError( - f"GitHub API call failed for endpoint {endpoint}: {e.stderr}" - ) - except json.JSONDecodeError as e: - self._logger.error(f"Failed to parse GitHub API response: {e}") - raise RuntimeError(f"Failed to parse GitHub API response: {e}") - - def get_file_content(self, file_path, ref=None): - """ - Get file content from GitHub repository. - - :param file_path: Path to file in repository (e.g. "flavors.yaml") - :param ref: Git reference (commit, branch, tag). If None, uses default branch - - :return: (str) File content - :since: 0.9.0 - """ - - endpoint = f"/repos/{self._owner}/{self._repo}/contents/{file_path}" - - # Add ref parameter if specified - if ref is not None: - endpoint = f"{endpoint}?ref={ref}" - - self._logger.debug( - f"Fetching file content: {file_path} (ref: {ref or 'default'})" - ) - - try: - response = self.api(endpoint) - - # Decode base64 content - content = base64.b64decode(response["content"]).decode("utf-8") - - self._logger.debug( - f"Successfully fetched {len(content)} characters from {file_path}" - ) - - return content - - except Exception as e: - self._logger.error(f"Failed to fetch file content for {file_path}: {e}") - raise RuntimeError(f"Failed to fetch file content for {file_path}: {e}") - - def get_flavors_yaml(self, commit="latest"): - """ - Get flavors.yaml content from the repository. - - :param commit: Commit hash or "latest" for default branch - - :return: (str) flavors.yaml content - :since: 0.9.0 - """ - - ref = None if commit == "latest" else commit - commit_short = commit if commit == "latest" else commit[:8] - - self._logger.debug(f"Fetching flavors.yaml for commit {commit_short}") - - try: - content = self.get_file_content("flavors.yaml", ref=ref) - self._logger.debug( - f"Successfully fetched flavors.yaml for commit {commit_short}" - ) - return content - - except Exception as e: - self._logger.error( - f"Failed to fetch flavors.yaml for commit {commit_short}: {e}" - ) - raise RuntimeError( - f"Failed to fetch flavors.yaml for commit {commit_short}: {e}" - ) - - @property - def repository_url(self): - """ - Returns the GitHub repository URL. - - :return: (str) GitHub repository URL - :since: 0.9.0 - """ - - return f"https://github.com/{self._owner}/{self._repo}" - - @property - def api_url(self): - """ - Returns the GitHub API base URL for this repository. - - :return: (str) GitHub API base URL - :since: 0.9.0 - """ - - return f"https://api.github.com/repos/{self._owner}/{self._repo}" From 01fda54d235bba92bc373693ee897f283059be26 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Tue, 12 Aug 2025 10:17:49 +0200 Subject: [PATCH 7/7] Verify `bucket` property in `gardenlinux.s3.S3Artifacts` Signed-off-by: Tobias Wolf --- tests/s3/test_s3_artifacts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/s3/test_s3_artifacts.py b/tests/s3/test_s3_artifacts.py index 1cf6d845..a105a19d 100644 --- a/tests/s3/test_s3_artifacts.py +++ b/tests/s3/test_s3_artifacts.py @@ -22,7 +22,7 @@ def test_s3artifacts_init_success(s3_setup): s3_artifacts = S3Artifacts(env.bucket_name) # Assert - assert s3_artifacts._bucket.name == env.bucket_name + assert s3_artifacts.bucket.name == env.bucket_name def tets_s3artifacts_invalid_bucket():