a missing features for GLRD and gardenlinux's tests

.github/actions/features_parse/action.yml

@@ -11,7 +11,7 @@ outputs:
 runs:
   using: composite
   steps:
-    - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.8.9
+    - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.9.0
     - id: result
       shell: bash
       run: |

.github/actions/flavors_parse/action.yml

@@ -13,7 +13,7 @@ outputs:
 runs:
   using: composite
   steps:
-    - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.8.9
+    - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.9.0
     - id: matrix
       shell: bash
       run: |

.github/actions/setup/action.yml

@@ -4,7 +4,7 @@ description: Installs the given GardenLinux Python library
 inputs:
   version:
     description: GardenLinux Python library version
-    default: "0.8.9"
+    default: "0.9.0"
   python_version:
     description: Python version to setup
     default: "3.13"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gardenlinux"
-version = "0.8.9"
+version = "0.9.0"
 description = "Contains tools to work with the features directory of gardenlinux, for example deducting dependencies from feature sets or validating cnames"
 authors = ["Garden Linux Maintainers <contact@gardenlinux.io>"]
 license = "Apache-2.0"

src/gardenlinux/constants.py

@@ -140,6 +140,8 @@
     "secureboot.aws-efivars": "application/io.gardenlinux.cert.secureboot.aws-efivars",
 }
 
+GL_REPOSITORY_URL = "https://github.com/gardenlinux/gardenlinux"
+
 OCI_ANNOTATION_SIGNATURE_KEY = "io.gardenlinux.oci.signature"
 OCI_ANNOTATION_SIGNED_STRING_KEY = "io.gardenlinux.oci.signed-string"
 OCI_IMAGE_INDEX_MEDIA_TYPE = "application/vnd.oci.image.index.v1+json"

src/gardenlinux/flavors/__main__.py

@@ -6,13 +6,27 @@
 """
 
 from argparse import ArgumentParser
+from pathlib import Path
+from tempfile import TemporaryDirectory
 import json
 import os
 import sys
 
-from ..git import Git
+from git import Repo
+from git.exc import GitError
 
 from .parser import Parser
+from ..constants import GL_REPOSITORY_URL
+from ..git import Git
+
+
+def _get_flavors_file_data(flavors_file):
+    if not flavors_file.exists():
+        raise RuntimeError(f"Error: {flavors_file} does not exist.")
+
+    # Load and validate the flavors.yaml
+    with flavors_file.open("r") as fp:
+        return fp.read()
 
 
 def generate_markdown_table(combinations, no_arch):
@@ -48,6 +62,11 @@ def parse_args():
 
     parser = ArgumentParser(description="Parse flavors.yaml and generate combinations.")
 
+    parser.add_argument(
+        "--commit",
+        default=None,
+        help="Commit hash to fetch flavors.yaml from GitHub. An existing 'flavors.yaml' file will be preferred.",
+    )
     parser.add_argument(
         "--no-arch",
         action="store_true",
@@ -120,14 +139,22 @@ def main():
 
     args = parse_args()
 
-    flavors_file = os.path.join(Git().root, "flavors.yaml")
+    try:
+        flavors_data = _get_flavors_file_data(Path(Git().root, "flavors.yaml"))
+    except (GitError, RuntimeError):
+        with TemporaryDirectory() as git_directory:
+            repo = Repo.clone_from(
+                GL_REPOSITORY_URL, git_directory, no_origin=True, sparse=True
+            )
 
-    if not os.path.isfile(flavors_file):
-        sys.exit(f"Error: {flavors_file} does not exist.")
+            ref = repo.heads.main
 
-    # Load and validate the flavors.yaml
-    with open(flavors_file, "r") as file:
-        flavors_data = file.read()
+            if args.commit is not None:
+                ref = ref.set_commit(args.commit)
+
+            flavors_data = _get_flavors_file_data(
+                Path(repo.working_dir, "flavors.yaml")
+            )
 
     combinations = Parser(flavors_data).filter(
         include_only_patterns=args.include_only,

src/gardenlinux/git/git.py

@@ -1,10 +1,11 @@
 # -*- coding: utf-8 -*-
 
+from os import PathLike
+from pathlib import Path
 import sys
+
 from git import Repo
 from git import Git as _Git
-from os import PathLike
-from pathlib import Path
 
 from ..logger import LoggerSetup
 

src/gardenlinux/s3/bucket.py

@@ -4,10 +4,15 @@
 S3 bucket
 """
 
-import boto3
+import json
 import logging
+from os import PathLike
+from pathlib import Path
+from time import time
 from typing import Any, Optional
 
+import boto3
+
 from ..logger import LoggerSetup
 
 
@@ -111,6 +116,35 @@ def download_fileobj(self, key, fp, *args, **kwargs):
 
         self._logger.info(f"Downloaded {key} from S3 as binary data")
 
+    def read_cache_file_or_filter(self, cache_file, cache_ttl: int = 3600, **kwargs):
+        """
+        Read S3 object keys from cache if valid or filter for S3 object keys.
+
+        :param cache_file: Path to cache file
+        :param cache_ttl:  Cache time-to-live in seconds
+
+        :returns: S3 object keys read or filtered
+
+        :since: 0.9.0
+        """
+
+        if not isinstance(cache_file, PathLike):
+            cache_file = Path(cache_file)
+
+        if cache_file.exists() and (time() - cache_file.stat().st_mtime) < cache_ttl:
+            with cache_file.open("r") as fp:
+                return json.loads(fp.read())
+
+        artifacts = [
+            s3_object.key for s3_object in self._bucket.objects.filter(**kwargs).all()
+        ]
+
+        if cache_file is not None:
+            with cache_file.open("w") as fp:
+                fp.write(json.dumps(artifacts))
+
+        return artifacts
+
     def upload_file(self, file_name, key, *args, **kwargs):
         """
         boto3: Upload a file to an S3 object.

src/gardenlinux/s3/s3_artifacts.py

@@ -16,11 +16,10 @@
 from typing import Any, Optional
 from urllib.parse import urlencode
 
+from .bucket import Bucket
 from ..features.cname import CName
 from ..logger import LoggerSetup
 
-from .bucket import Bucket
-
 
 class S3Artifacts(object):
     """
@@ -55,6 +54,17 @@ def __init__(
 
         self._bucket = Bucket(bucket_name, endpoint_url, s3_resource_config)
 
+    @property
+    def bucket(self):
+        """
+        Returns the underlying S3 bucket.
+
+        :return: (boto3.Bucket) S3 bucket
+        :since:  0.9.0
+        """
+
+        return self._bucket
+
     def download_to_directory(
         self,
         cname,

tests/flavors/test_main.py

@@ -2,7 +2,8 @@
 import sys
 import pytest
 
-import gardenlinux.flavors.__main__ as fm
+from gardenlinux.flavors import __main__ as fm
+from gardenlinux.git import Git
 
 
 def test_generate_markdown_table():
@@ -98,22 +99,17 @@ def remove_arch(combinations):
     return DummyParser
 
 
-def test_main_exits_when_flavors_missing(tmp_path, monkeypatch):
-    # Arrange
-    # make Git().root point to a tmp dir that does NOT contain flavors.yaml
+def _make_git_class(tmp_path):
+    """
+    Factory to create a fake Parser class
+    Instances ignore the favors_data passed to __init__.
+    """
+
     class DummyGit:
         def __init__(self):
-            self.root = str(tmp_path)
+            self.root = tmp_path
 
-    monkeypatch.setattr(fm, "Git", DummyGit)
-
-    # ensure no flavors.yaml
-    monkeypatch.setattr(sys, "argv", ["prog"])
-
-    # Act / Assert
-    with pytest.raises(SystemExit) as excinfo:
-        fm.main()
-    assert "does not exist" in str(excinfo.value)
+    return DummyGit
 
 
 def test_main_json_by_arch_prints_json(tmp_path, monkeypatch, capsys):
@@ -122,14 +118,11 @@ def test_main_json_by_arch_prints_json(tmp_path, monkeypatch, capsys):
     flavors_file = tmp_path / "flavors.yaml"
     flavors_file.write_text("dummy: content")
 
-    class DummyGit:
-        def __init__(self):
-            self.root = str(tmp_path)
-
     # define combinations and expected grouped mapping
     combinations = [("x86", "linux-x86"), ("arm", "android-arm")]
     grouped = {"x86": ["linux-x86"], "arm": ["android-arm"]}
 
+    DummyGit = _make_git_class(str(tmp_path))
     DummyParser = _make_parser_class(filter_result=combinations, group_result=grouped)
     monkeypatch.setattr(fm, "Git", DummyGit)
     monkeypatch.setattr(fm, "Parser", DummyParser)
@@ -151,13 +144,11 @@ def test_main_json_by_arch_with_no_arch_strips_arch_suffix(
     flavors_file = tmp_path / "flavors.yaml"
     flavors_file.write_text("dummy: content")
 
-    class DummyGit:
-        def __init__(self):
-            self.root = str(tmp_path)
-
     combinations = [("x86", "linux-x86"), ("arm", "android-arm")]
     # group_by_arch returns items that include architecture suffixes
     grouped = {"x86": ["linux-x86"], "arm": ["android-arm"]}
+
+    DummyGit = _make_git_class(str(tmp_path))
     DummyParser = _make_parser_class(filter_result=combinations, group_result=grouped)
 
     monkeypatch.setattr(fm, "Git", DummyGit)
@@ -179,11 +170,9 @@ def test_main_markdown_table_branch(tmp_path, monkeypatch, capsys):
     flavors_file = tmp_path / "flavors.yaml"
     flavors_file.write_text("dummy: content")
 
-    class DummyGit:
-        def __init__(self):
-            self.root = str(tmp_path)
-
     combinations = [("x86_64", "linux-x86_64"), ("armv7", "android-armv7")]
+
+    DummyGit = _make_git_class(str(tmp_path))
     DummyParser = _make_parser_class(filter_result=combinations)
 
     monkeypatch.setattr(fm, "Git", DummyGit)
@@ -205,12 +194,10 @@ def test_main_default_prints_flavors_list(tmp_path, monkeypatch, capsys):
     flavors_file = tmp_path / "flavors.yaml"
     flavors_file.write_text("dummy: content")
 
-    class DummyGit:
-        def __init__(self):
-            self.root = str(tmp_path)
-
     # filter returns tuples; main's default branch prints comb[1] values, sorted unique
     combinations = [("x86", "linux-x86"), ("arm", "android-arm")]
+
+    DummyGit = _make_git_class(str(tmp_path))
     DummyParser = _make_parser_class(filter_result=combinations)
 
     monkeypatch.setattr(fm, "Git", DummyGit)
@@ -224,3 +211,25 @@ def __init__(self):
 
     # Assert
     assert sorted(lines) == sorted(["linux-x86", "android-arm"])
+
+
+def test_main_default_prints_git_flavors_list(tmp_path, monkeypatch, capsys):
+    # Arrange
+    flavors_file = tmp_path / "flavors.yaml"
+    flavors_file.write_text("dummy: content")
+
+    # filter returns tuples; main's default branch prints comb[1] values, sorted unique
+    combinations = [("x86", "linux-x86"), ("arm", "android-arm")]
+
+    DummyParser = _make_parser_class(filter_result=combinations)
+
+    monkeypatch.setattr(fm, "Parser", DummyParser)
+    monkeypatch.setattr(sys, "argv", ["prog"])
+
+    # Act
+    fm.main()
+    out = capsys.readouterr().out
+    lines = out.strip().splitlines()
+
+    # Assert
+    assert sorted(lines) == sorted(["linux-x86", "android-arm"])

tests/s3/test_bucket.py

@@ -69,6 +69,29 @@ def test_download_file(s3_setup):
     assert target_path.read_text() == "some data"
 
 
+def test_read_cache_file_or_filter(s3_setup):
+    """
+    Try to read with cache
+    """
+
+    env = s3_setup
+    env.s3.Object(env.bucket_name, "file.txt").put(Body=b"some data")
+
+    bucket = Bucket(env.bucket_name, s3_resource_config={"region_name": REGION})
+    cache_file = env.tmp_path / "s3.cache.json"
+
+    result = bucket.read_cache_file_or_filter(cache_file, 1, Prefix="file")
+    assert result == ["file.txt"]
+
+    env.s3.Object(env.bucket_name, "file2.txt").put(Body=b"some data")
+
+    result = bucket.read_cache_file_or_filter(cache_file, 3600, Prefix="file")
+    assert result == ["file.txt"]
+
+    result = bucket.read_cache_file_or_filter(cache_file, 0, Prefix="file")
+    assert result == ["file.txt", "file2.txt"]
+
+
 def test_upload_fileobj(s3_setup):
     """
     Upload a file-like in-memory object to the bucket

tests/s3/test_s3_artifacts.py

@@ -22,7 +22,7 @@ def test_s3artifacts_init_success(s3_setup):
     s3_artifacts = S3Artifacts(env.bucket_name)
 
     # Assert
-    assert s3_artifacts._bucket.name == env.bucket_name
+    assert s3_artifacts.bucket.name == env.bucket_name
 
 
 def tets_s3artifacts_invalid_bucket():