diff --git a/.github/actions/features_parse/action.yml b/.github/actions/features_parse/action.yml index 71797cf0..306eea80 100644 --- a/.github/actions/features_parse/action.yml +++ b/.github/actions/features_parse/action.yml @@ -11,7 +11,7 @@ outputs: runs: using: composite steps: - - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.8.9 + - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.9.0 - id: result shell: bash run: | diff --git a/.github/actions/flavors_parse/action.yml b/.github/actions/flavors_parse/action.yml index f30aa571..71e478fe 100644 --- a/.github/actions/flavors_parse/action.yml +++ b/.github/actions/flavors_parse/action.yml @@ -13,7 +13,7 @@ outputs: runs: using: composite steps: - - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.8.9 + - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@0.9.0 - id: matrix shell: bash run: | diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 47a974f0..0f9a19ba 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -4,7 +4,7 @@ description: Installs the given GardenLinux Python library inputs: version: description: GardenLinux Python library version - default: "0.8.9" + default: "0.9.0" python_version: description: Python version to setup default: "3.13" diff --git a/pyproject.toml b/pyproject.toml index da452fe0..c5f1c205 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gardenlinux" -version = "0.8.9" +version = "0.9.0" description = "Contains tools to work with the features directory of gardenlinux, for example deducting dependencies from feature sets or validating cnames" authors = ["Garden Linux Maintainers "] license = "Apache-2.0" diff --git a/src/gardenlinux/constants.py b/src/gardenlinux/constants.py index 9d054913..3c7090b2 100644 --- a/src/gardenlinux/constants.py +++ b/src/gardenlinux/constants.py @@ -140,6 +140,8 @@ "secureboot.aws-efivars": "application/io.gardenlinux.cert.secureboot.aws-efivars", } +GL_REPOSITORY_URL = "https://github.com/gardenlinux/gardenlinux" + OCI_ANNOTATION_SIGNATURE_KEY = "io.gardenlinux.oci.signature" OCI_ANNOTATION_SIGNED_STRING_KEY = "io.gardenlinux.oci.signed-string" OCI_IMAGE_INDEX_MEDIA_TYPE = "application/vnd.oci.image.index.v1+json" diff --git a/src/gardenlinux/flavors/__main__.py b/src/gardenlinux/flavors/__main__.py index 9d7997ac..d037d5d3 100644 --- a/src/gardenlinux/flavors/__main__.py +++ b/src/gardenlinux/flavors/__main__.py @@ -6,13 +6,27 @@ """ from argparse import ArgumentParser +from pathlib import Path +from tempfile import TemporaryDirectory import json import os import sys -from ..git import Git +from git import Repo +from git.exc import GitError from .parser import Parser +from ..constants import GL_REPOSITORY_URL +from ..git import Git + + +def _get_flavors_file_data(flavors_file): + if not flavors_file.exists(): + raise RuntimeError(f"Error: {flavors_file} does not exist.") + + # Load and validate the flavors.yaml + with flavors_file.open("r") as fp: + return fp.read() def generate_markdown_table(combinations, no_arch): @@ -48,6 +62,11 @@ def parse_args(): parser = ArgumentParser(description="Parse flavors.yaml and generate combinations.") + parser.add_argument( + "--commit", + default=None, + help="Commit hash to fetch flavors.yaml from GitHub. An existing 'flavors.yaml' file will be preferred.", + ) parser.add_argument( "--no-arch", action="store_true", @@ -120,14 +139,22 @@ def main(): args = parse_args() - flavors_file = os.path.join(Git().root, "flavors.yaml") + try: + flavors_data = _get_flavors_file_data(Path(Git().root, "flavors.yaml")) + except (GitError, RuntimeError): + with TemporaryDirectory() as git_directory: + repo = Repo.clone_from( + GL_REPOSITORY_URL, git_directory, no_origin=True, sparse=True + ) - if not os.path.isfile(flavors_file): - sys.exit(f"Error: {flavors_file} does not exist.") + ref = repo.heads.main - # Load and validate the flavors.yaml - with open(flavors_file, "r") as file: - flavors_data = file.read() + if args.commit is not None: + ref = ref.set_commit(args.commit) + + flavors_data = _get_flavors_file_data( + Path(repo.working_dir, "flavors.yaml") + ) combinations = Parser(flavors_data).filter( include_only_patterns=args.include_only, diff --git a/src/gardenlinux/git/git.py b/src/gardenlinux/git/git.py index 8bf44a9e..64524ad2 100755 --- a/src/gardenlinux/git/git.py +++ b/src/gardenlinux/git/git.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- +from os import PathLike +from pathlib import Path import sys + from git import Repo from git import Git as _Git -from os import PathLike -from pathlib import Path from ..logger import LoggerSetup diff --git a/src/gardenlinux/s3/bucket.py b/src/gardenlinux/s3/bucket.py index d50d4784..e383e791 100644 --- a/src/gardenlinux/s3/bucket.py +++ b/src/gardenlinux/s3/bucket.py @@ -4,10 +4,15 @@ S3 bucket """ -import boto3 +import json import logging +from os import PathLike +from pathlib import Path +from time import time from typing import Any, Optional +import boto3 + from ..logger import LoggerSetup @@ -111,6 +116,35 @@ def download_fileobj(self, key, fp, *args, **kwargs): self._logger.info(f"Downloaded {key} from S3 as binary data") + def read_cache_file_or_filter(self, cache_file, cache_ttl: int = 3600, **kwargs): + """ + Read S3 object keys from cache if valid or filter for S3 object keys. + + :param cache_file: Path to cache file + :param cache_ttl: Cache time-to-live in seconds + + :returns: S3 object keys read or filtered + + :since: 0.9.0 + """ + + if not isinstance(cache_file, PathLike): + cache_file = Path(cache_file) + + if cache_file.exists() and (time() - cache_file.stat().st_mtime) < cache_ttl: + with cache_file.open("r") as fp: + return json.loads(fp.read()) + + artifacts = [ + s3_object.key for s3_object in self._bucket.objects.filter(**kwargs).all() + ] + + if cache_file is not None: + with cache_file.open("w") as fp: + fp.write(json.dumps(artifacts)) + + return artifacts + def upload_file(self, file_name, key, *args, **kwargs): """ boto3: Upload a file to an S3 object. diff --git a/src/gardenlinux/s3/s3_artifacts.py b/src/gardenlinux/s3/s3_artifacts.py index 0bccf557..7938277a 100644 --- a/src/gardenlinux/s3/s3_artifacts.py +++ b/src/gardenlinux/s3/s3_artifacts.py @@ -16,11 +16,10 @@ from typing import Any, Optional from urllib.parse import urlencode +from .bucket import Bucket from ..features.cname import CName from ..logger import LoggerSetup -from .bucket import Bucket - class S3Artifacts(object): """ @@ -55,6 +54,17 @@ def __init__( self._bucket = Bucket(bucket_name, endpoint_url, s3_resource_config) + @property + def bucket(self): + """ + Returns the underlying S3 bucket. + + :return: (boto3.Bucket) S3 bucket + :since: 0.9.0 + """ + + return self._bucket + def download_to_directory( self, cname, diff --git a/tests/flavors/test_main.py b/tests/flavors/test_main.py index 274f22a4..4df43a51 100644 --- a/tests/flavors/test_main.py +++ b/tests/flavors/test_main.py @@ -2,7 +2,8 @@ import sys import pytest -import gardenlinux.flavors.__main__ as fm +from gardenlinux.flavors import __main__ as fm +from gardenlinux.git import Git def test_generate_markdown_table(): @@ -98,22 +99,17 @@ def remove_arch(combinations): return DummyParser -def test_main_exits_when_flavors_missing(tmp_path, monkeypatch): - # Arrange - # make Git().root point to a tmp dir that does NOT contain flavors.yaml +def _make_git_class(tmp_path): + """ + Factory to create a fake Parser class + Instances ignore the favors_data passed to __init__. + """ + class DummyGit: def __init__(self): - self.root = str(tmp_path) + self.root = tmp_path - monkeypatch.setattr(fm, "Git", DummyGit) - - # ensure no flavors.yaml - monkeypatch.setattr(sys, "argv", ["prog"]) - - # Act / Assert - with pytest.raises(SystemExit) as excinfo: - fm.main() - assert "does not exist" in str(excinfo.value) + return DummyGit def test_main_json_by_arch_prints_json(tmp_path, monkeypatch, capsys): @@ -122,14 +118,11 @@ def test_main_json_by_arch_prints_json(tmp_path, monkeypatch, capsys): flavors_file = tmp_path / "flavors.yaml" flavors_file.write_text("dummy: content") - class DummyGit: - def __init__(self): - self.root = str(tmp_path) - # define combinations and expected grouped mapping combinations = [("x86", "linux-x86"), ("arm", "android-arm")] grouped = {"x86": ["linux-x86"], "arm": ["android-arm"]} + DummyGit = _make_git_class(str(tmp_path)) DummyParser = _make_parser_class(filter_result=combinations, group_result=grouped) monkeypatch.setattr(fm, "Git", DummyGit) monkeypatch.setattr(fm, "Parser", DummyParser) @@ -151,13 +144,11 @@ def test_main_json_by_arch_with_no_arch_strips_arch_suffix( flavors_file = tmp_path / "flavors.yaml" flavors_file.write_text("dummy: content") - class DummyGit: - def __init__(self): - self.root = str(tmp_path) - combinations = [("x86", "linux-x86"), ("arm", "android-arm")] # group_by_arch returns items that include architecture suffixes grouped = {"x86": ["linux-x86"], "arm": ["android-arm"]} + + DummyGit = _make_git_class(str(tmp_path)) DummyParser = _make_parser_class(filter_result=combinations, group_result=grouped) monkeypatch.setattr(fm, "Git", DummyGit) @@ -179,11 +170,9 @@ def test_main_markdown_table_branch(tmp_path, monkeypatch, capsys): flavors_file = tmp_path / "flavors.yaml" flavors_file.write_text("dummy: content") - class DummyGit: - def __init__(self): - self.root = str(tmp_path) - combinations = [("x86_64", "linux-x86_64"), ("armv7", "android-armv7")] + + DummyGit = _make_git_class(str(tmp_path)) DummyParser = _make_parser_class(filter_result=combinations) monkeypatch.setattr(fm, "Git", DummyGit) @@ -205,12 +194,10 @@ def test_main_default_prints_flavors_list(tmp_path, monkeypatch, capsys): flavors_file = tmp_path / "flavors.yaml" flavors_file.write_text("dummy: content") - class DummyGit: - def __init__(self): - self.root = str(tmp_path) - # filter returns tuples; main's default branch prints comb[1] values, sorted unique combinations = [("x86", "linux-x86"), ("arm", "android-arm")] + + DummyGit = _make_git_class(str(tmp_path)) DummyParser = _make_parser_class(filter_result=combinations) monkeypatch.setattr(fm, "Git", DummyGit) @@ -224,3 +211,25 @@ def __init__(self): # Assert assert sorted(lines) == sorted(["linux-x86", "android-arm"]) + + +def test_main_default_prints_git_flavors_list(tmp_path, monkeypatch, capsys): + # Arrange + flavors_file = tmp_path / "flavors.yaml" + flavors_file.write_text("dummy: content") + + # filter returns tuples; main's default branch prints comb[1] values, sorted unique + combinations = [("x86", "linux-x86"), ("arm", "android-arm")] + + DummyParser = _make_parser_class(filter_result=combinations) + + monkeypatch.setattr(fm, "Parser", DummyParser) + monkeypatch.setattr(sys, "argv", ["prog"]) + + # Act + fm.main() + out = capsys.readouterr().out + lines = out.strip().splitlines() + + # Assert + assert sorted(lines) == sorted(["linux-x86", "android-arm"]) diff --git a/tests/s3/test_bucket.py b/tests/s3/test_bucket.py index 94b8118d..903b7b0f 100644 --- a/tests/s3/test_bucket.py +++ b/tests/s3/test_bucket.py @@ -69,6 +69,29 @@ def test_download_file(s3_setup): assert target_path.read_text() == "some data" +def test_read_cache_file_or_filter(s3_setup): + """ + Try to read with cache + """ + + env = s3_setup + env.s3.Object(env.bucket_name, "file.txt").put(Body=b"some data") + + bucket = Bucket(env.bucket_name, s3_resource_config={"region_name": REGION}) + cache_file = env.tmp_path / "s3.cache.json" + + result = bucket.read_cache_file_or_filter(cache_file, 1, Prefix="file") + assert result == ["file.txt"] + + env.s3.Object(env.bucket_name, "file2.txt").put(Body=b"some data") + + result = bucket.read_cache_file_or_filter(cache_file, 3600, Prefix="file") + assert result == ["file.txt"] + + result = bucket.read_cache_file_or_filter(cache_file, 0, Prefix="file") + assert result == ["file.txt", "file2.txt"] + + def test_upload_fileobj(s3_setup): """ Upload a file-like in-memory object to the bucket diff --git a/tests/s3/test_s3_artifacts.py b/tests/s3/test_s3_artifacts.py index 1cf6d845..a105a19d 100644 --- a/tests/s3/test_s3_artifacts.py +++ b/tests/s3/test_s3_artifacts.py @@ -22,7 +22,7 @@ def test_s3artifacts_init_success(s3_setup): s3_artifacts = S3Artifacts(env.bucket_name) # Assert - assert s3_artifacts._bucket.name == env.bucket_name + assert s3_artifacts.bucket.name == env.bucket_name def tets_s3artifacts_invalid_bucket():