add S3ObjectIndex Class

Author: yeoldegrove

src/gardenlinux/s3/__init__.py

@@ -6,5 +6,6 @@
 
 from .bucket import Bucket
 from .s3_artifacts import S3Artifacts
+from .s3_object_index import S3ObjectIndex
 
-__all__ = ["Bucket", "S3Artifacts"]
+__all__ = ["Bucket", "S3Artifacts", "S3ObjectIndex"]

src/gardenlinux/s3/s3_object_index.py

@@ -0,0 +1,286 @@
+# -*- coding: utf-8 -*-
+
+"""
+S3 object index with flavors filtering
+"""
+
+import base64
+import json
+import logging
+import os
+import subprocess
+import time
+import yaml
+from typing import Any, Optional
+
+from ..flavors.parser import Parser
+from ..logger import LoggerSetup
+from .bucket import Bucket
+
+
+class S3ObjectIndex(object):
+    """
+    S3 object index class with flavors filtering capabilities
+
+    :author:     Garden Linux Maintainers
+    :copyright:  Copyright 2024 SAP SE
+    :package:    gardenlinux
+    :subpackage: s3
+    :since:      0.9.0
+    :license:    https://www.apache.org/licenses/LICENSE-2.0
+                 Apache License, Version 2.0
+    """
+
+    def __init__(
+        self,
+        bucket_name: str,
+        endpoint_url: Optional[str] = None,
+        s3_resource_config: Optional[dict[str, Any]] = None,
+        logger: Optional[logging.Logger] = None,
+    ):
+        """
+        Constructor __init__(S3ObjectIndex)
+
+        :param bucket_name: S3 bucket name
+        :param endpoint_url: S3 endpoint URL
+        :param s3_resource_config: Additional boto3 S3 config values
+        :param logger: Logger instance
+
+        :since: 0.9.0
+        """
+
+        if logger is None or not logger.hasHandlers():
+            logger = LoggerSetup.get_logger("gardenlinux.s3")
+
+        self._bucket = Bucket(bucket_name, endpoint_url, s3_resource_config)
+        self._logger = logger
+
+    def get_index(
+        self,
+        prefix: str,
+        cache_file: Optional[str] = None,
+        cache_ttl: int = 3600,
+    ) -> dict[str, Any]:
+        """
+        Get and cache S3 objects with an indexed list of objects.
+
+        :param prefix:     Prefix for S3 objects
+        :param cache_file: Path to cache file (optional, enables caching when provided)
+        :param cache_ttl:  Cache time-to-live in seconds
+
+        :returns: Dictionary containing 'index' and 'artifacts' keys
+
+        :since: 0.9.0
+        """
+
+        self._logger.debug(f"Getting object index for prefix: {prefix}")
+
+        # Fetch directly if no caching
+        if cache_file is None:
+            artifacts = [s3_object.key for s3_object in self._bucket.objects.filter(Prefix=prefix).all()]
+            self._logger.debug(f"Fetched {len(artifacts)} artifacts without caching")
+            return {"index": self._build_index(artifacts), "artifacts": artifacts}
+
+        # Check cache
+        index_file = cache_file + ".index.json"
+        if (os.path.exists(cache_file) and os.path.exists(index_file) and
+            time.time() - os.path.getmtime(cache_file) < cache_ttl):
+            try:
+                with open(cache_file, "r") as f:
+                    artifacts = json.load(f)
+                with open(index_file, "r") as f:
+                    index = json.load(f)
+                self._logger.debug("Using cached object index")
+                return {"index": index, "artifacts": artifacts}
+            except (json.JSONDecodeError, IOError):
+                self._logger.warning("Cache files corrupted, fetching fresh data")
+
+        # Fetch from S3 and cache
+        artifacts = [s3_object.key for s3_object in self._bucket.objects.filter(Prefix=prefix).all()]
+        index = self._build_index(artifacts)
+
+        self._logger.info(f"Fetched {len(artifacts)} artifacts from S3")
+
+        # Save cache
+        try:
+            with open(cache_file, "w") as f:
+                json.dump(artifacts, f)
+            with open(index_file, "w") as f:
+                json.dump(index, f)
+            self._logger.debug("Saved object index to cache")
+        except IOError:
+            self._logger.warning("Failed to save cache files")
+
+        return {"index": index, "artifacts": artifacts}
+
+    def filter_by_commit(
+        self,
+        commit: str = "latest",
+        prefix: str = "objects/",
+        include_only_patterns: list = [],
+        wildcard_excludes: list = [],
+        only_build: bool = False,
+        only_test: bool = False,
+        only_test_platform: bool = False,
+        only_publish: bool = False,
+        filter_categories: list = [],
+        exclude_categories: list = [],
+        cache_file: Optional[str] = None,
+        cache_ttl: int = 3600,
+    ) -> dict[str, Any]:
+        """
+        Filter S3 objects based on flavors.yaml from a specific commit.
+
+        :param commit:                Git commit hash or "latest"
+        :param prefix:                S3 prefix for objects
+        :param include_only_patterns: Include pattern list
+        :param wildcard_excludes:     Exclude wildcard list
+        :param only_build:            Return only build-enabled flavors
+        :param only_test:             Return only test-enabled flavors
+        :param only_test_platform:    Return only platform-test-enabled flavors
+        :param only_publish:          Return only flavors to be published
+        :param filter_categories:     List of categories to include
+        :param exclude_categories:    List of categories to exclude
+        :param cache_file:            Path to cache file (optional)
+        :param cache_ttl:             Cache time-to-live in seconds
+
+        :returns: Dictionary containing filtered flavors and artifacts
+
+        :since: 0.9.0
+        """
+
+        commit_short = commit[:8] if commit != "latest" else "latest"
+        self._logger.debug(f"Filtering objects by commit {commit_short}")
+
+        # Get flavors data from GitHub
+        flavors_data = self._fetch_flavors_from_github(commit)
+        if not flavors_data:
+            self._logger.warning(f"Could not fetch flavors.yaml for commit {commit_short}")
+            return {"filtered_combinations": [], "artifacts": [], "index": {}}
+
+        # Parse flavors and apply filters
+        try:
+            parser = Parser(flavors_data, logger=self._logger)
+            filtered_combinations = parser.filter(
+                include_only_patterns=include_only_patterns,
+                wildcard_excludes=wildcard_excludes,
+                only_build=only_build,
+                only_test=only_test,
+                only_test_platform=only_test_platform,
+                only_publish=only_publish,
+                filter_categories=filter_categories,
+                exclude_categories=exclude_categories,
+            )
+        except Exception as e:
+            self._logger.error(f"Error parsing flavors data: {e}")
+            return {"filtered_combinations": [], "artifacts": [], "index": {}}
+
+        # Get S3 objects
+        s3_data = self.get_index(prefix, cache_file, cache_ttl)
+        artifacts = s3_data.get("artifacts", [])
+        index = s3_data.get("index", {})
+
+        # Filter artifacts based on flavors
+        filtered_artifacts = self._filter_artifacts_by_flavors(artifacts, filtered_combinations)
+
+        self._logger.info(f"Filtered {len(artifacts)} artifacts to {len(filtered_artifacts)} based on commit {commit_short}")
+
+        return {
+            "filtered_combinations": filtered_combinations,
+            "filtered_artifacts": filtered_artifacts,
+            "all_artifacts": artifacts,
+            "index": index,
+            "commit": commit,
+        }
+
+    def _fetch_flavors_from_github(self, commit: str) -> Optional[str]:
+        """
+        Fetch flavors.yaml from GitHub for a specific commit.
+
+        :param commit: Git commit hash or "latest"
+        :returns: Flavors YAML content as string, or None if failed
+
+        :since: 0.9.0
+        """
+
+        try:
+            # Try flavors.yaml first
+            api_path = "/repos/gardenlinux/gardenlinux/contents/flavors.yaml"
+            if commit != "latest":
+                api_path = f"{api_path}?ref={commit}"
+
+            command = ["gh", "api", api_path]
+            commit_short = commit[:8] if commit != "latest" else "latest"
+            self._logger.debug(f"Fetching flavors.yaml from GitHub for commit {commit_short}")
+
+            result = subprocess.run(
+                command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
+            )
+
+            if result.returncode != 0:
+                self._logger.warning(f"Failed to fetch flavors.yaml: {result.stderr}")
+                return None
+
+            # Parse GitHub API response
+            response_data = json.loads(result.stdout)
+            if "content" not in response_data:
+                self._logger.warning("No content field in GitHub API response")
+                return None
+
+            # Decode base64 content
+            flavors_content = base64.b64decode(response_data["content"]).decode("utf-8")
+            self._logger.debug(f"Successfully fetched flavors.yaml for commit {commit_short}")
+            return flavors_content
+
+        except subprocess.CalledProcessError as e:
+            self._logger.error(f"GitHub CLI command failed: {e}")
+            return None
+        except json.JSONDecodeError as e:
+            self._logger.error(f"Failed to parse GitHub API response: {e}")
+            return None
+        except Exception as e:
+            self._logger.error(f"Unexpected error fetching flavors.yaml: {e}")
+            return None
+
+    def _filter_artifacts_by_flavors(self, artifacts: list[str], combinations: list[tuple]) -> list[str]:
+        """
+        Filter artifacts based on flavor combinations.
+
+        :param artifacts: List of S3 artifact keys
+        :param combinations: List of (arch, combination) tuples from flavors filter
+        :returns: Filtered list of artifacts
+
+        :since: 0.9.0
+        """
+
+        if not combinations:
+            return []
+
+        # Extract combination names (without arch prefix)
+        combination_names = {combo[1] for combo in combinations}
+
+        filtered_artifacts = []
+        for artifact in artifacts:
+            if artifact.startswith("objects/"):
+                # Format: objects/{cname}/filename
+                parts = artifact.split("/")
+                if len(parts) >= 3:
+                    cname = parts[1]
+                    # Check if this cname matches any of the filtered combinations
+                    if any(cname.startswith(combo_name) for combo_name in combination_names):
+                        filtered_artifacts.append(artifact)
+
+        return filtered_artifacts
+
+    def _build_index(self, objects: list[str]) -> dict[str, list[str]]:
+        """
+        Build an index of objects for faster searching.
+
+        :param objects: List of object keys
+        :returns: Dictionary index with simple objects list
+        :since: 0.9.0
+        """
+
+        cnames = {obj.split("/")[1] for obj in objects if obj.startswith("objects/") and len(obj.split("/")) >= 3}
+        self._logger.debug(f"Built index with {len(cnames)} unique objects")
+        return {"objects": sorted(cnames)}