From c635266b80fd0fc0b5d49238ee5839740156a969 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Fri, 18 Apr 2025 13:20:43 +0200 Subject: [PATCH 01/21] Fix `parse_flavors` result to comply expected formats and `flags` handling Signed-off-by: Tobias Wolf --- pyproject.toml | 7 +- src/python_gardenlinux_lib/constants.py | 38 +- .../flavors/__main__.py | 140 +++++++ .../flavors/parse_flavors.py | 388 +++--------------- src/python_gardenlinux_lib/flavors/parser.py | 103 +++++ 5 files changed, 337 insertions(+), 339 deletions(-) create mode 100644 src/python_gardenlinux_lib/flavors/__main__.py create mode 100644 src/python_gardenlinux_lib/flavors/parser.py diff --git a/pyproject.toml b/pyproject.toml index b3310d97..f4837c87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ python = "^3.10" networkx = "^3.3" PyYAML = "^6.0.2" pytest = "^8.3.2" -gitpython = "^3.1.43" +gitpython = "^3.1.44" sphinx-rtd-theme = "^2.0.0" apt-repo = "^0.5" jsonschema = "^4.23.0" @@ -26,9 +26,8 @@ boto3 = "*" black = "^24.8.0" [tool.poetry.scripts] -gl-cname = "python_gardenlinux_lib.cname:main" -gl-flavors-parse = "python_gardenlinux_lib.flavors.parse_flavors:main" -flavors-parse = "python_gardenlinux_lib.flavors.parse_flavors:main" +gl-flavors-parse = "python_gardenlinux_lib.flavors.__main__:main" +flavors-parse = "python_gardenlinux_lib.flavors.__main__:main" [tool.pytest.ini_options] pythonpath = [ diff --git a/src/python_gardenlinux_lib/constants.py b/src/python_gardenlinux_lib/constants.py index ffd4494d..55219ec9 100644 --- a/src/python_gardenlinux_lib/constants.py +++ b/src/python_gardenlinux_lib/constants.py @@ -1,4 +1,40 @@ -#!/usr/bin/env python3 +# GardenLinux flavors schema for validation +GL_FLAVORS_SCHEMA = { + "type": "object", + "version": {"type": "integer"}, + "properties": { + "targets": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "category": {"type": "string"}, + "flavors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "features": { + "type": "array", + "items": {"type": "string"}, + }, + "arch": {"type": "string"}, + "build": {"type": "boolean"}, + "test": {"type": "boolean"}, + "test-platform": {"type": "boolean"}, + "publish": {"type": "boolean"}, + }, + "required": ["features", "arch", "build", "test", "test-platform", "publish"], + }, + }, + }, + "required": ["name", "category", "flavors"], + }, + }, + }, + "required": ["targets"] +} # It is important that this list is sorted in descending length of the entries GL_MEDIA_TYPES = [ diff --git a/src/python_gardenlinux_lib/flavors/__main__.py b/src/python_gardenlinux_lib/flavors/__main__.py new file mode 100644 index 00000000..dc7a7564 --- /dev/null +++ b/src/python_gardenlinux_lib/flavors/__main__.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 + +from argparse import ArgumentParser +from git import Git +import json +import os +import sys +import yaml + +from jsonschema import ValidationError +from .parser import group_by_arch, parse_flavors, validate_flavors + + +def generate_markdown_table(combinations, no_arch): + """Generate a markdown table of platforms and their flavors.""" + table = "| Platform | Architecture | Flavor |\n" + table += "|------------|--------------------|------------------------------------------|\n" + + for arch, combination in combinations: + platform = combination.split("-")[0] + table += f"| {platform:<10} | {arch:<18} | `{combination}` |\n" + + return table + +def parse_args(): + parser = ArgumentParser(description="Parse flavors.yaml and generate combinations.") + + parser.add_argument("--no-arch", action="store_true", help="Exclude architecture from the flavor output.") + parser.add_argument( + "--include-only", + action="append", + default=[], + help="Restrict combinations to those matching wildcard patterns (can be specified multiple times)." + ) + parser.add_argument( + "--exclude", + action="append", + default=[], + help="Exclude combinations based on wildcard patterns (can be specified multiple times)." + ) + parser.add_argument( + "--build", + action="store_true", + help="Filter combinations to include only those with build enabled." + ) + parser.add_argument( + "--publish", + action="store_true", + help="Filter combinations to include only those with publish enabled." + ) + parser.add_argument( + "--test", + action="store_true", + help="Filter combinations to include only those with test enabled." + ) + parser.add_argument( + "--test-platform", + action="store_true", + help="Filter combinations to include only platforms with test-platform: true." + ) + parser.add_argument( + "--category", + action="append", + default=[], + help="Filter combinations to include only platforms belonging to the specified categories (can be specified multiple times)." + ) + parser.add_argument( + "--exclude-category", + action="append", + default=[], + help="Exclude platforms belonging to the specified categories (can be specified multiple times)." + ) + parser.add_argument( + "--json-by-arch", + action="store_true", + help="Output a JSON dictionary where keys are architectures and values are lists of flavors." + ) + parser.add_argument( + "--markdown-table-by-platform", + action="store_true", + help="Generate a markdown table by platform." + ) + + return parser.parse_args() + +def main(): + args = parse_args() + + repo_path = Git(".").rev_parse("--show-superproject-working-tree") + flavors_file = os.path.join(repo_path, 'flavors.yaml') + + if not os.path.isfile(flavors_file): + sys.exit(f"Error: {flavors_file} does not exist.") + + # Load and validate the flavors.yaml + with open(flavors_file, 'r') as file: + flavors_data = yaml.safe_load(file) + + try: + validate_flavors(flavors_data) + except ValidationError as e: + sys.exit(f"Validation Error: {e.message}") + + combinations = parse_flavors( + flavors_data, + include_only_patterns=args.include_only, + wildcard_excludes=args.exclude, + only_build=args.build, + only_test=args.test, + only_test_platform=args.test_platform, + only_publish=args.publish, + filter_categories=args.category, + exclude_categories=args.exclude_category + ) + + if args.json_by_arch: + grouped_combinations = group_by_arch(combinations) + + # If --no-arch, strip architectures from the grouped output + if args.no_arch: + grouped_combinations = { + arch: sorted(set(item.replace(f"-{arch}", "") for item in items)) + for arch, items in grouped_combinations.items() + } + + print(json.dumps(grouped_combinations, indent=2)) + elif args.markdown_table_by_platform: + print(generate_markdown_table(combinations, args.no_arch)) + else: + if args.no_arch: + printable_combinations = sorted(set(remove_arch(combinations))) + else: + printable_combinations = sorted(set(comb[1] for comb in combinations)) + + print("\n".join(sorted(set(printable_combinations)))) + + +if __name__ == "__main__": + # Create a null logger as default + main() diff --git a/src/python_gardenlinux_lib/flavors/parse_flavors.py b/src/python_gardenlinux_lib/flavors/parse_flavors.py index d236051d..ab152497 100755 --- a/src/python_gardenlinux_lib/flavors/parse_flavors.py +++ b/src/python_gardenlinux_lib/flavors/parse_flavors.py @@ -1,224 +1,40 @@ #!/usr/bin/env python -import argparse + +from git import Git import base64 -import fnmatch import json import logging import os -import re import subprocess -import sys -import time - -import boto3 import yaml -from botocore.exceptions import ClientError -from jsonschema import validate, ValidationError -# Create a null logger as default -null_logger = logging.getLogger("gardenlinux.lib.flavors") -null_logger.addHandler(logging.NullHandler()) +from .parser import ( + group_by_arch, + remove_arch, + should_include_only, + should_exclude, + validate_flavors +) + +from .parser import parse_flavors as parse_flavors_data + +from ..constants import GL_FLAVORS_SCHEMA # Define the schema for validation -SCHEMA = { - "type": "object", - "properties": { - "targets": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": {"type": "string"}, - "category": {"type": "string"}, - "flavors": { - "type": "array", - "items": { - "type": "object", - "properties": { - "features": { - "type": "array", - "items": {"type": "string"}, - }, - "arch": {"type": "string"}, - "build": {"type": "boolean"}, - "test": {"type": "boolean"}, - "test-platform": {"type": "boolean"}, - "publish": {"type": "boolean"}, - }, - "required": [ - "features", - "arch", - "build", - "test", - "test-platform", - "publish", - ], - }, - }, - }, - "required": ["name", "category", "flavors"], - }, - }, - }, - "required": ["targets"], -} +SCHEMA = GL_FLAVORS_SCHEMA def find_repo_root(): """Finds the root directory of the Git repository.""" - try: - root = subprocess.check_output( - ["git", "rev-parse", "--show-toplevel"], text=True - ).strip() - return root - except subprocess.CalledProcessError: - sys.exit("Error: Unable to determine Git repository root.") - - -def validate_flavors(data): - """Validate the flavors.yaml data against the schema.""" - try: - validate(instance=data, schema=SCHEMA) - except ValidationError as e: - sys.exit(f"Validation Error: {e.message}") - - -def should_exclude(combination, excludes, wildcard_excludes): - """ - Checks if a combination should be excluded based on exact match or wildcard patterns. - """ - # Exclude if in explicit excludes - if combination in excludes: - return True - # Exclude if matches any wildcard pattern - return any(fnmatch.fnmatch(combination, pattern) for pattern in wildcard_excludes) - - -def should_include_only(combination, include_only_patterns): - """ - Checks if a combination should be included based on `--include-only` wildcard patterns. - If no patterns are provided, all combinations are included by default. - """ - if not include_only_patterns: - return True - return any( - fnmatch.fnmatch(combination, pattern) for pattern in include_only_patterns - ) - - -def parse_flavors_data( - data, - include_only_patterns=None, - wildcard_excludes=None, - only_build=False, - only_test=False, - only_test_platform=False, - only_publish=False, - filter_categories=None, - exclude_categories=None, -): - """Parse flavors.yaml data and generate combinations.""" - combinations = [] - - # Validate input data against schema - validate_flavors(data) - - # Process each target platform - for target in data["targets"]: - platform_name = target["name"] - platform_category = target["category"] - - # Skip if platform category is in exclude list - if exclude_categories and platform_category in exclude_categories: - continue - - # Skip if filtering by category and platform category not in filter list - if filter_categories and platform_category not in filter_categories: - continue - - # Skip if platform test-platform flag doesn't match filter - if only_test_platform and not target.get("test-platform", False): - continue - - # Process each flavor configuration for this platform - for flavor in target["flavors"]: - # Skip if build/test/publish flags don't match filters - if only_build and not flavor.get("build", False): - continue - if only_test and not flavor.get("test", False): - continue - if only_publish and not flavor.get("publish", False): - continue - - # Generate the flavor string with architecture - features = flavor["features"] - arch = flavor["arch"] - - # Build the flavor string - if features: - # Sort features to ensure consistent order - # Remove any leading/trailing underscores and handle special cases - cleaned_features = [] - for feature in sorted(features): - # Remove leading/trailing underscores - feature = feature.strip("_") - # Handle special cases (like 'gardener' that should come first) - if feature == "gardener": - cleaned_features.insert(0, feature) - else: - cleaned_features.append(feature) - - # Join features with underscores - feature_string = "_".join(cleaned_features) - # Combine platform and features - combination = f"{platform_name}-{feature_string}-{arch}" - else: - combination = f"{platform_name}-{arch}" - - # Add to combinations if it matches include patterns and doesn't match exclude patterns - if should_include_only( - combination, include_only_patterns or [] - ) and not should_exclude(combination, wildcard_excludes or [], []): - combinations.append((arch, combination)) - - return combinations - - -def group_by_arch(combinations): - """Groups combinations by architecture into a JSON dictionary.""" - arch_dict = {} - for arch, combination in combinations: - arch_dict.setdefault(arch, []).append(combination) - for arch in arch_dict: - arch_dict[arch] = sorted(set(arch_dict[arch])) # Deduplicate and sort - return arch_dict - - -def remove_arch(combinations): - """Removes the architecture from combinations.""" - return [combination.replace(f"-{arch}", "") for arch, combination in combinations] - - -def generate_markdown_table(combinations, no_arch): - """Generate a markdown table of platforms and their flavors.""" - table = "| Platform | Architecture | Flavor |\n" - table += "|------------|--------------------|------------------------------------------|\n" - - for arch, combination in combinations: - platform = combination.split("-")[0] - table += ( - f"| {platform:<10} | {arch:<18} | `{combination}` |\n" - ) - - return table + return Git(".").rev_parse("--show-superproject-working-tree") def parse_flavors_commit( commit=None, version=None, query_s3=False, s3_objects=None, - logger=null_logger, + logger=None, include_only_patterns=None, wildcard_excludes=None, only_build=False, @@ -257,8 +73,44 @@ def parse_flavors_commit( commit = "latest" commit_short = commit[:8] - logger.debug( - f"Checking flavors for version {version_info} (commit {commit_short})" + if logger is None: + logger = logging.getLogger("gardenlinux.lib.flavors") + logger.addHandler(logging.NullHandler()) + + version_info = ( + f"{version['major']}.{version.get('minor', 0)}" if version else "unknown" + ) + if commit is None: + commit = "latest" + commit_short = commit[:8] + + logger.debug( + f"Checking flavors for version {version_info} (commit {commit_short})" + ) + + flavors_content = None + + if os.access("./flavors.yaml", os.F_OK | os.R_OK): + with open("./flavors.yaml", "r") as fp: + flavors_content = fp.read() + else: + try: + flavors_content = _get_flavors_from_github() + except Exception as exc: + logger.debug(exc) + + if flavors_content is not None: + # Parse flavors with all filters + combinations = parse_flavors_data( + yaml.safe_load(flavors_content), + include_only_patterns=include_only_patterns or [], + wildcard_excludes=wildcard_excludes or [], + only_build=only_build, + only_test=only_test, + only_test_platform=only_test_platform, + only_publish=only_publish, + filter_categories=filter_categories or [], + exclude_categories=exclude_categories or [], ) # Try flavors.yaml first @@ -357,135 +209,3 @@ def parse_flavors_commit( except Exception as e: logger.error(f"Error parsing flavors for commit {commit_short}: {e}") return [] - - -def parse_arguments(): - parser = argparse.ArgumentParser( - description="Parse flavors.yaml and generate combinations." - ) - parser.add_argument( - "--commit", - type=str, - default="latest", - help="The git commit hash (short or long) to use.", - ) - parser.add_argument( - "--no-arch", - action="store_true", - help="Exclude architecture from the flavor output.", - ) - parser.add_argument( - "--include-only", - action="append", - help="Restrict combinations to those matching wildcard patterns (can be specified multiple times).", - ) - parser.add_argument( - "--exclude", - action="append", - help="Exclude combinations based on wildcard patterns (can be specified multiple times).", - ) - parser.add_argument( - "--build", - action="store_true", - help="Filter combinations to include only those with build enabled.", - ) - parser.add_argument( - "--test", - action="store_true", - help="Filter combinations to include only those with test enabled.", - ) - parser.add_argument( - "--test-platform", - action="store_true", - help="Filter combinations to include only platforms with test-platform: true.", - ) - parser.add_argument( - "--publish", - action="store_true", - help="Filter combinations to include only those with publish enabled.", - ) - parser.add_argument( - "--category", - action="append", - help="Filter combinations to include only platforms belonging to the specified categories (can be specified multiple times).", - ) - parser.add_argument( - "--exclude-category", - action="append", - help="Exclude platforms belonging to the specified categories (can be specified multiple times).", - ) - parser.add_argument( - "--json-by-arch", - action="store_true", - help="Output a JSON dictionary where keys are architectures and values are lists of flavors.", - ) - parser.add_argument( - "--markdown-table-by-platform", - action="store_true", - help="Generate a markdown table by platform.", - ) - args = parser.parse_args() - return args - - -def main(): - """Main function for command line usage.""" - args = parse_arguments() - - # Get flavors using parse_flavors_commit - flavors = parse_flavors_commit( - commit=args.commit, - include_only_patterns=args.include_only or [], - wildcard_excludes=args.exclude or [], - only_build=args.build, - only_test=args.test, - only_test_platform=args.test_platform, - only_publish=args.publish, - filter_categories=args.category or [], - exclude_categories=args.exclude_category or [], - ) - - if not flavors: - sys.exit(1) - - # Output the results in the requested format - if args.json_by_arch: - # Convert flavors to (arch, flavor) tuples for grouping - combinations = [] - for flavor in flavors: - arch = flavor.split("-")[-1] # Get architecture from the end - combinations.append((arch, flavor)) - - grouped_combinations = group_by_arch(combinations) - # If --no-arch, strip architectures from the grouped output - if args.no_arch: - grouped_combinations = { - arch: sorted(set(item.replace(f"-{arch}", "") for item in items)) - for arch, items in grouped_combinations.items() - } - print(json.dumps(grouped_combinations, indent=2)) - elif args.markdown_table_by_platform: - # Convert flavors to (arch, flavor) tuples for table - combinations = [] - for flavor in flavors: - arch = flavor.split("-")[-1] # Get architecture from the end - combinations.append((arch, flavor)) - - markdown_table = generate_markdown_table(combinations, args.no_arch) - print(markdown_table) - else: - if args.no_arch: - # Remove architecture from each flavor - no_arch_flavors = [] - for flavor in flavors: - no_arch_flavor = "-".join( - flavor.split("-")[:-1] - ) # Remove last component (arch) - no_arch_flavors.append(no_arch_flavor) - print("\n".join(sorted(set(no_arch_flavors)))) - else: - print("\n".join(sorted(flavors))) - - -if __name__ == "__main__": - main() diff --git a/src/python_gardenlinux_lib/flavors/parser.py b/src/python_gardenlinux_lib/flavors/parser.py new file mode 100644 index 00000000..2ce2e1e5 --- /dev/null +++ b/src/python_gardenlinux_lib/flavors/parser.py @@ -0,0 +1,103 @@ +from jsonschema import validate as jsonschema_validate +import fnmatch + +from ..constants import GL_FLAVORS_SCHEMA + +def validate_flavors(data): + """Validate the flavors.yaml data against the schema.""" + jsonschema_validate(instance=data, schema=GL_FLAVORS_SCHEMA) + +def should_exclude(combination, excludes, wildcard_excludes): + """ + Checks if a combination should be excluded based on exact match or wildcard patterns. + """ + # Exclude if in explicit excludes + if combination in excludes: + return True + # Exclude if matches any wildcard pattern + return any(fnmatch.fnmatch(combination, pattern) for pattern in wildcard_excludes) + +def should_include_only(combination, include_only_patterns): + """ + Checks if a combination should be included based on `--include-only` wildcard patterns. + If no patterns are provided, all combinations are included by default. + """ + if not include_only_patterns: + return True + return any(fnmatch.fnmatch(combination, pattern) for pattern in include_only_patterns) + +def parse_flavors( + data, + include_only_patterns=[], + wildcard_excludes=[], + only_build=False, + only_test=False, + only_test_platform=False, + only_publish=False, + filter_categories=[], + exclude_categories=[] +): + """Parses the flavors.yaml file and generates combinations.""" + combinations = [] # Use a list for consistent order + + for target in data['targets']: + name = target['name'] + category = target.get('category', '') + + # Apply category filters + if filter_categories and category not in filter_categories: + continue + if exclude_categories and category in exclude_categories: + continue + + for flavor in target['flavors']: + features = flavor.get('features', []) + arch = flavor.get('arch', 'amd64') + build = flavor.get('build', False) + test = flavor.get('test', False) + test_platform = flavor.get('test-platform', False) + publish = flavor.get('publish', False) + + # Apply flag-specific filters in the order: build, test, test-platform, publish + if only_build and not build: + continue + if only_test and not test: + continue + if only_test_platform and not test_platform: + continue + if only_publish and not publish: + continue + + # Process features + formatted_features = f"-{'-'.join(features)}" if features else "" + + # Construct the combination + combination = f"{name}-{formatted_features}-{arch}" + + # Format the combination to clean up "--" and "-_" + combination = combination.replace("--", "-").replace("-_", "_") + + # Exclude combinations explicitly + if should_exclude(combination, [], wildcard_excludes): + continue + + # Apply include-only filters + if not should_include_only(combination, include_only_patterns): + continue + + combinations.append((arch, combination)) + + return sorted(combinations, key=lambda x: x[1].split("-")[0]) # Sort by platform name + +def group_by_arch(combinations): + """Groups combinations by architecture into a JSON dictionary.""" + arch_dict = {} + for arch, combination in combinations: + arch_dict.setdefault(arch, []).append(combination) + for arch in arch_dict: + arch_dict[arch] = sorted(set(arch_dict[arch])) # Deduplicate and sort + return arch_dict + +def remove_arch(combinations): + """Removes the architecture from combinations.""" + return [combination.replace(f"-{arch}", "") for arch, combination in combinations] From ceade8a8279348e00ca99ca9fdb0cc235e70838a Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Wed, 16 Apr 2025 20:34:41 +0200 Subject: [PATCH 02/21] Add support to read existing `flavors.yaml` with `gl-flavors-parse` Signed-off-by: Tobias Wolf --- .../flavors/parse_flavors.py | 179 ++++++++---------- 1 file changed, 79 insertions(+), 100 deletions(-) diff --git a/src/python_gardenlinux_lib/flavors/parse_flavors.py b/src/python_gardenlinux_lib/flavors/parse_flavors.py index ab152497..43bb24aa 100755 --- a/src/python_gardenlinux_lib/flavors/parse_flavors.py +++ b/src/python_gardenlinux_lib/flavors/parse_flavors.py @@ -29,6 +29,26 @@ def find_repo_root(): return Git(".").rev_parse("--show-superproject-working-tree") +def _get_flavors_from_github(commit): + """Returns the flavors.yaml from GitHub if readable.""" + + # Try flavors.yaml first + api_path = "/repos/gardenlinux/gardenlinux/contents/flavors.yaml" + if commit != "latest": + api_path = f"{api_path}?ref={commit}" + command = ["gh", "api", api_path] + logger.debug(f"Fetching flavors.yaml from GitHub for commit {commit_short}") + result = subprocess.run( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + + if result.returncode == 0: + content_data = json.loads(result.stdout) + return base64.b64decode(content_data["content"]).decode("utf-8") + else: + raise RuntimeError("Failed receiving result from GitHub: {0}".format(result.stderr)) + + def parse_flavors_commit( commit=None, version=None, @@ -65,13 +85,6 @@ def parse_flavors_commit( Returns: list: List of flavor strings, or empty list if no flavors found """ - try: - version_info = ( - f"{version['major']}.{version.get('minor', 0)}" if version else "unknown" - ) - if commit is None: - commit = "latest" - commit_short = commit[:8] if logger is None: logger = logging.getLogger("gardenlinux.lib.flavors") @@ -113,99 +126,65 @@ def parse_flavors_commit( exclude_categories=exclude_categories or [], ) - # Try flavors.yaml first - api_path = "/repos/gardenlinux/gardenlinux/contents/flavors.yaml" - if commit != "latest": - api_path = f"{api_path}?ref={commit}" - command = ["gh", "api", api_path] - logger.debug(f"Fetching flavors.yaml from GitHub for commit {commit_short}") - result = subprocess.run( - command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True - ) - - if result.returncode == 0: - content_data = json.loads(result.stdout) - yaml_content = base64.b64decode(content_data["content"]).decode("utf-8") - flavors_data = yaml.safe_load(yaml_content) - - # Parse flavors with all filters - combinations = parse_flavors_data( - flavors_data, - include_only_patterns=include_only_patterns or [], - wildcard_excludes=wildcard_excludes or [], - only_build=only_build, - only_test=only_test, - only_test_platform=only_test_platform, - only_publish=only_publish, - filter_categories=filter_categories or [], - exclude_categories=exclude_categories or [], + all_flavors = set() + for _, combination in combinations: + all_flavors.add(combination) + + if all_flavors: + logger.info(f"Found {len(all_flavors)} flavors in flavors.yaml") + return sorted(all_flavors) + else: + logger.info("No flavors found in flavors.yaml") + elif query_s3 and s3_objects and isinstance(s3_objects, dict): + logger.debug("Checking S3 artifacts") + index = s3_objects.get("index", {}) + artifacts = s3_objects.get("artifacts", []) + + # Try index lookup first + search_key = f"{version_info}-{commit_short}" + if search_key in index: + flavors = index[search_key] + logger.debug(f"Found flavors in S3 index for {search_key}") + else: + # If no index match, search through artifacts + found_flavors = set() + + # Search for artifacts matching version and commit + for key in artifacts: + if version_info in key and commit_short in key: + try: + parts = key.split("/") + if len(parts) >= 2: + flavor_with_version = parts[1] + flavor = flavor_with_version.rsplit( + "-" + version_info, 1 + )[0] + if flavor: + found_flavors.add(flavor) + except Exception as e: + logger.debug(f"Error parsing artifact key {key}: {e}") + continue + + flavors = list(found_flavors) + + # Apply filters to S3 flavors + filtered_flavors = [] + for flavor in flavors: + # Create a dummy combination with amd64 architecture for filtering + combination = ("amd64", flavor) + if should_include_only( + flavor, include_only_patterns or [] + ) and not should_exclude(flavor, wildcard_excludes or [], []): + filtered_flavors.append(flavor) + + if filtered_flavors: + logger.info( + f"Found {len(filtered_flavors)} flavors in S3 artifacts after filtering" + ) + return sorted(filtered_flavors) + else: + logger.info( + f"No flavors found in S3 for version {version_info} and commit {commit_short} after filtering" ) - all_flavors = set() - for _, combination in combinations: - all_flavors.add(combination) - - if all_flavors: - logger.info(f"Found {len(all_flavors)} flavors in flavors.yaml") - return sorted(all_flavors) - else: - logger.info("No flavors found in flavors.yaml") - - # If no flavors.yaml found and query_s3 is enabled, try S3 artifacts - if query_s3 and s3_objects and isinstance(s3_objects, dict): - logger.debug("Checking S3 artifacts") - index = s3_objects.get("index", {}) - artifacts = s3_objects.get("artifacts", []) - - # Try index lookup first - search_key = f"{version_info}-{commit_short}" - if search_key in index: - flavors = index[search_key] - logger.debug(f"Found flavors in S3 index for {search_key}") - else: - # If no index match, search through artifacts - found_flavors = set() - - # Search for artifacts matching version and commit - for key in artifacts: - if version_info in key and commit_short in key: - try: - parts = key.split("/") - if len(parts) >= 2: - flavor_with_version = parts[1] - flavor = flavor_with_version.rsplit( - "-" + version_info, 1 - )[0] - if flavor: - found_flavors.add(flavor) - except Exception as e: - logger.debug(f"Error parsing artifact key {key}: {e}") - continue - - flavors = list(found_flavors) - - # Apply filters to S3 flavors - filtered_flavors = [] - for flavor in flavors: - # Create a dummy combination with amd64 architecture for filtering - combination = ("amd64", flavor) - if should_include_only( - flavor, include_only_patterns or [] - ) and not should_exclude(flavor, wildcard_excludes or [], []): - filtered_flavors.append(flavor) - - if filtered_flavors: - logger.info( - f"Found {len(filtered_flavors)} flavors in S3 artifacts after filtering" - ) - return sorted(filtered_flavors) - else: - logger.info( - f"No flavors found in S3 for version {version_info} and commit {commit_short} after filtering" - ) - - return [] - - except Exception as e: - logger.error(f"Error parsing flavors for commit {commit_short}: {e}") return [] From ff3c0adc6b0fd5bf241db7b179571d2b9ed8720c Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Fri, 4 Apr 2025 08:23:54 +0200 Subject: [PATCH 03/21] Add support to parse and return the GardenLinux "cname" Relates: https://github.com/gardenlinux/gardenlinux/issues/2830 Signed-off-by: Tobias Wolf --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f4837c87..51314bde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ boto3 = "*" black = "^24.8.0" [tool.poetry.scripts] -gl-flavors-parse = "python_gardenlinux_lib.flavors.__main__:main" +gl-cname = "src.python_gardenlinux_lib.cname:main" flavors-parse = "python_gardenlinux_lib.flavors.__main__:main" [tool.pytest.ini_options] From c2217880a7c33f45748554b3719d7a152b65738a Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Mon, 28 Apr 2025 06:38:18 +0200 Subject: [PATCH 04/21] Restructure GL flavors parsing and module structure Signed-off-by: Tobias Wolf --- src/gardenlinux/__init__.py | 0 .../constants.py | 0 src/gardenlinux/flavors/__init__.py | 1 + .../flavors/__main__.py | 20 +-- src/gardenlinux/flavors/parser.py | 123 ++++++++++++++++++ .../flavors/parse_flavors.py | 64 +++++++-- src/python_gardenlinux_lib/flavors/parser.py | 103 --------------- tests/conftest.py | 2 +- tests/test_push_image.py | 2 - 9 files changed, 183 insertions(+), 132 deletions(-) create mode 100644 src/gardenlinux/__init__.py rename src/{python_gardenlinux_lib => gardenlinux}/constants.py (100%) create mode 100644 src/gardenlinux/flavors/__init__.py rename src/{python_gardenlinux_lib => gardenlinux}/flavors/__main__.py (88%) create mode 100644 src/gardenlinux/flavors/parser.py delete mode 100644 src/python_gardenlinux_lib/flavors/parser.py diff --git a/src/gardenlinux/__init__.py b/src/gardenlinux/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/python_gardenlinux_lib/constants.py b/src/gardenlinux/constants.py similarity index 100% rename from src/python_gardenlinux_lib/constants.py rename to src/gardenlinux/constants.py diff --git a/src/gardenlinux/flavors/__init__.py b/src/gardenlinux/flavors/__init__.py new file mode 100644 index 00000000..2a3855a1 --- /dev/null +++ b/src/gardenlinux/flavors/__init__.py @@ -0,0 +1 @@ +from .parser import Parser diff --git a/src/python_gardenlinux_lib/flavors/__main__.py b/src/gardenlinux/flavors/__main__.py similarity index 88% rename from src/python_gardenlinux_lib/flavors/__main__.py rename to src/gardenlinux/flavors/__main__.py index dc7a7564..893eb879 100644 --- a/src/python_gardenlinux_lib/flavors/__main__.py +++ b/src/gardenlinux/flavors/__main__.py @@ -5,10 +5,8 @@ import json import os import sys -import yaml -from jsonschema import ValidationError -from .parser import group_by_arch, parse_flavors, validate_flavors +from .parser import Parser def generate_markdown_table(combinations, no_arch): @@ -93,16 +91,10 @@ def main(): sys.exit(f"Error: {flavors_file} does not exist.") # Load and validate the flavors.yaml - with open(flavors_file, 'r') as file: - flavors_data = yaml.safe_load(file) + with open(flavors_file, "r") as file: + flavors_data = file.read() - try: - validate_flavors(flavors_data) - except ValidationError as e: - sys.exit(f"Validation Error: {e.message}") - - combinations = parse_flavors( - flavors_data, + combinations = Parser(flavors_data).filter( include_only_patterns=args.include_only, wildcard_excludes=args.exclude, only_build=args.build, @@ -114,7 +106,7 @@ def main(): ) if args.json_by_arch: - grouped_combinations = group_by_arch(combinations) + grouped_combinations = Parser.group_by_arch(combinations) # If --no-arch, strip architectures from the grouped output if args.no_arch: @@ -128,7 +120,7 @@ def main(): print(generate_markdown_table(combinations, args.no_arch)) else: if args.no_arch: - printable_combinations = sorted(set(remove_arch(combinations))) + printable_combinations = sorted(set(Parser.remove_arch(combinations))) else: printable_combinations = sorted(set(comb[1] for comb in combinations)) diff --git a/src/gardenlinux/flavors/parser.py b/src/gardenlinux/flavors/parser.py new file mode 100644 index 00000000..98f5a472 --- /dev/null +++ b/src/gardenlinux/flavors/parser.py @@ -0,0 +1,123 @@ +from jsonschema import validate as jsonschema_validate +import fnmatch +import logging +import yaml + +from ..constants import GL_FLAVORS_SCHEMA + +class Parser(object): + def __init__(self, data, logger = None): + flavors_data = (yaml.safe_load(data) if isinstance(data, str) else data) + jsonschema_validate(instance=flavors_data, schema=GL_FLAVORS_SCHEMA) + + self._flavors_data = flavors_data + self._logger = logger + + if self._logger is None: + self._logger = logging.getLogger("gardenlinux.flavors") + + if not self._logger.hasHandlers(): + self._logger.addHandler(logging.NullHandler()) + + self._logger.debug("flavors.Parser initialized with data: {0!r}".format(flavors_data)) + + def filter( + self, + include_only_patterns=[], + wildcard_excludes=[], + only_build=False, + only_test=False, + only_test_platform=False, + only_publish=False, + filter_categories=[], + exclude_categories=[] + ): + """Parses the flavors.yaml file and generates combinations.""" + self._logger.debug("flavors.Parser filtering with {0}".format(locals())) + + combinations = [] # Use a list for consistent order + + for target in self._flavors_data['targets']: + name = target['name'] + category = target.get('category', '') + + # Apply category filters + if filter_categories and category not in filter_categories: + continue + if exclude_categories and category in exclude_categories: + continue + + for flavor in target['flavors']: + features = flavor.get('features', []) + arch = flavor.get('arch', 'amd64') + build = flavor.get('build', False) + test = flavor.get('test', False) + test_platform = flavor.get('test-platform', False) + publish = flavor.get('publish', False) + + # Apply flag-specific filters in the order: build, test, test-platform, publish + if only_build and not build: + continue + if only_test and not test: + continue + if only_test_platform and not test_platform: + continue + if only_publish and not publish: + continue + + # Process features + formatted_features = f"-{'-'.join(features)}" if features else "" + + # Construct the combination + combination = f"{name}-{formatted_features}-{arch}" + + # Format the combination to clean up "--" and "-_" + combination = combination.replace("--", "-").replace("-_", "_") + + # Exclude combinations explicitly + if Parser.should_exclude(combination, [], wildcard_excludes): + continue + + # Apply include-only filters + if not Parser.should_include_only(combination, include_only_patterns): + continue + + combinations.append((arch, combination)) + + return sorted(combinations, key=lambda platform: platform[1].split("-")[0]) # Sort by platform name + + @staticmethod + def group_by_arch(combinations): + """Groups combinations by architecture into a JSON dictionary.""" + arch_dict = {} + for arch, combination in combinations: + arch_dict.setdefault(arch, []).append(combination) + for arch in arch_dict: + arch_dict[arch] = sorted(set(arch_dict[arch])) # Deduplicate and sort + return arch_dict + + @staticmethod + def remove_arch(combinations): + """Removes the architecture from combinations.""" + return [combination.replace(f"-{arch}", "") for arch, combination in combinations] + + @staticmethod + def should_exclude(combination, excludes, wildcard_excludes): + """ + Checks if a combination should be excluded based on exact match or wildcard patterns. + """ + # Exclude if in explicit excludes + if combination in excludes: + return True + # Exclude if matches any wildcard pattern + return any(fnmatch.fnmatch(combination, pattern) for pattern in wildcard_excludes) + + @staticmethod + def should_include_only(combination, include_only_patterns): + """ + Checks if a combination should be included based on `--include-only` wildcard patterns. + If no patterns are provided, all combinations are included by default. + """ + if not include_only_patterns: + return True + return any(fnmatch.fnmatch(combination, pattern) for pattern in include_only_patterns) diff --git a/src/python_gardenlinux_lib/flavors/parse_flavors.py b/src/python_gardenlinux_lib/flavors/parse_flavors.py index 43bb24aa..941d3c73 100755 --- a/src/python_gardenlinux_lib/flavors/parse_flavors.py +++ b/src/python_gardenlinux_lib/flavors/parse_flavors.py @@ -1,24 +1,18 @@ #!/usr/bin/env python +# @TODO: This code is provided for backward compatibility only and deprecated. + +from gardenlinux.flavors.parser import Parser from git import Git import base64 import json import logging import os +import sys import subprocess import yaml -from .parser import ( - group_by_arch, - remove_arch, - should_include_only, - should_exclude, - validate_flavors -) - -from .parser import parse_flavors as parse_flavors_data - -from ..constants import GL_FLAVORS_SCHEMA +from gardenlinux.constants import GL_FLAVORS_SCHEMA # Define the schema for validation SCHEMA = GL_FLAVORS_SCHEMA @@ -29,6 +23,52 @@ def find_repo_root(): return Git(".").rev_parse("--show-superproject-working-tree") +def validate_flavors(data): + """Validate the flavors.yaml data against the schema.""" + try: + validate(instance=data, schema=SCHEMA) + except ValidationError as e: + sys.exit(f"Validation Error: {e.message}") + + +def should_exclude(combination, excludes, wildcard_excludes): + """ + Checks if a combination should be excluded based on exact match or wildcard patterns. + """ + return Parser.should_exclude(combination, excludes, wildcard_excludes) + + +def should_include_only(combination, include_only_patterns): + """ + Checks if a combination should be included based on `--include-only` wildcard patterns. + If no patterns are provided, all combinations are included by default. + """ + return Parser.should_include_only(combination, include_only_patterns) + + +def parse_flavors_data( + data, + include_only_patterns=None, + wildcard_excludes=None, + only_build=False, + only_test=False, + only_test_platform=False, + only_publish=False, + filter_categories=None, + exclude_categories=None, +): + """Parse flavors.yaml data and generate combinations.""" + return Parser(data).filter( + include_only_patterns, + wildcard_excludes, + only_build, + only_test, + only_test_platform, + only_publish, + filter_categories, + exclude_categories + ) + def _get_flavors_from_github(commit): """Returns the flavors.yaml from GitHub if readable.""" @@ -87,7 +127,7 @@ def parse_flavors_commit( """ if logger is None: - logger = logging.getLogger("gardenlinux.lib.flavors") + logger = logging.getLogger("gardenlinux.flavors") logger.addHandler(logging.NullHandler()) version_info = ( diff --git a/src/python_gardenlinux_lib/flavors/parser.py b/src/python_gardenlinux_lib/flavors/parser.py deleted file mode 100644 index 2ce2e1e5..00000000 --- a/src/python_gardenlinux_lib/flavors/parser.py +++ /dev/null @@ -1,103 +0,0 @@ -from jsonschema import validate as jsonschema_validate -import fnmatch - -from ..constants import GL_FLAVORS_SCHEMA - -def validate_flavors(data): - """Validate the flavors.yaml data against the schema.""" - jsonschema_validate(instance=data, schema=GL_FLAVORS_SCHEMA) - -def should_exclude(combination, excludes, wildcard_excludes): - """ - Checks if a combination should be excluded based on exact match or wildcard patterns. - """ - # Exclude if in explicit excludes - if combination in excludes: - return True - # Exclude if matches any wildcard pattern - return any(fnmatch.fnmatch(combination, pattern) for pattern in wildcard_excludes) - -def should_include_only(combination, include_only_patterns): - """ - Checks if a combination should be included based on `--include-only` wildcard patterns. - If no patterns are provided, all combinations are included by default. - """ - if not include_only_patterns: - return True - return any(fnmatch.fnmatch(combination, pattern) for pattern in include_only_patterns) - -def parse_flavors( - data, - include_only_patterns=[], - wildcard_excludes=[], - only_build=False, - only_test=False, - only_test_platform=False, - only_publish=False, - filter_categories=[], - exclude_categories=[] -): - """Parses the flavors.yaml file and generates combinations.""" - combinations = [] # Use a list for consistent order - - for target in data['targets']: - name = target['name'] - category = target.get('category', '') - - # Apply category filters - if filter_categories and category not in filter_categories: - continue - if exclude_categories and category in exclude_categories: - continue - - for flavor in target['flavors']: - features = flavor.get('features', []) - arch = flavor.get('arch', 'amd64') - build = flavor.get('build', False) - test = flavor.get('test', False) - test_platform = flavor.get('test-platform', False) - publish = flavor.get('publish', False) - - # Apply flag-specific filters in the order: build, test, test-platform, publish - if only_build and not build: - continue - if only_test and not test: - continue - if only_test_platform and not test_platform: - continue - if only_publish and not publish: - continue - - # Process features - formatted_features = f"-{'-'.join(features)}" if features else "" - - # Construct the combination - combination = f"{name}-{formatted_features}-{arch}" - - # Format the combination to clean up "--" and "-_" - combination = combination.replace("--", "-").replace("-_", "_") - - # Exclude combinations explicitly - if should_exclude(combination, [], wildcard_excludes): - continue - - # Apply include-only filters - if not should_include_only(combination, include_only_patterns): - continue - - combinations.append((arch, combination)) - - return sorted(combinations, key=lambda x: x[1].split("-")[0]) # Sort by platform name - -def group_by_arch(combinations): - """Groups combinations by architecture into a JSON dictionary.""" - arch_dict = {} - for arch, combination in combinations: - arch_dict.setdefault(arch, []).append(combination) - for arch in arch_dict: - arch_dict[arch] = sorted(set(arch_dict[arch])) # Deduplicate and sort - return arch_dict - -def remove_arch(combinations): - """Removes the architecture from combinations.""" - return [combination.replace(f"-{arch}", "") for arch, combination in combinations] diff --git a/tests/conftest.py b/tests/conftest.py index f0ec12ac..5586b567 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,7 @@ import pytest from dotenv import load_dotenv -GL_ROOT_DIR = "test-data/gardenlinux/" +GL_ROOT_DIR = "test-data/gardenlinux" def write_zot_config(config_dict, file_path): diff --git a/tests/test_push_image.py b/tests/test_push_image.py index 46f4fc44..9fa406bd 100644 --- a/tests/test_push_image.py +++ b/tests/test_push_image.py @@ -1,5 +1,3 @@ -from idlelib.window import registry - import pytest import os From b41949ceb301e83157eba0ae35c7b1ffd4d6c305 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Mon, 28 Apr 2025 16:48:58 +0200 Subject: [PATCH 05/21] Restructure GL features parsing and module structure Signed-off-by: Tobias Wolf --- .github/actions/flavors_parse/action.yml | 34 +++ .github/actions/python_lib/action.yml | 17 ++ poetry.lock | 91 +++++-- pyproject.toml | 15 +- src/gardenlinux/features/__init__.py | 1 + src/gardenlinux/features/__main__.py | 172 +++++++++++++ .../features/cname_main.py} | 49 ++-- src/gardenlinux/features/parser.py | 225 ++++++++++++++++++ .../features/parse_features.py | 154 +----------- 9 files changed, 568 insertions(+), 190 deletions(-) create mode 100644 .github/actions/flavors_parse/action.yml create mode 100644 .github/actions/python_lib/action.yml create mode 100644 src/gardenlinux/features/__init__.py create mode 100644 src/gardenlinux/features/__main__.py rename src/{python_gardenlinux_lib/cname.py => gardenlinux/features/cname_main.py} (61%) create mode 100644 src/gardenlinux/features/parser.py diff --git a/.github/actions/flavors_parse/action.yml b/.github/actions/flavors_parse/action.yml new file mode 100644 index 00000000..47733ab6 --- /dev/null +++ b/.github/actions/flavors_parse/action.yml @@ -0,0 +1,34 @@ +name: flavors_parse +description: Parses the given GardenLinux flavors parameters +inputs: + flags: + description: 'Flags passed to bin/flavors_parse.py' + required: true + flavors_matrix: + description: 'Generated GitHub workflow flavors matrix' +outputs: + matrix: + description: 'Flavors matrix' + value: ${{ steps.matrix.outputs.matrix }} +runs: + using: composite + steps: + - uses: gardenlinux/python-gardenlinux-lib/.github/actions/python_lib@feature/gardenlinux-restructure + - id: matrix + shell: bash + run: | + MATRIX='${{ inputs.flavors_matrix }}' + + if [[ $(echo "${MATRIX}" | jq -r 'type') != 'object' ]]; then + FLAVORS=$(gl-flavors-parse ${{ inputs.flags }}) + MATRIX=$(jq -nc \ + --argjson flavors "$(echo $FLAVORS)" \ + '{ + include: ( + $flavors | reduce (to_entries[]) as $item ([]; . + ($item.value | map({"arch": $item.key, "flavor": .}))) + ) + }' + ) + fi + + echo "matrix=$MATRIX" | tee -a $GITHUB_OUTPUT diff --git a/.github/actions/python_lib/action.yml b/.github/actions/python_lib/action.yml new file mode 100644 index 00000000..b8c90aaa --- /dev/null +++ b/.github/actions/python_lib/action.yml @@ -0,0 +1,17 @@ +name: python_lib +description: Installs the given GardenLinux Python library +inputs: + version: + description: GardenLinux Python library version + default: "feature/gardenlinux-restructure" +runs: + using: composite + steps: + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + - name: Install GardenLinux Python library + shell: bash + run: | + pip install git+https://github.com/gardenlinux/python-gardenlinux-lib.git@${{ inputs.version }} diff --git a/poetry.lock b/poetry.lock index 56c24638..9d89670a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. [[package]] name = "alabaster" @@ -6,6 +6,7 @@ version = "0.7.16" description = "A light, configurable Sphinx theme" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "alabaster-0.7.16-py3-none-any.whl", hash = "sha256:b46733c07dce03ae4e150330b975c75737fa60f0a7c591b6c8bf4928a28e2c92"}, {file = "alabaster-0.7.16.tar.gz", hash = "sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65"}, @@ -17,6 +18,7 @@ version = "0.5" description = "Python library to query APT repositories" optional = false python-versions = ">=3.5" +groups = ["main"] files = [ {file = "apt-repo-0.5.tar.gz", hash = "sha256:b566195884b8ea59e6b831f814fd106c7e683dccc86ca95f6494a447572f30ea"}, ] @@ -27,18 +29,19 @@ version = "25.1.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "attrs-25.1.0-py3-none-any.whl", hash = "sha256:c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a"}, {file = "attrs-25.1.0.tar.gz", hash = "sha256:1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e"}, ] [package.extras] -benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +benchmark = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] +cov = ["cloudpickle ; platform_python_implementation == \"CPython\"", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] +dev = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] -tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] +tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\""] [[package]] name = "babel" @@ -46,13 +49,14 @@ version = "2.17.0" description = "Internationalization utilities" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2"}, {file = "babel-2.17.0.tar.gz", hash = "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d"}, ] [package.extras] -dev = ["backports.zoneinfo", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata"] +dev = ["backports.zoneinfo ; python_version < \"3.9\"", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata ; sys_platform == \"win32\""] [[package]] name = "black" @@ -60,6 +64,7 @@ version = "24.10.0" description = "The uncompromising code formatter." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "black-24.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6668650ea4b685440857138e5fe40cde4d652633b1bdffc62933d0db4ed9812"}, {file = "black-24.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1c536fcf674217e87b8cc3657b81809d3c085d7bf3ef262ead700da345bfa6ea"}, @@ -106,6 +111,7 @@ version = "1.36.21" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "boto3-1.36.21-py3-none-any.whl", hash = "sha256:f94faa7cf932d781f474d87f8b4c14a033af95ac1460136b40d75e7a30086ef0"}, {file = "boto3-1.36.21.tar.gz", hash = "sha256:41eb2b73eb612d300e629e3328b83f1ffea0fc6633e75c241a72a76746c1db26"}, @@ -125,6 +131,7 @@ version = "1.36.21" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "botocore-1.36.21-py3-none-any.whl", hash = "sha256:24a7052e792639dc2726001bd474cd0aaa959c1e18ddd92c17f3adc6efa1b132"}, {file = "botocore-1.36.21.tar.gz", hash = "sha256:da746240e2ad64fd4997f7f3664a0a8e303d18075fc1d473727cb6375080ea16"}, @@ -144,6 +151,7 @@ version = "2025.1.31" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "certifi-2025.1.31-py3-none-any.whl", hash = "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe"}, {file = "certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651"}, @@ -155,6 +163,8 @@ version = "1.17.1" description = "Foreign Function Interface for Python calling C code." optional = false python-versions = ">=3.8" +groups = ["main"] +markers = "platform_python_implementation != \"PyPy\"" files = [ {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, @@ -234,6 +244,7 @@ version = "3.4.1" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "charset_normalizer-3.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:91b36a978b5ae0ee86c394f5a54d6ef44db1de0815eb43de826d41d21e4af3de"}, {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7461baadb4dc00fd9e0acbe254e3d7d2112e7f92ced2adc96e54ef6501c5f176"}, @@ -335,6 +346,7 @@ version = "8.1.8" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"}, @@ -349,10 +361,12 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main", "dev"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +markers = {main = "sys_platform == \"win32\"", dev = "platform_system == \"Windows\""} [[package]] name = "cryptography" @@ -360,6 +374,7 @@ version = "44.0.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = "!=3.9.0,!=3.9.1,>=3.7" +groups = ["main"] files = [ {file = "cryptography-44.0.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bf688f615c29bfe9dfc44312ca470989279f0e94bb9f631f85e3459af8efc009"}, {file = "cryptography-44.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd7c7e2d71d908dc0f8d2027e1604102140d84b155e658c20e8ad1304317691f"}, @@ -398,10 +413,10 @@ files = [ cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} [package.extras] -docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=3.0.0)"] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=3.0.0) ; python_version >= \"3.8\""] docstest = ["pyenchant (>=3)", "readme-renderer (>=30.0)", "sphinxcontrib-spelling (>=7.3.1)"] -nox = ["nox (>=2024.4.15)", "nox[uv] (>=2024.3.2)"] -pep8test = ["check-sdist", "click (>=8.0.1)", "mypy (>=1.4)", "ruff (>=0.3.6)"] +nox = ["nox (>=2024.4.15)", "nox[uv] (>=2024.3.2) ; python_version >= \"3.8\""] +pep8test = ["check-sdist ; python_version >= \"3.8\"", "click (>=8.0.1)", "mypy (>=1.4)", "ruff (>=0.3.6)"] sdist = ["build (>=1.0.0)"] ssh = ["bcrypt (>=3.1.5)"] test = ["certifi (>=2024)", "cryptography-vectors (==44.0.1)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] @@ -413,6 +428,7 @@ version = "0.20.1" description = "Docutils -- Python Documentation Utilities" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "docutils-0.20.1-py3-none-any.whl", hash = "sha256:96f387a2c5562db4476f09f13bbab2192e764cac08ebbf3a34a95d9b1e4a59d6"}, {file = "docutils-0.20.1.tar.gz", hash = "sha256:f08a4e276c3a1583a86dce3e34aba3fe04d02bba2dd51ed16106244e8a923e3b"}, @@ -424,6 +440,8 @@ version = "1.2.2" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" +groups = ["main"] +markers = "python_version == \"3.10\"" files = [ {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, @@ -438,6 +456,7 @@ version = "4.0.12" description = "Git Object Database" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf"}, {file = "gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571"}, @@ -452,6 +471,7 @@ version = "3.1.44" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "GitPython-3.1.44-py3-none-any.whl", hash = "sha256:9e0e10cda9bed1ee64bc9a6de50e7e38a9c9943241cd7f585f6df3ed28011110"}, {file = "gitpython-3.1.44.tar.gz", hash = "sha256:c87e30b26253bf5418b01b0660f818967f3c503193838337fe5e573331249269"}, @@ -462,7 +482,7 @@ gitdb = ">=4.0.1,<5" [package.extras] doc = ["sphinx (>=7.1.2,<7.2)", "sphinx-autodoc-typehints", "sphinx_rtd_theme"] -test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions"] +test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3.8\"", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions ; python_version < \"3.11\""] [[package]] name = "idna" @@ -470,6 +490,7 @@ version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, @@ -484,6 +505,7 @@ version = "1.4.1" description = "Getting image size from png/jpeg/jpeg2000/gif file" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["main"] files = [ {file = "imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b"}, {file = "imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"}, @@ -495,6 +517,7 @@ version = "2.0.0" description = "brain-dead simple config-ini parsing" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, @@ -506,6 +529,7 @@ version = "3.1.5" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb"}, {file = "jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb"}, @@ -523,6 +547,7 @@ version = "1.0.1" description = "JSON Matching Expressions" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, @@ -534,6 +559,7 @@ version = "4.23.0" description = "An implementation of JSON Schema validation for Python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566"}, {file = "jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4"}, @@ -555,6 +581,7 @@ version = "2024.10.1" description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "jsonschema_specifications-2024.10.1-py3-none-any.whl", hash = "sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf"}, {file = "jsonschema_specifications-2024.10.1.tar.gz", hash = "sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272"}, @@ -569,6 +596,7 @@ version = "3.0.2" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8"}, {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158"}, @@ -639,6 +667,7 @@ version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." optional = false python-versions = ">=3.5" +groups = ["dev"] files = [ {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, @@ -650,6 +679,7 @@ version = "3.4.2" description = "Python package for creating and manipulating graphs and networks" optional = false python-versions = ">=3.10" +groups = ["main"] files = [ {file = "networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f"}, {file = "networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1"}, @@ -669,6 +699,7 @@ version = "0.2.0" description = "OCI Registry as Storage Python SDK" optional = false python-versions = "*" +groups = ["main"] files = [] develop = false @@ -693,6 +724,7 @@ version = "24.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, @@ -704,6 +736,7 @@ version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, @@ -715,6 +748,7 @@ version = "4.3.6" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"}, {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"}, @@ -731,6 +765,7 @@ version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, @@ -746,6 +781,8 @@ version = "2.22" description = "C parser in Python" optional = false python-versions = ">=3.8" +groups = ["main"] +markers = "platform_python_implementation != \"PyPy\"" files = [ {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, @@ -757,6 +794,7 @@ version = "2.19.1" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c"}, {file = "pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f"}, @@ -771,6 +809,7 @@ version = "8.3.4" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "pytest-8.3.4-py3-none-any.whl", hash = "sha256:50e16d954148559c9a74109af1eaf0c945ba2d8f30f0a3d3335edde19788b6f6"}, {file = "pytest-8.3.4.tar.gz", hash = "sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761"}, @@ -793,6 +832,7 @@ version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -807,6 +847,7 @@ version = "1.0.1" description = "Read key-value pairs from a .env file and set them as environment variables" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"}, {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"}, @@ -821,6 +862,7 @@ version = "6.0.2" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -883,6 +925,7 @@ version = "0.36.2" description = "JSON Referencing + Python" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "referencing-0.36.2-py3-none-any.whl", hash = "sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0"}, {file = "referencing-0.36.2.tar.gz", hash = "sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa"}, @@ -899,6 +942,7 @@ version = "2.32.3" description = "Python HTTP for Humans." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, @@ -920,6 +964,7 @@ version = "0.22.3" description = "Python bindings to Rust's persistent data structures (rpds)" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "rpds_py-0.22.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:6c7b99ca52c2c1752b544e310101b98a659b720b21db00e65edca34483259967"}, {file = "rpds_py-0.22.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:be2eb3f2495ba669d2a985f9b426c1797b7d48d6963899276d22f23e33d47e37"}, @@ -1032,6 +1077,7 @@ version = "0.11.2" description = "An Amazon S3 Transfer Manager" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "s3transfer-0.11.2-py3-none-any.whl", hash = "sha256:be6ecb39fadd986ef1701097771f87e4d2f821f27f6071c872143884d2950fbc"}, {file = "s3transfer-0.11.2.tar.gz", hash = "sha256:3b39185cb72f5acc77db1a58b6e25b977f28d20496b6e58d6813d75f464d632f"}, @@ -1049,6 +1095,7 @@ version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, @@ -1060,6 +1107,7 @@ version = "5.0.2" description = "A pure Python implementation of a sliding window memory map manager" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "smmap-5.0.2-py3-none-any.whl", hash = "sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e"}, {file = "smmap-5.0.2.tar.gz", hash = "sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5"}, @@ -1071,6 +1119,7 @@ version = "2.2.0" description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." optional = false python-versions = "*" +groups = ["main"] files = [ {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, @@ -1082,6 +1131,7 @@ version = "7.4.7" description = "Python documentation generator" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "sphinx-7.4.7-py3-none-any.whl", hash = "sha256:c2419e2135d11f1951cd994d6eb18a1835bd8fdd8429f9ca375dc1f3281bd239"}, {file = "sphinx-7.4.7.tar.gz", hash = "sha256:242f92a7ea7e6c5b406fdc2615413890ba9f699114a9c09192d7dfead2ee9cfe"}, @@ -1117,6 +1167,7 @@ version = "2.0.0" description = "Read the Docs theme for Sphinx" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "sphinx_rtd_theme-2.0.0-py2.py3-none-any.whl", hash = "sha256:ec93d0856dc280cf3aee9a4c9807c60e027c7f7b461b77aeffed682e68f0e586"}, {file = "sphinx_rtd_theme-2.0.0.tar.gz", hash = "sha256:bd5d7b80622406762073a04ef8fadc5f9151261563d47027de09910ce03afe6b"}, @@ -1136,6 +1187,7 @@ version = "2.0.0" description = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", hash = "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5"}, {file = "sphinxcontrib_applehelp-2.0.0.tar.gz", hash = "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1"}, @@ -1152,6 +1204,7 @@ version = "2.0.0" description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp documents" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2"}, {file = "sphinxcontrib_devhelp-2.0.0.tar.gz", hash = "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad"}, @@ -1168,6 +1221,7 @@ version = "2.1.0" description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8"}, {file = "sphinxcontrib_htmlhelp-2.1.0.tar.gz", hash = "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9"}, @@ -1184,6 +1238,7 @@ version = "4.1" description = "Extension to include jQuery on newer Sphinx releases" optional = false python-versions = ">=2.7" +groups = ["main"] files = [ {file = "sphinxcontrib-jquery-4.1.tar.gz", hash = "sha256:1620739f04e36a2c779f1a131a2dfd49b2fd07351bf1968ced074365933abc7a"}, {file = "sphinxcontrib_jquery-4.1-py2.py3-none-any.whl", hash = "sha256:f936030d7d0147dd026a4f2b5a57343d233f1fc7b363f68b3d4f1cb0993878ae"}, @@ -1198,6 +1253,7 @@ version = "1.0.1" description = "A sphinx extension which renders display math in HTML via JavaScript" optional = false python-versions = ">=3.5" +groups = ["main"] files = [ {file = "sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"}, {file = "sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178"}, @@ -1212,6 +1268,7 @@ version = "2.0.0" description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp documents" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", hash = "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb"}, {file = "sphinxcontrib_qthelp-2.0.0.tar.gz", hash = "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab"}, @@ -1228,6 +1285,7 @@ version = "2.0.0" description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331"}, {file = "sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d"}, @@ -1244,6 +1302,8 @@ version = "2.2.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] +markers = "python_version == \"3.10\"" files = [ {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, @@ -1285,10 +1345,12 @@ version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] +markers = {main = "python_version < \"3.13\"", dev = "python_version == \"3.10\""} [[package]] name = "urllib3" @@ -1296,18 +1358,19 @@ version = "2.3.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df"}, {file = "urllib3-2.3.0.tar.gz", hash = "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] [metadata] -lock-version = "2.0" +lock-version = "2.1" python-versions = "^3.10" -content-hash = "441f74ad13f2e0aa3cc7fecd8e245ba73d2a8410055a83af28cafcc486bed18a" +content-hash = "301412fce5875d4e31bdea597a1f3f856c06a1bb5c5f9a1d993b4d465889b6f7" diff --git a/pyproject.toml b/pyproject.toml index 51314bde..e9ddf684 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,11 +1,11 @@ [tool.poetry] -name = "python_gardenlinux_lib" +name = "gardenlinux" version = "0.6.0" description = "Contains tools to work with the features directory of gardenlinux, for example deducting dependencies from feature sets or validating cnames" authors = ["Garden Linux Maintainers "] license = "Apache-2.0" readme = "README.md" -packages = [{include = "python_gardenlinux_lib", from="src"}] +packages = [{include = "gardenlinux", from="src"}, {include = "python_gardenlinux_lib", from="src"}] [tool.poetry.dependencies] python = "^3.10" @@ -21,18 +21,17 @@ python-dotenv = "^1.0.1" cryptography = "^44.0.0" boto3 = "*" - [tool.poetry.group.dev.dependencies] black = "^24.8.0" [tool.poetry.scripts] -gl-cname = "src.python_gardenlinux_lib.cname:main" -flavors-parse = "python_gardenlinux_lib.flavors.__main__:main" +gl-cname = "gardenlinux.features.cname_main:main" +gl-features-parse = "gardenlinux.features.__main__:main" +gl-flavors-parse = "gardenlinux.flavors.__main__:main" +flavors-parse = "gardenlinux.flavors.__main__:main" [tool.pytest.ini_options] -pythonpath = [ - "src" -] +pythonpath = ["src"] norecursedirs = "test-data" [build-system] diff --git a/src/gardenlinux/features/__init__.py b/src/gardenlinux/features/__init__.py new file mode 100644 index 00000000..2a3855a1 --- /dev/null +++ b/src/gardenlinux/features/__init__.py @@ -0,0 +1 @@ +from .parser import Parser diff --git a/src/gardenlinux/features/__main__.py b/src/gardenlinux/features/__main__.py new file mode 100644 index 00000000..29c43bea --- /dev/null +++ b/src/gardenlinux/features/__main__.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 + +from .parser import Parser + +from functools import reduce +from os.path import basename, dirname + +import argparse +import re +import sys + + +_ARGS_TYPE_ALLOWED = [ + "cname", + "cname_base", + "features", + "platforms", + "flags", + "elements", + "arch", + "version", + "graph" +] + + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument("--arch", dest = "arch") + parser.add_argument("--feature-dir", default = "features") + parser.add_argument("--features", type = lambda arg: set([f for f in arg.split(",") if f])) + parser.add_argument("--ignore", dest = "ignore", type = lambda arg: set([f for f in arg.split(",") if f]), default = set()) + parser.add_argument("--cname") + parser.add_argument("--default-arch") + parser.add_argument("--default-version") + parser.add_argument("--version", dest="version") + parser.add_argument("type", nargs="?", choices = _ARGS_TYPE_ALLOWED, default = "cname") + + args = parser.parse_args() + + assert bool(args.features) or bool(args.cname), "Please provide either `--features` or `--cname` argument" + + arch = None + cname_base = None + commit_id = None + version = None + + if args.cname: + re_match = re.match( + "([a-zA-Z0-9]+(-[a-zA-Z0-9\\_\\-]*?)?)(-([a-z0-9]+)(-([a-z0-9.]+)-([a-z0-9]+))*)?$", + args.cname + ) + + assert re_match, f"Not a valid GardenLinux canonical name {args.cname}" + + if re_match.lastindex == 1: + data_splitted = re_match[1].split("-", 1) + + if len(data_splitted) > 1: + arch = data_splitted[1] + + cname_base = data_splitted[0] + else: + arch = re_match[4] + cname_base = re_match[1] + commit_id = re_match[7] + version = re_match[6] + + input_features = Parser.get_cname_as_feature_set(cname_base) + else: + input_features = args.features + + if args.arch is not None: + arch = args.arch + + if args.version is not None: + re_match = re.match("([a-z0-9.]+)(-([a-z0-9]+))?$", args.version) + assert re_match, f"Not a valid version {args.version}" + + commit_id = re_match[3] + version = re_match[1] + + if arch is None or arch == "" and (args.type in ( "cname", "arch" )): + assert args.default_arch, "Architecture could not be determined and no default architecture set" + arch = args.default_arch + + if not version and (args.type in ("cname", "version" )): + assert args.default_version, "version not specified and no default version set" + version = args.default_version + + gardenlinux_root = dirname(args.feature_dir) + feature_dir_name = basename(args.feature_dir) + + if gardenlinux_root == "": + gardenlinux_root = "." + + if gardenlinux_root == "": + gardenlinux_root = "." + + additional_filter_func = lambda node: node not in args.ignore + + if args.type == "arch": + print(arch) + elif args.type in ( "cname_base", "cname", "graph" ): + graph = Parser(gardenlinux_root, feature_dir_name).filter(cname_base, additional_filter_func = additional_filter_func) + + sorted_features = Parser.sort_reversed_graph_nodes(graph) + + minimal_feature_set = get_minimal_feature_set(graph) + + sorted_minimal_features = sort_subset( + minimal_feature_set, sorted_features + ) + + cname_base = get_cname_base(sorted_minimal_features) + + if args.type == "cname_base": + print(cname_base) + elif args.type == "cname": + cname = cname_base + + if arch is not None: + cname += f"-{arch}" + + if commit_id is not None: + cname += f"-{version}-{commit_id}" + + print(cname) + elif args.type == "graph": + print(graph_as_mermaid_markup(cname_base, graph)) + elif args.type == "features": + print(Parser(gardenlinux_root, feature_dir_name).filter_as_string(cname_base, additional_filter_func = additional_filter_func)) + elif args.type in ( "flags", "elements", "platforms" ): + features_by_type = Parser(gardenlinux_root, feature_dir_name).filter_as_dict(cname_base, additional_filter_func = additional_filter_func) + + if args.type == "platforms": + print(",".join(features_by_type["platform"])) + elif args.type == "elements": + print(",".join(features_by_type["element"])) + elif args.type == "flags": + print(",".join(features_by_type["flag"])) + elif args.type == "version": + print(f"{version}-{commit_id}") + + +def get_cname_base(sorted_features): + return reduce( + lambda a, b : a + ("-" if not b.startswith("_") else "") + b, sorted_features + ) + +def get_minimal_feature_set(graph): + return set([node for (node, degree) in graph.in_degree() if degree == 0]) + +def graph_as_mermaid_markup(cname_base, graph): + """ + Generates a mermaid.js representation of the graph. + This is helpful to identify dependencies between features. + + Syntax docs: + https://mermaid.js.org/syntax/flowchart.html?id=flowcharts-basic-syntax + """ + markup = f"---\ntitle: Dependency Graph for Feature {cname_base}\n---\ngraph TD;\n" + for u,v in graph.edges: + markup += f" {u}-->{v};\n" + return markup + +def sort_subset(input_set, order_list): + return [item for item in order_list if item in input_set] + + +if __name__ == "__main__": + main() diff --git a/src/python_gardenlinux_lib/cname.py b/src/gardenlinux/features/cname_main.py similarity index 61% rename from src/python_gardenlinux_lib/cname.py rename to src/gardenlinux/features/cname_main.py index 03c77b0a..7bc2e4b6 100644 --- a/src/python_gardenlinux_lib/cname.py +++ b/src/gardenlinux/features/cname_main.py @@ -1,13 +1,13 @@ #!/usr/bin/env python3 -from .features import parse_features - from functools import reduce from os.path import basename, dirname - import argparse import re +from .__main__ import get_cname_base, get_minimal_feature_set, sort_subset +from .parser import Parser + def main(): parser = argparse.ArgumentParser() @@ -24,12 +24,19 @@ def main(): args.cname ) - assert re_match, f"not a valid cname {args.cname}" + assert re_match, f"Not a valid GardenLinux canonical name {args.cname}" + + arch = None + commit_id = None + version = None if re_match.lastindex == 1: - cname_base, arch = re_match[1].split("-", 1) - commit_id = None - version = None + data_splitted = re_match[1].split("-", 1) + + if len(data_splitted) > 1: + arch = data_splitted[1] + + cname_base = data_splitted[0] else: arch = re_match[4] cname_base = re_match[1] @@ -39,9 +46,11 @@ def main(): if args.arch is not None: arch = args.arch + assert arch is None or arch == "", "Architecture could not be determined" + if args.version is not None: - re_match = re.match("([a-z0-9.]+)(-([a-z0-9]+))?$", args.cname) - assert re_match, f"not a valid version {args.version}" + re_match = re.match("([a-z0-9.]+)(-([a-z0-9]+))?$", args.version) + assert re_match, f"Not a valid version {args.version}" commit_id = re_match[3] version = re_match[1] @@ -52,34 +61,26 @@ def main(): if gardenlinux_root == "": gardenlinux_root = "." - graph = parse_features.get_features_graph( - cname_base, gardenlinux_root, feature_dir_name - ) + graph = Parser(gardenlinux_root, feature_dir_name).filter(cname_base) - sorted_features = parse_features.sort_nodes(graph) + sorted_features = Parser.sort_reversed_graph_nodes(graph) minimal_feature_set = get_minimal_feature_set(graph) - sorted_minimal_features = parse_features.sort_set( + sorted_minimal_features = sort_subset( minimal_feature_set, sorted_features ) - cname_base = get_cname_base(sorted_minimal_features) + cname = get_cname_base(sorted_minimal_features) + + if arch is not None: + cname += f"-{arch}" - cname = f"{cname_base}-{arch}" if commit_id is not None: cname += f"-{version}-{commit_id}" print(cname) -def get_cname_base(sorted_features): - return reduce( - lambda a, b : a + ("-" if not b.startswith("_") else "") + b, sorted_features - ) - -def get_minimal_feature_set(graph): - return set([node for (node, degree) in graph.in_degree() if degree == 0]) - if __name__ == "__main__": main() diff --git a/src/gardenlinux/features/parser.py b/src/gardenlinux/features/parser.py new file mode 100644 index 00000000..699c1229 --- /dev/null +++ b/src/gardenlinux/features/parser.py @@ -0,0 +1,225 @@ +from glob import glob +from typing import Callable, Optional +import logging +import networkx +import os +import re +import subprocess +import yaml + + +class Parser(object): + def __init__(self, gardenlinux_root: str = ".", feature_dir_name: str = "features", logger: Optional[logging.Logger] = None): + feature_base_dir = os.path.join(gardenlinux_root, feature_dir_name) + + if not os.access(feature_base_dir, os.R_OK): + raise ValueError("Feature directory given is invalid: {0}".format(feature_base_dir)) + + self._feature_base_dir = feature_base_dir + + self._graph = None + self._logger = logger + + if self._logger is None: + self._logger = logging.getLogger("gardenlinux.features") + + if not self._logger.hasHandlers(): + self._logger.addHandler(logging.NullHandler()) + + self._logger.debug("features.Parser initialized for directory: {0}".format(feature_base_dir)) + + @property + def graph(self) -> networkx.Graph: + if self._graph is None: + feature_yaml_files = glob("{0}/*/info.yaml".format(self._feature_base_dir)) + features = [self._read_feature_yaml(i) for i in feature_yaml_files] + + feature_graph = networkx.DiGraph() + + for feature in features: + feature_graph.add_node(feature["name"], content=feature["content"]) + + for node in feature_graph.nodes(): + node_features = self._get_node_features(feature_graph.nodes[node]) + + for attr in node_features: + if attr not in ["include", "exclude"]: + continue + + for ref in node_features[attr]: + if not os.path.isfile("{0}/{1}/info.yaml".format(self._feature_base_dir, ref)): + raise ValueError( + f"feature {node} references feature {ref}, but {feature_dir}/{ref}/info.yaml does not exist" + ) + + feature_graph.add_edge(node, ref, attr=attr) + + if not networkx.is_directed_acyclic_graph(feature_graph): + raise ValueError("Graph is not directed acyclic graph") + + self._graph = feature_graph + + return self._graph + + def filter( + self, + cname: str, + ignore_excludes: bool = False, + additional_filter_func: Optional[Callable[(str,), bool]] = None + ) -> networkx.Graph: + input_features = Parser.get_cname_as_feature_set(cname) + filter_set = input_features.copy() + + for feature in input_features: + filter_set.update(networkx.descendants(Parser._get_graph_view_for_attr(self.graph, "include"), feature)) + + graph = networkx.subgraph_view(self.graph, filter_node = self._get_filter_set_callable(filter_set, additional_filter_func)) + + if not ignore_excludes: + Parser._exclude_from_filter_set(graph, input_features, filter_set) + + return graph + + def filter_as_dict( + self, + cname: str, + ignore_excludes: bool = False, + additional_filter_func: Optional[Callable[(str,), bool]] = None + ) -> dict: + """ +:param str cname: the target cname to get the feature dict for +:param str gardenlinux_root: path of garden linux src root + +:return: dict with list of features for a given cname, split into platform, element and flag + """ + + graph = self.filter(cname, ignore_excludes, additional_filter_func) + features = Parser.sort_reversed_graph_nodes(graph) + + features_by_type = {} + + for feature in features: + node_type = Parser._get_graph_node_type(graph.nodes[feature]) + + if node_type not in features_by_type: + features_by_type[node_type] = [] + + features_by_type[node_type].append(feature) + + return features_by_type + + def filter_as_list( + self, + cname: str, + ignore_excludes: bool = False, + additional_filter_func: Optional[Callable[(str,), bool]] = None + ) -> list: + """ +:param str cname: the target cname to get the feature dict for +:param str gardenlinux_root: path of garden linux src root + +:return: list of features for a given cname + """ + + graph = self.filter(cname, ignore_excludes, additional_filter_func) + return Parser.sort_reversed_graph_nodes(graph) + + def filter_as_string( + self, + cname: str, + ignore_excludes: bool = False, + additional_filter_func: Optional[Callable[(str,), bool]] = None + ) -> str: + """ +:param str cname: the target cname to get the feature set for +:param str gardenlinux_root: path of garden linux src root + +:return: a comma separated string with the expanded feature set for the cname + """ + + graph = self.filter(cname, ignore_excludes, additional_filter_func) + features = Parser.sort_reversed_graph_nodes(graph) + + return ",".join(features) + + def _exclude_from_filter_set(graph, input_features, filter_set): + exclude_graph_view = Parser._get_graph_view_for_attr(graph, "exclude") + exclude_list = [] + + for node in networkx.lexicographical_topological_sort(graph): + for exclude in exclude_graph_view.successors(node): + if exclude not in exclude_list: + exclude_list.append(exclude) + + for exclude in exclude_list: + if exclude in input_features: + raise ValueError( + f"Excluding explicitly included feature {exclude}, unsatisfiable condition" + ) + + if exclude in filter_set: + filter_set.remove(exclude) + + if exclude_graph_view.edges(): + raise ValueError("Including explicitly excluded feature") + + def _get_node_features(self, node): + return node.get("content", {}).get("features", {}) + + def _read_feature_yaml(self, feature_yaml_file: str): + """ + Legacy function copied from gardenlinux/builder + + extracts the feature name from the feature_yaml_file param, + reads the info.yaml into a dict and outputs a dict containing the cname and the info yaml + + :param str feature_yaml_file: path to the target info.yaml that must be read + """ + + name = os.path.basename(os.path.dirname(feature_yaml_file)) + + with open(feature_yaml_file) as f: + content = yaml.safe_load(f) + + return {"name": name, "content": content} + + @staticmethod + def get_cname_as_feature_set(cname): + cname = cname.replace("_", "-_") + return set(cname.split("-")) + + @staticmethod + def _get_filter_set_callable(filter_set, additional_filter_func): + def filter_func(node): + additional_filter_result = True if additional_filter_func is None else additional_filter_func(node) + return (node in filter_set and additional_filter_result) + + return filter_func + + @staticmethod + def _get_graph_view_for_attr(graph, attr): + return networkx.subgraph_view( + graph, filter_edge = Parser._get_graph_view_for_attr_callable(graph, attr) + ) + + @staticmethod + def _get_graph_view_for_attr_callable(graph, attr): + def filter_func(a, b): + return graph.get_edge_data(a, b)["attr"] == attr + + return filter_func + + @staticmethod + def _get_graph_node_type(node): + return node.get("content", {}).get("type") + + @staticmethod + def sort_reversed_graph_nodes(graph): + def key_function(node): + prefix_map = {"platform": "0", "element": "1", "flag": "2"} + node_type = Parser._get_graph_node_type(graph.nodes.get(node, {})) + prefix = prefix_map[node_type] + + return f"{prefix}-{node}" + + return list(networkx.lexicographical_topological_sort(graph.reverse(), key = key_function)) diff --git a/src/python_gardenlinux_lib/features/parse_features.py b/src/python_gardenlinux_lib/features/parse_features.py index d5d575cf..f1e95d57 100644 --- a/src/python_gardenlinux_lib/features/parse_features.py +++ b/src/python_gardenlinux_lib/features/parse_features.py @@ -1,14 +1,12 @@ -from ..constants import GL_MEDIA_TYPE_LOOKUP, GL_MEDIA_TYPES +from gardenlinux.constants import GL_MEDIA_TYPE_LOOKUP, GL_MEDIA_TYPES -from glob import glob -import yaml +from gardenlinux.features import Parser +from typing import Optional import networkx import os import re import subprocess -from typing import Optional - -from pygments.filter import apply_filters +import yaml def get_gardenlinux_commit(gardenlinux_root: str, limit: Optional[int] = None) -> str: @@ -48,22 +46,11 @@ def get_features_dict( """ - graph = get_features_graph(cname, gardenlinux_root, feature_dir_name) - features = __reverse_sort_nodes(graph) - features_by_type = dict() - - for type in ["platform", "element", "flag"]: - features_by_type[type] = [ - feature - for feature in features - if __get_node_type(graph.nodes[feature]) == type - ] - - return features_by_type + return Parser(gardenlinux_root, feature_dir_name).filter_as_dict(cname) def get_features_graph( cname: str, gardenlinux_root: str, feature_dir_name: str = "features" -) -> networkx.graph: +) -> networkx.Graph: """ :param str cname: the target cname to get the feature dict for :param str gardenlinux_root: path of garden linux src root @@ -71,12 +58,7 @@ def get_features_graph( """ - feature_base_dir = f"{gardenlinux_root}/{feature_dir_name}" - input_features = __reverse_cname_base(cname) - feature_graph = read_feature_files(feature_base_dir) - graph = filter_graph(feature_graph, input_features) - - return graph + return Parser(gardenlinux_root, feature_dir_name).filter(cname) def get_features_list( cname: str, gardenlinux_root: str, feature_dir_name: str = "features" @@ -88,10 +70,7 @@ def get_features_list( """ - graph = get_features_graph(cname, gardenlinux_root, feature_dir_name) - features = __reverse_sort_nodes(graph) - - return features + return Parser(gardenlinux_root, feature_dir_name).filter_as_list(cname) def get_features( cname: str, gardenlinux_root: str, feature_dir_name: str = "features" @@ -102,10 +81,7 @@ def get_features( :return: a comma separated string with the expanded feature set for the cname """ - graph = get_features_graph(cname, gardenlinux_root, feature_dir_name) - features = __reverse_sort_nodes(graph) - - return ",".join(features) + return Parser(gardenlinux_root, feature_dir_name).filter_as_string(cname) def construct_layer_metadata( filetype: str, cname: str, version: str, arch: str, commit: str @@ -283,115 +259,5 @@ def deduce_filetypes_from_string(feature_dir: str, script_base_name: str): return sorted(result) -def read_feature_files(feature_dir): - """ - Legacy function copied from gardenlinux/builder - - TODO: explain the structure of the graph - - :param str feature_dir: feature directory to create the graph for - :returns: an networkx based feature graph - """ - feature_yaml_files = glob(f"{feature_dir}/*/info.yaml") - features = [parse_feature_yaml(i) for i in feature_yaml_files] - feature_graph = networkx.DiGraph() - for feature in features: - feature_graph.add_node(feature["name"], content=feature["content"]) - for node in feature_graph.nodes(): - node_features = __get_node_features(feature_graph.nodes[node]) - for attr in node_features: - if attr not in ["include", "exclude"]: - continue - for ref in node_features[attr]: - if not os.path.isfile(f"{feature_dir}/{ref}/info.yaml"): - raise ValueError( - f"feature {node} references feature {ref}, but {feature_dir}/{ref}/info.yaml does not exist" - ) - feature_graph.add_edge(node, ref, attr=attr) - if not networkx.is_directed_acyclic_graph(feature_graph): - raise ValueError("Graph is not directed acyclic graph") - return feature_graph - -def parse_feature_yaml(feature_yaml_file: str): - """ - Legacy function copied from gardenlinux/builder - - extracts the feature name from the feature_yaml_file param, - reads the info.yaml into a dict and outputs a dict containing the cname and the info yaml - - :param str feature_yaml_file: path to the target info.yaml that must be read - """ - if os.path.basename(feature_yaml_file) != "info.yaml": - raise ValueError("expected info.yaml") - name = os.path.basename(os.path.dirname(feature_yaml_file)) - with open(feature_yaml_file) as f: - content = yaml.safe_load(f) - return {"name": name, "content": content} - -def __get_node_features(node): - return node.get("content", {}).get("features", {}) - -def filter_graph(feature_graph, feature_set, ignore_excludes=False): - filter_set = set(feature_graph.nodes()) - - def filter_func(node): - return node in filter_set - - graph = networkx.subgraph_view(feature_graph, filter_node=filter_func) - graph_by_edge = dict() - for attr in ["include", "exclude"]: - edge_filter_func = ( - lambda attr: lambda a, b: graph.get_edge_data(a, b)["attr"] == attr - )(attr) - graph_by_edge[attr] = networkx.subgraph_view( - graph, filter_edge=edge_filter_func - ) - while True: - include_set = feature_set.copy() - for feature in feature_set: - include_set.update(networkx.descendants(graph_by_edge["include"], feature)) - filter_set = include_set - if ignore_excludes: - break - exclude_list = [] - for node in networkx.lexicographical_topological_sort(graph): - for exclude in graph_by_edge["exclude"].successors(node): - exclude_list.append(exclude) - if not exclude_list: - break - exclude = exclude_list[0] - if exclude in feature_set: - raise ValueError( - f"excluding explicitly included feature {exclude}, unsatisfiable condition" - ) - filter_set.remove(exclude) - if graph_by_edge["exclude"].edges() and (not ignore_excludes): - raise ValueError("Including explicitly excluded feature") - return graph - def sort_set(input_set, order_list): - return [item for item in order_list if item in input_set] - -def __sort_key(graph, node): - prefix_map = {"platform": "0", "element": "1", "flag": "2"} - node_type = __get_node_type(graph.nodes.get(node, {})) - prefix = prefix_map[node_type] - return f"{prefix}-{node}" - -def sort_nodes(graph): - def key_function(node): - return __sort_key(graph, node) - - return list(networkx.lexicographical_topological_sort(graph, key=key_function)) - -def __reverse_cname_base(cname): - cname = cname.replace("_", "-_") - return set(cname.split("-")) - -def __reverse_sort_nodes(graph): - reverse_graph = graph.reverse() - assert networkx.is_directed_acyclic_graph(reverse_graph) - return sort_nodes(reverse_graph) - -def __get_node_type(node): - return node.get("content", {}).get("type") + return [item for item in order_list if item in input_set] \ No newline at end of file From c8046d154430fe4f99c4377de85297250ec3c143 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Tue, 6 May 2025 10:49:33 +0200 Subject: [PATCH 06/21] Use more common name `setup` for GitHub action `python_lib` Signed-off-by: Tobias Wolf --- .github/actions/flavors_parse/action.yml | 2 +- .github/actions/{python_lib => setup}/action.yml | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename .github/actions/{python_lib => setup}/action.yml (100%) diff --git a/.github/actions/flavors_parse/action.yml b/.github/actions/flavors_parse/action.yml index 47733ab6..1c22a4d7 100644 --- a/.github/actions/flavors_parse/action.yml +++ b/.github/actions/flavors_parse/action.yml @@ -13,7 +13,7 @@ outputs: runs: using: composite steps: - - uses: gardenlinux/python-gardenlinux-lib/.github/actions/python_lib@feature/gardenlinux-restructure + - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@feature/gardenlinux-restructure - id: matrix shell: bash run: | diff --git a/.github/actions/python_lib/action.yml b/.github/actions/setup/action.yml similarity index 100% rename from .github/actions/python_lib/action.yml rename to .github/actions/setup/action.yml From b1884e5c37ec46e1d67c8a293f75e5d4a8c21460 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Wed, 7 May 2025 17:16:03 +0200 Subject: [PATCH 07/21] Move `logger` to module `gardenlinux` Signed-off-by: Tobias Wolf --- src/gardenlinux/features/parser.py | 13 +++++++------ src/gardenlinux/flavors/parser.py | 13 ++++++------- .../logger.py | 2 ++ 3 files changed, 15 insertions(+), 13 deletions(-) rename src/{python_gardenlinux_lib => gardenlinux}/logger.py (97%) diff --git a/src/gardenlinux/features/parser.py b/src/gardenlinux/features/parser.py index 699c1229..8d6fa7bb 100644 --- a/src/gardenlinux/features/parser.py +++ b/src/gardenlinux/features/parser.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + from glob import glob from typing import Callable, Optional import logging @@ -7,6 +9,8 @@ import subprocess import yaml +from ..logger import LoggerSetup + class Parser(object): def __init__(self, gardenlinux_root: str = ".", feature_dir_name: str = "features", logger: Optional[logging.Logger] = None): @@ -15,17 +19,14 @@ def __init__(self, gardenlinux_root: str = ".", feature_dir_name: str = "feature if not os.access(feature_base_dir, os.R_OK): raise ValueError("Feature directory given is invalid: {0}".format(feature_base_dir)) + if logger is None or not logger.hasHandlers(): + logger = LoggerSetup.get_logger("gardenlinux.features") + self._feature_base_dir = feature_base_dir self._graph = None self._logger = logger - if self._logger is None: - self._logger = logging.getLogger("gardenlinux.features") - - if not self._logger.hasHandlers(): - self._logger.addHandler(logging.NullHandler()) - self._logger.debug("features.Parser initialized for directory: {0}".format(feature_base_dir)) @property diff --git a/src/gardenlinux/flavors/parser.py b/src/gardenlinux/flavors/parser.py index 98f5a472..3f0052c3 100644 --- a/src/gardenlinux/flavors/parser.py +++ b/src/gardenlinux/flavors/parser.py @@ -1,24 +1,23 @@ +# -*- coding: utf-8 -*- + from jsonschema import validate as jsonschema_validate import fnmatch -import logging import yaml from ..constants import GL_FLAVORS_SCHEMA +from ..logger import LoggerSetup class Parser(object): def __init__(self, data, logger = None): flavors_data = (yaml.safe_load(data) if isinstance(data, str) else data) jsonschema_validate(instance=flavors_data, schema=GL_FLAVORS_SCHEMA) + if logger is None or not logger.hasHandlers(): + logger = LoggerSetup.get_logger("gardenlinux.flavors") + self._flavors_data = flavors_data self._logger = logger - if self._logger is None: - self._logger = logging.getLogger("gardenlinux.flavors") - - if not self._logger.hasHandlers(): - self._logger.addHandler(logging.NullHandler()) - self._logger.debug("flavors.Parser initialized with data: {0!r}".format(flavors_data)) def filter( diff --git a/src/python_gardenlinux_lib/logger.py b/src/gardenlinux/logger.py similarity index 97% rename from src/python_gardenlinux_lib/logger.py rename to src/gardenlinux/logger.py index 0a57d2cb..158c85ce 100644 --- a/src/python_gardenlinux_lib/logger.py +++ b/src/gardenlinux/logger.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import logging From e6327b67f4ba1b58aa9de64dcc042ffef3d42494 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Wed, 7 May 2025 17:16:55 +0200 Subject: [PATCH 08/21] Move `apt` to module `gardenlinux` Signed-off-by: Tobias Wolf --- src/{python_gardenlinux_lib => gardenlinux}/apt/__init__.py | 0 .../apt/package_repo_info.py | 2 ++ .../apt/parse_debsource.py | 1 + 3 files changed, 3 insertions(+) rename src/{python_gardenlinux_lib => gardenlinux}/apt/__init__.py (100%) rename src/{python_gardenlinux_lib => gardenlinux}/apt/package_repo_info.py (99%) rename src/{python_gardenlinux_lib => gardenlinux}/apt/parse_debsource.py (98%) diff --git a/src/python_gardenlinux_lib/apt/__init__.py b/src/gardenlinux/apt/__init__.py similarity index 100% rename from src/python_gardenlinux_lib/apt/__init__.py rename to src/gardenlinux/apt/__init__.py diff --git a/src/python_gardenlinux_lib/apt/package_repo_info.py b/src/gardenlinux/apt/package_repo_info.py similarity index 99% rename from src/python_gardenlinux_lib/apt/package_repo_info.py rename to src/gardenlinux/apt/package_repo_info.py index a6b0462b..ae758e0a 100644 --- a/src/python_gardenlinux_lib/apt/package_repo_info.py +++ b/src/gardenlinux/apt/package_repo_info.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + from apt_repo import APTRepository from typing import Optional diff --git a/src/python_gardenlinux_lib/apt/parse_debsource.py b/src/gardenlinux/apt/parse_debsource.py similarity index 98% rename from src/python_gardenlinux_lib/apt/parse_debsource.py rename to src/gardenlinux/apt/parse_debsource.py index 6fb9d450..f6f74d6d 100644 --- a/src/python_gardenlinux_lib/apt/parse_debsource.py +++ b/src/gardenlinux/apt/parse_debsource.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # SPDX-License-Identifier: MIT # Based on code from glvd https://github.com/gardenlinux/glvd/blob/7ca2ff54e01da5e9eae61d1cd565eaf75f3c62ce/src/glvd/data/debsrc.py#L1 From 96816a486bf810c96e91fde95e1d030785073b4d Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Wed, 7 May 2025 17:17:23 +0200 Subject: [PATCH 09/21] Move `git` to module `gardenlinux` Signed-off-by: Tobias Wolf --- .../git/__init__.py | 2 ++ src/gardenlinux/git/git.py | 30 +++++++++++++++++ src/python_gardenlinux_lib/git/git.py | 32 ------------------- 3 files changed, 32 insertions(+), 32 deletions(-) rename src/{python_gardenlinux_lib => gardenlinux}/git/__init__.py (61%) create mode 100755 src/gardenlinux/git/git.py delete mode 100755 src/python_gardenlinux_lib/git/git.py diff --git a/src/python_gardenlinux_lib/git/__init__.py b/src/gardenlinux/git/__init__.py similarity index 61% rename from src/python_gardenlinux_lib/git/__init__.py rename to src/gardenlinux/git/__init__.py index 9dea2260..7a9e04ae 100644 --- a/src/python_gardenlinux_lib/git/__init__.py +++ b/src/gardenlinux/git/__init__.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + from .git import Git __all__ = ["Git"] diff --git a/src/gardenlinux/git/git.py b/src/gardenlinux/git/git.py new file mode 100755 index 00000000..fcae4c8e --- /dev/null +++ b/src/gardenlinux/git/git.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- + +from git import Git as _Git +from pathlib import Path +import sys + +from ..logger import LoggerSetup + + +class Git: + """Git operations handler.""" + + def __init__(self, logger=None): + """Initialize Git handler. + + Args: + logger: Optional logger instance + """ + + if logger is None or not logger.hasHandlers(): + logger = LoggerSetup.get_logger("gardenlinux.git") + + self._logger = logger + + def get_root(self): + """Get the root directory of the current Git repository.""" + root_dir = Git(".").rev_parse("--show-superproject-working-tree") + self.log.debug(f"Git root directory: {root_dir}") + + return Path(root_dir) diff --git a/src/python_gardenlinux_lib/git/git.py b/src/python_gardenlinux_lib/git/git.py deleted file mode 100755 index ef1d77cb..00000000 --- a/src/python_gardenlinux_lib/git/git.py +++ /dev/null @@ -1,32 +0,0 @@ -import subprocess -from pathlib import Path -import sys - -from ..logger import LoggerSetup - - -class Git: - """Git operations handler.""" - - def __init__(self, logger=None): - """Initialize Git handler. - - Args: - logger: Optional logger instance - """ - self.log = logger or LoggerSetup.get_logger("gardenlinux.git") - - def get_root(self): - """Get the root directory of the current Git repository.""" - try: - root_dir = subprocess.check_output( - ["git", "rev-parse", "--show-toplevel"], text=True - ).strip() - self.log.debug(f"Git root directory: {root_dir}") - return Path(root_dir) - except subprocess.CalledProcessError as e: - self.log.error( - "Not a git repository or unable to determine root directory." - ) - self.log.debug(f"Git command failed with: {e}") - sys.exit(1) From f13917df1a1c364573ef5ceed24d7ad20350c96f Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Wed, 7 May 2025 17:17:58 +0200 Subject: [PATCH 10/21] Code cleanup Signed-off-by: Tobias Wolf --- .github/actions/flavors_parse/action.yml | 2 +- src/gardenlinux/constants.py | 2 ++ src/gardenlinux/features/__init__.py | 4 ++++ src/gardenlinux/features/__main__.py | 1 + src/gardenlinux/features/cname_main.py | 1 + src/gardenlinux/flavors/__init__.py | 4 ++++ src/gardenlinux/flavors/__main__.py | 1 + 7 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/actions/flavors_parse/action.yml b/.github/actions/flavors_parse/action.yml index 1c22a4d7..cc842247 100644 --- a/.github/actions/flavors_parse/action.yml +++ b/.github/actions/flavors_parse/action.yml @@ -2,7 +2,7 @@ name: flavors_parse description: Parses the given GardenLinux flavors parameters inputs: flags: - description: 'Flags passed to bin/flavors_parse.py' + description: 'Flags passed to `gl-flavors-parse`' required: true flavors_matrix: description: 'Generated GitHub workflow flavors matrix' diff --git a/src/gardenlinux/constants.py b/src/gardenlinux/constants.py index 55219ec9..8160f367 100644 --- a/src/gardenlinux/constants.py +++ b/src/gardenlinux/constants.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + # GardenLinux flavors schema for validation GL_FLAVORS_SCHEMA = { "type": "object", diff --git a/src/gardenlinux/features/__init__.py b/src/gardenlinux/features/__init__.py index 2a3855a1..bff333bd 100644 --- a/src/gardenlinux/features/__init__.py +++ b/src/gardenlinux/features/__init__.py @@ -1 +1,5 @@ +# -*- coding: utf-8 -*- + from .parser import Parser + +__all__ = ["Parser"] diff --git a/src/gardenlinux/features/__main__.py b/src/gardenlinux/features/__main__.py index 29c43bea..c93cf077 100644 --- a/src/gardenlinux/features/__main__.py +++ b/src/gardenlinux/features/__main__.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# -*- coding: utf-8 -*- from .parser import Parser diff --git a/src/gardenlinux/features/cname_main.py b/src/gardenlinux/features/cname_main.py index 7bc2e4b6..86d2500a 100644 --- a/src/gardenlinux/features/cname_main.py +++ b/src/gardenlinux/features/cname_main.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# -*- coding: utf-8 -*- from functools import reduce from os.path import basename, dirname diff --git a/src/gardenlinux/flavors/__init__.py b/src/gardenlinux/flavors/__init__.py index 2a3855a1..bff333bd 100644 --- a/src/gardenlinux/flavors/__init__.py +++ b/src/gardenlinux/flavors/__init__.py @@ -1 +1,5 @@ +# -*- coding: utf-8 -*- + from .parser import Parser + +__all__ = ["Parser"] diff --git a/src/gardenlinux/flavors/__main__.py b/src/gardenlinux/flavors/__main__.py index 893eb879..1dd12a79 100644 --- a/src/gardenlinux/flavors/__main__.py +++ b/src/gardenlinux/flavors/__main__.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# -*- coding: utf-8 -*- from argparse import ArgumentParser from git import Git From 471c9fc19a81a431a7e432252914706e4d9068f2 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Wed, 7 May 2025 17:18:08 +0200 Subject: [PATCH 11/21] Add GitHub action `features_parse` Signed-off-by: Tobias Wolf --- .github/actions/features_parse/action.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/actions/features_parse/action.yml diff --git a/.github/actions/features_parse/action.yml b/.github/actions/features_parse/action.yml new file mode 100644 index 00000000..2ea77b4a --- /dev/null +++ b/.github/actions/features_parse/action.yml @@ -0,0 +1,18 @@ +name: features_parse +description: Parses the given GardenLinux features parameters +inputs: + flags: + description: 'Flags passed to `gl-features-parse`' + required: true +outputs: + result: + description: 'features result' + value: ${{ steps.result.outputs.result }} +runs: + using: composite + steps: + - uses: gardenlinux/python-gardenlinux-lib/.github/actions/setup@feature/gardenlinux-restructure + - id: result + shell: bash + run: | + echo "result=$(gl-features-parse ${{ inputs.flags }})" | tee -a $GITHUB_OUTPUT From 05be222137494150578402a6b65733fbfe7d8a03 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Wed, 7 May 2025 17:35:09 +0200 Subject: [PATCH 12/21] Add support to read `COMMIT` and `VERSION` files Signed-off-by: Tobias Wolf --- src/gardenlinux/features/__main__.py | 25 +++++++++++++++++++++---- src/gardenlinux/features/cname_main.py | 7 +++++-- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/gardenlinux/features/__main__.py b/src/gardenlinux/features/__main__.py index c93cf077..f8ddb5fb 100644 --- a/src/gardenlinux/features/__main__.py +++ b/src/gardenlinux/features/__main__.py @@ -4,9 +4,9 @@ from .parser import Parser from functools import reduce -from os.path import basename, dirname - +from os import path import argparse +import os import re import sys @@ -44,6 +44,7 @@ def main(): arch = None cname_base = None commit_id = None + gardenlinux_root = path.dirname(args.feature_dir) version = None if args.cname: @@ -85,12 +86,14 @@ def main(): assert args.default_arch, "Architecture could not be determined and no default architecture set" arch = args.default_arch + if not commit_id or not version: + version, commit_id = get_version_and_commit_id_from_files(gardenlinux_root) + if not version and (args.type in ("cname", "version" )): assert args.default_version, "version not specified and no default version set" version = args.default_version - gardenlinux_root = dirname(args.feature_dir) - feature_dir_name = basename(args.feature_dir) + feature_dir_name = path.basename(args.feature_dir) if gardenlinux_root == "": gardenlinux_root = "." @@ -149,6 +152,20 @@ def get_cname_base(sorted_features): lambda a, b : a + ("-" if not b.startswith("_") else "") + b, sorted_features ) +def get_version_and_commit_id_from_files(gardenlinux_root): + commit_id = None + version = None + + if os.access(path.join(gardenlinux_root, "COMMIT"), os.R_OK): + with open(path.join(gardenlinux_root, "COMMIT"), "r") as fp: + commit_id = fp.read().strip() + + if os.access(path.join(gardenlinux_root, "VERSION"), os.R_OK): + with open(path.join(gardenlinux_root, "VERSION"), "r") as fp: + version = fp.read().strip() + + return (version, commit_id) + def get_minimal_feature_set(graph): return set([node for (node, degree) in graph.in_degree() if degree == 0]) diff --git a/src/gardenlinux/features/cname_main.py b/src/gardenlinux/features/cname_main.py index 86d2500a..e3dfe6ae 100644 --- a/src/gardenlinux/features/cname_main.py +++ b/src/gardenlinux/features/cname_main.py @@ -6,7 +6,7 @@ import argparse import re -from .__main__ import get_cname_base, get_minimal_feature_set, sort_subset +from .__main__ import get_cname_base, get_minimal_feature_set, get_version_and_commit_id_from_files, sort_subset from .parser import Parser @@ -29,6 +29,7 @@ def main(): arch = None commit_id = None + gardenlinux_root = dirname(args.feature_dir) version = None if re_match.lastindex == 1: @@ -49,6 +50,9 @@ def main(): assert arch is None or arch == "", "Architecture could not be determined" + if not commit_id or not version: + version, commit_id = get_version_and_commit_id_from_files(gardenlinux_root) + if args.version is not None: re_match = re.match("([a-z0-9.]+)(-([a-z0-9]+))?$", args.version) assert re_match, f"Not a valid version {args.version}" @@ -56,7 +60,6 @@ def main(): commit_id = re_match[3] version = re_match[1] - gardenlinux_root = dirname(args.feature_dir) feature_dir_name = basename(args.feature_dir) if gardenlinux_root == "": From e798faa038555a3a8f693ff50daf98052c4b5dfd Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Wed, 7 May 2025 17:40:53 +0200 Subject: [PATCH 13/21] Fix architecture `assert` in `gl-cname` Signed-off-by: Tobias Wolf --- src/gardenlinux/features/cname_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gardenlinux/features/cname_main.py b/src/gardenlinux/features/cname_main.py index e3dfe6ae..01ed6df3 100644 --- a/src/gardenlinux/features/cname_main.py +++ b/src/gardenlinux/features/cname_main.py @@ -48,7 +48,7 @@ def main(): if args.arch is not None: arch = args.arch - assert arch is None or arch == "", "Architecture could not be determined" + assert arch is not None and arch != "", "Architecture could not be determined" if not commit_id or not version: version, commit_id = get_version_and_commit_id_from_files(gardenlinux_root) From a2904183f7d37b73a1699c6845d118096580be96 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Thu, 8 May 2025 07:59:27 +0200 Subject: [PATCH 14/21] Always shorten commit ID to 8 characters for "cname" creation Signed-off-by: Tobias Wolf --- src/gardenlinux/features/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gardenlinux/features/__main__.py b/src/gardenlinux/features/__main__.py index f8ddb5fb..adef80ea 100644 --- a/src/gardenlinux/features/__main__.py +++ b/src/gardenlinux/features/__main__.py @@ -158,7 +158,7 @@ def get_version_and_commit_id_from_files(gardenlinux_root): if os.access(path.join(gardenlinux_root, "COMMIT"), os.R_OK): with open(path.join(gardenlinux_root, "COMMIT"), "r") as fp: - commit_id = fp.read().strip() + commit_id = fp.read().strip()[:8] if os.access(path.join(gardenlinux_root, "VERSION"), os.R_OK): with open(path.join(gardenlinux_root, "VERSION"), "r") as fp: From ad5dcd61da535eafb58532e32cdd9cacbb06f489 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Thu, 8 May 2025 16:49:40 +0200 Subject: [PATCH 15/21] Fix "cname" generation with non-reversed graph as source Signed-off-by: Tobias Wolf --- src/gardenlinux/features/__main__.py | 3 +-- src/gardenlinux/features/cname_main.py | 3 +-- src/gardenlinux/features/parser.py | 8 ++++++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/gardenlinux/features/__main__.py b/src/gardenlinux/features/__main__.py index adef80ea..3b483215 100644 --- a/src/gardenlinux/features/__main__.py +++ b/src/gardenlinux/features/__main__.py @@ -108,8 +108,7 @@ def main(): elif args.type in ( "cname_base", "cname", "graph" ): graph = Parser(gardenlinux_root, feature_dir_name).filter(cname_base, additional_filter_func = additional_filter_func) - sorted_features = Parser.sort_reversed_graph_nodes(graph) - + sorted_features = Parser.sort_graph_nodes(graph) minimal_feature_set = get_minimal_feature_set(graph) sorted_minimal_features = sort_subset( diff --git a/src/gardenlinux/features/cname_main.py b/src/gardenlinux/features/cname_main.py index 01ed6df3..4d1e18f5 100644 --- a/src/gardenlinux/features/cname_main.py +++ b/src/gardenlinux/features/cname_main.py @@ -67,8 +67,7 @@ def main(): graph = Parser(gardenlinux_root, feature_dir_name).filter(cname_base) - sorted_features = Parser.sort_reversed_graph_nodes(graph) - + sorted_features = Parser.sort_graph_nodes(graph) minimal_feature_set = get_minimal_feature_set(graph) sorted_minimal_features = sort_subset( diff --git a/src/gardenlinux/features/parser.py b/src/gardenlinux/features/parser.py index 8d6fa7bb..a79468ff 100644 --- a/src/gardenlinux/features/parser.py +++ b/src/gardenlinux/features/parser.py @@ -215,7 +215,7 @@ def _get_graph_node_type(node): return node.get("content", {}).get("type") @staticmethod - def sort_reversed_graph_nodes(graph): + def sort_graph_nodes(graph): def key_function(node): prefix_map = {"platform": "0", "element": "1", "flag": "2"} node_type = Parser._get_graph_node_type(graph.nodes.get(node, {})) @@ -223,4 +223,8 @@ def key_function(node): return f"{prefix}-{node}" - return list(networkx.lexicographical_topological_sort(graph.reverse(), key = key_function)) + return list(networkx.lexicographical_topological_sort(graph, key = key_function)) + + @staticmethod + def sort_reversed_graph_nodes(graph): + return Parser.sort_graph_nodes(graph.reverse()) From d39426166040bc504099aea8efb9f4e60d42f67c Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Thu, 8 May 2025 17:30:27 +0200 Subject: [PATCH 16/21] Fix "cname" creation from flavor if `--arch` is specified Signed-off-by: Tobias Wolf --- src/gardenlinux/features/__main__.py | 9 ++++++--- src/gardenlinux/features/cname_main.py | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/gardenlinux/features/__main__.py b/src/gardenlinux/features/__main__.py index 3b483215..1dda210f 100644 --- a/src/gardenlinux/features/__main__.py +++ b/src/gardenlinux/features/__main__.py @@ -58,10 +58,13 @@ def main(): if re_match.lastindex == 1: data_splitted = re_match[1].split("-", 1) - if len(data_splitted) > 1: - arch = data_splitted[1] - cname_base = data_splitted[0] + + if len(data_splitted) > 1: + if args.arch is None: + arch = data_splitted[1] + else: + cname_base += "-" + data_splitted[1] else: arch = re_match[4] cname_base = re_match[1] diff --git a/src/gardenlinux/features/cname_main.py b/src/gardenlinux/features/cname_main.py index 4d1e18f5..75cba2b9 100644 --- a/src/gardenlinux/features/cname_main.py +++ b/src/gardenlinux/features/cname_main.py @@ -35,10 +35,13 @@ def main(): if re_match.lastindex == 1: data_splitted = re_match[1].split("-", 1) - if len(data_splitted) > 1: - arch = data_splitted[1] - cname_base = data_splitted[0] + + if len(data_splitted) > 1: + if args.arch is None: + arch = data_splitted[1] + else: + cname_base += "-" + data_splitted[1] else: arch = re_match[4] cname_base = re_match[1] From 22cef2af1d137c1695e4d32c82e2f5afb8187214 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Mon, 12 May 2025 18:06:37 +0200 Subject: [PATCH 17/21] Add "special" `bare` feature as `platform` for the graph parser Signed-off-by: Tobias Wolf --- src/gardenlinux/constants.py | 3 +++ src/gardenlinux/features/parser.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/src/gardenlinux/constants.py b/src/gardenlinux/constants.py index 8160f367..05427cad 100644 --- a/src/gardenlinux/constants.py +++ b/src/gardenlinux/constants.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- +# GardenLinux "bare" feature +BARE_FLAVOR_FEATURE_CONTENT = { "description": "Bare flavor", "type": "platform" } + # GardenLinux flavors schema for validation GL_FLAVORS_SCHEMA = { "type": "object", diff --git a/src/gardenlinux/features/parser.py b/src/gardenlinux/features/parser.py index a79468ff..2e056e69 100644 --- a/src/gardenlinux/features/parser.py +++ b/src/gardenlinux/features/parser.py @@ -9,6 +9,7 @@ import subprocess import yaml +from ..constants import BARE_FLAVOR_FEATURE_CONTENT from ..logger import LoggerSetup @@ -72,6 +73,11 @@ def filter( filter_set = input_features.copy() for feature in input_features: + # @TODO: Remove "special" handling once "bare" is a first-class citizen of the feature graph + if feature == "bare": + self.graph.add_node("bare", content=BARE_FLAVOR_FEATURE_CONTENT) + continue + filter_set.update(networkx.descendants(Parser._get_graph_view_for_attr(self.graph, "include"), feature)) graph = networkx.subgraph_view(self.graph, filter_node = self._get_filter_set_callable(filter_set, additional_filter_func)) From 8eefc933bd0f3f61d028a70f9782f8f4f02d8789 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Wed, 14 May 2025 17:57:38 +0200 Subject: [PATCH 18/21] Add "special" bare `libc` feature as `element` for the graph parser Signed-off-by: Tobias Wolf --- src/gardenlinux/constants.py | 1 + src/gardenlinux/features/parser.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/gardenlinux/constants.py b/src/gardenlinux/constants.py index 05427cad..3c1d274f 100644 --- a/src/gardenlinux/constants.py +++ b/src/gardenlinux/constants.py @@ -2,6 +2,7 @@ # GardenLinux "bare" feature BARE_FLAVOR_FEATURE_CONTENT = { "description": "Bare flavor", "type": "platform" } +BARE_FLAVOR_LIBC_FEATURE_CONTENT = { "description": "Bare libc feature", "type": "element" } # GardenLinux flavors schema for validation GL_FLAVORS_SCHEMA = { diff --git a/src/gardenlinux/features/parser.py b/src/gardenlinux/features/parser.py index 2e056e69..7e7bbd4b 100644 --- a/src/gardenlinux/features/parser.py +++ b/src/gardenlinux/features/parser.py @@ -9,7 +9,7 @@ import subprocess import yaml -from ..constants import BARE_FLAVOR_FEATURE_CONTENT +from ..constants import BARE_FLAVOR_FEATURE_CONTENT, BARE_FLAVOR_LIBC_FEATURE_CONTENT from ..logger import LoggerSetup @@ -75,7 +75,11 @@ def filter( for feature in input_features: # @TODO: Remove "special" handling once "bare" is a first-class citizen of the feature graph if feature == "bare": - self.graph.add_node("bare", content=BARE_FLAVOR_FEATURE_CONTENT) + if not self.graph.has_node("bare"): + self.graph.add_node("bare", content=BARE_FLAVOR_FEATURE_CONTENT) + if not self.graph.has_node("libc"): + self.graph.add_node("libc", content=BARE_FLAVOR_LIBC_FEATURE_CONTENT) + continue filter_set.update(networkx.descendants(Parser._get_graph_view_for_attr(self.graph, "include"), feature)) From 4b57caf006c03083b176055f4e08e282a8436f17 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Thu, 15 May 2025 08:23:58 +0200 Subject: [PATCH 19/21] Fix race-condition in `features.Parser.filter()` Signed-off-by: Tobias Wolf --- src/gardenlinux/features/parser.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/gardenlinux/features/parser.py b/src/gardenlinux/features/parser.py index 7e7bbd4b..2eff0fca 100644 --- a/src/gardenlinux/features/parser.py +++ b/src/gardenlinux/features/parser.py @@ -72,16 +72,14 @@ def filter( input_features = Parser.get_cname_as_feature_set(cname) filter_set = input_features.copy() - for feature in input_features: - # @TODO: Remove "special" handling once "bare" is a first-class citizen of the feature graph - if feature == "bare": - if not self.graph.has_node("bare"): - self.graph.add_node("bare", content=BARE_FLAVOR_FEATURE_CONTENT) - if not self.graph.has_node("libc"): - self.graph.add_node("libc", content=BARE_FLAVOR_LIBC_FEATURE_CONTENT) - - continue + # @TODO: Remove "special" handling once "bare" is a first-class citizen of the feature graph + if "bare" in input_features: + if not self.graph.has_node("bare"): + self.graph.add_node("bare", content=BARE_FLAVOR_FEATURE_CONTENT) + if not self.graph.has_node("libc"): + self.graph.add_node("libc", content=BARE_FLAVOR_LIBC_FEATURE_CONTENT) + for feature in input_features: filter_set.update(networkx.descendants(Parser._get_graph_view_for_attr(self.graph, "include"), feature)) graph = networkx.subgraph_view(self.graph, filter_node = self._get_filter_set_callable(filter_set, additional_filter_func)) From 377e95af0836e627897cfe4fbbaaf19bd124d3e8 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Thu, 15 May 2025 14:47:45 +0200 Subject: [PATCH 20/21] Fix RegEx to parse cname as `platform_element` correctly Signed-off-by: Tobias Wolf --- src/gardenlinux/features/__main__.py | 2 +- src/gardenlinux/features/cname_main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gardenlinux/features/__main__.py b/src/gardenlinux/features/__main__.py index 1dda210f..d2e50864 100644 --- a/src/gardenlinux/features/__main__.py +++ b/src/gardenlinux/features/__main__.py @@ -49,7 +49,7 @@ def main(): if args.cname: re_match = re.match( - "([a-zA-Z0-9]+(-[a-zA-Z0-9\\_\\-]*?)?)(-([a-z0-9]+)(-([a-z0-9.]+)-([a-z0-9]+))*)?$", + "([a-zA-Z0-9]+([\\_\\-][a-zA-Z0-9]+)+?)(-([a-z0-9]+)(-([a-z0-9.]+)-([a-z0-9]+))*)?$", args.cname ) diff --git a/src/gardenlinux/features/cname_main.py b/src/gardenlinux/features/cname_main.py index 75cba2b9..bbb1e312 100644 --- a/src/gardenlinux/features/cname_main.py +++ b/src/gardenlinux/features/cname_main.py @@ -21,7 +21,7 @@ def main(): args = parser.parse_args() re_match = re.match( - "([a-zA-Z0-9]+(-[a-zA-Z0-9\\_\\-]*?)?)(-([a-z0-9]+)(-([a-z0-9.]+)-([a-z0-9]+))*)?$", + "([a-zA-Z0-9]+([\\_\\-][a-zA-Z0-9]+)+?)(-([a-z0-9]+)(-([a-z0-9.]+)-([a-z0-9]+))*)?$", args.cname ) From 78a7ae24d0507cc9d6c1dfe6eb79f3766f0e9f7a Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Thu, 15 May 2025 15:40:09 +0200 Subject: [PATCH 21/21] Remove unused code in `python_gardenlinux_lib` module Signed-off-by: Tobias Wolf --- src/gardenlinux/apt/__init__.py | 5 + .../apt/{parse_debsource.py => debsource.py} | 0 src/gardenlinux/constants.py | 19 +- src/gardenlinux/features/__main__.py | 86 ++++--- src/gardenlinux/features/cname_main.py | 13 +- src/gardenlinux/features/parser.py | 68 ++++-- src/gardenlinux/flavors/__main__.py | 36 +-- src/gardenlinux/flavors/parser.py | 47 ++-- src/python_gardenlinux_lib/__init__.py | 3 +- .../features/parse_features.py | 62 ++--- .../flavors/parse_flavors.py | 230 ------------------ src/python_gardenlinux_lib/version.py | 3 +- tests/test_get_features_dict.py | 4 +- tests/test_parse_debsource.py | 2 +- tests/test_push_image.py | 10 +- 15 files changed, 203 insertions(+), 385 deletions(-) rename src/gardenlinux/apt/{parse_debsource.py => debsource.py} (100%) delete mode 100755 src/python_gardenlinux_lib/flavors/parse_flavors.py diff --git a/src/gardenlinux/apt/__init__.py b/src/gardenlinux/apt/__init__.py index e69de29b..d4c5261e 100644 --- a/src/gardenlinux/apt/__init__.py +++ b/src/gardenlinux/apt/__init__.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- + +from .debsource import Debsrc, DebsrcFile + +__all__ = ["Parser"] diff --git a/src/gardenlinux/apt/parse_debsource.py b/src/gardenlinux/apt/debsource.py similarity index 100% rename from src/gardenlinux/apt/parse_debsource.py rename to src/gardenlinux/apt/debsource.py diff --git a/src/gardenlinux/constants.py b/src/gardenlinux/constants.py index 3c1d274f..49bfbed6 100644 --- a/src/gardenlinux/constants.py +++ b/src/gardenlinux/constants.py @@ -1,8 +1,12 @@ # -*- coding: utf-8 -*- # GardenLinux "bare" feature -BARE_FLAVOR_FEATURE_CONTENT = { "description": "Bare flavor", "type": "platform" } -BARE_FLAVOR_LIBC_FEATURE_CONTENT = { "description": "Bare libc feature", "type": "element" } +BARE_FLAVOR_FEATURE_CONTENT = {"description": "Bare flavor", "type": "platform"} + +BARE_FLAVOR_LIBC_FEATURE_CONTENT = { + "description": "Bare libc feature", + "type": "element", +} # GardenLinux flavors schema for validation GL_FLAVORS_SCHEMA = { @@ -31,7 +35,14 @@ "test-platform": {"type": "boolean"}, "publish": {"type": "boolean"}, }, - "required": ["features", "arch", "build", "test", "test-platform", "publish"], + "required": [ + "features", + "arch", + "build", + "test", + "test-platform", + "publish", + ], }, }, }, @@ -39,7 +50,7 @@ }, }, }, - "required": ["targets"] + "required": ["targets"], } # It is important that this list is sorted in descending length of the entries diff --git a/src/gardenlinux/features/__main__.py b/src/gardenlinux/features/__main__.py index d2e50864..0c2a3378 100644 --- a/src/gardenlinux/features/__main__.py +++ b/src/gardenlinux/features/__main__.py @@ -20,26 +20,38 @@ "elements", "arch", "version", - "graph" + "graph", ] def main(): parser = argparse.ArgumentParser() - parser.add_argument("--arch", dest = "arch") - parser.add_argument("--feature-dir", default = "features") - parser.add_argument("--features", type = lambda arg: set([f for f in arg.split(",") if f])) - parser.add_argument("--ignore", dest = "ignore", type = lambda arg: set([f for f in arg.split(",") if f]), default = set()) + parser.add_argument("--arch", dest="arch") + parser.add_argument("--feature-dir", default="features") parser.add_argument("--cname") parser.add_argument("--default-arch") parser.add_argument("--default-version") parser.add_argument("--version", dest="version") - parser.add_argument("type", nargs="?", choices = _ARGS_TYPE_ALLOWED, default = "cname") + + parser.add_argument( + "--features", type=lambda arg: set([f for f in arg.split(",") if f]) + ) + + parser.add_argument( + "--ignore", + dest="ignore", + type=lambda arg: set([f for f in arg.split(",") if f]), + default=set(), + ) + + parser.add_argument("type", nargs="?", choices=_ARGS_TYPE_ALLOWED, default="cname") args = parser.parse_args() - assert bool(args.features) or bool(args.cname), "Please provide either `--features` or `--cname` argument" + assert bool(args.features) or bool( + args.cname + ), "Please provide either `--features` or `--cname` argument" arch = None cname_base = None @@ -50,7 +62,7 @@ def main(): if args.cname: re_match = re.match( "([a-zA-Z0-9]+([\\_\\-][a-zA-Z0-9]+)+?)(-([a-z0-9]+)(-([a-z0-9.]+)-([a-z0-9]+))*)?$", - args.cname + args.cname, ) assert re_match, f"Not a valid GardenLinux canonical name {args.cname}" @@ -85,14 +97,16 @@ def main(): commit_id = re_match[3] version = re_match[1] - if arch is None or arch == "" and (args.type in ( "cname", "arch" )): - assert args.default_arch, "Architecture could not be determined and no default architecture set" + if arch is None or arch == "" and (args.type in ("cname", "arch")): + assert ( + args.default_arch + ), "Architecture could not be determined and no default architecture set" arch = args.default_arch if not commit_id or not version: version, commit_id = get_version_and_commit_id_from_files(gardenlinux_root) - if not version and (args.type in ("cname", "version" )): + if not version and (args.type in ("cname", "version")): assert args.default_version, "version not specified and no default version set" version = args.default_version @@ -108,15 +122,15 @@ def main(): if args.type == "arch": print(arch) - elif args.type in ( "cname_base", "cname", "graph" ): - graph = Parser(gardenlinux_root, feature_dir_name).filter(cname_base, additional_filter_func = additional_filter_func) + elif args.type in ("cname_base", "cname", "graph"): + graph = Parser(gardenlinux_root, feature_dir_name).filter( + cname_base, additional_filter_func=additional_filter_func + ) sorted_features = Parser.sort_graph_nodes(graph) minimal_feature_set = get_minimal_feature_set(graph) - sorted_minimal_features = sort_subset( - minimal_feature_set, sorted_features - ) + sorted_minimal_features = sort_subset(minimal_feature_set, sorted_features) cname_base = get_cname_base(sorted_minimal_features) @@ -135,9 +149,15 @@ def main(): elif args.type == "graph": print(graph_as_mermaid_markup(cname_base, graph)) elif args.type == "features": - print(Parser(gardenlinux_root, feature_dir_name).filter_as_string(cname_base, additional_filter_func = additional_filter_func)) - elif args.type in ( "flags", "elements", "platforms" ): - features_by_type = Parser(gardenlinux_root, feature_dir_name).filter_as_dict(cname_base, additional_filter_func = additional_filter_func) + print( + Parser(gardenlinux_root, feature_dir_name).filter_as_string( + cname_base, additional_filter_func=additional_filter_func + ) + ) + elif args.type in ("flags", "elements", "platforms"): + features_by_type = Parser(gardenlinux_root, feature_dir_name).filter_as_dict( + cname_base, additional_filter_func=additional_filter_func + ) if args.type == "platforms": print(",".join(features_by_type["platform"])) @@ -151,9 +171,10 @@ def main(): def get_cname_base(sorted_features): return reduce( - lambda a, b : a + ("-" if not b.startswith("_") else "") + b, sorted_features + lambda a, b: a + ("-" if not b.startswith("_") else "") + b, sorted_features ) + def get_version_and_commit_id_from_files(gardenlinux_root): commit_id = None version = None @@ -168,21 +189,24 @@ def get_version_and_commit_id_from_files(gardenlinux_root): return (version, commit_id) + def get_minimal_feature_set(graph): return set([node for (node, degree) in graph.in_degree() if degree == 0]) + def graph_as_mermaid_markup(cname_base, graph): - """ - Generates a mermaid.js representation of the graph. - This is helpful to identify dependencies between features. - - Syntax docs: - https://mermaid.js.org/syntax/flowchart.html?id=flowcharts-basic-syntax - """ - markup = f"---\ntitle: Dependency Graph for Feature {cname_base}\n---\ngraph TD;\n" - for u,v in graph.edges: - markup += f" {u}-->{v};\n" - return markup + """ + Generates a mermaid.js representation of the graph. + This is helpful to identify dependencies between features. + + Syntax docs: + https://mermaid.js.org/syntax/flowchart.html?id=flowcharts-basic-syntax + """ + markup = f"---\ntitle: Dependency Graph for Feature {cname_base}\n---\ngraph TD;\n" + for u, v in graph.edges: + markup += f" {u}-->{v};\n" + return markup + def sort_subset(input_set, order_list): return [item for item in order_list if item in input_set] diff --git a/src/gardenlinux/features/cname_main.py b/src/gardenlinux/features/cname_main.py index bbb1e312..5743f13d 100644 --- a/src/gardenlinux/features/cname_main.py +++ b/src/gardenlinux/features/cname_main.py @@ -6,7 +6,12 @@ import argparse import re -from .__main__ import get_cname_base, get_minimal_feature_set, get_version_and_commit_id_from_files, sort_subset +from .__main__ import ( + get_cname_base, + get_minimal_feature_set, + get_version_and_commit_id_from_files, + sort_subset, +) from .parser import Parser @@ -22,7 +27,7 @@ def main(): re_match = re.match( "([a-zA-Z0-9]+([\\_\\-][a-zA-Z0-9]+)+?)(-([a-z0-9]+)(-([a-z0-9.]+)-([a-z0-9]+))*)?$", - args.cname + args.cname, ) assert re_match, f"Not a valid GardenLinux canonical name {args.cname}" @@ -73,9 +78,7 @@ def main(): sorted_features = Parser.sort_graph_nodes(graph) minimal_feature_set = get_minimal_feature_set(graph) - sorted_minimal_features = sort_subset( - minimal_feature_set, sorted_features - ) + sorted_minimal_features = sort_subset(minimal_feature_set, sorted_features) cname = get_cname_base(sorted_minimal_features) diff --git a/src/gardenlinux/features/parser.py b/src/gardenlinux/features/parser.py index 2eff0fca..a016e15c 100644 --- a/src/gardenlinux/features/parser.py +++ b/src/gardenlinux/features/parser.py @@ -14,11 +14,18 @@ class Parser(object): - def __init__(self, gardenlinux_root: str = ".", feature_dir_name: str = "features", logger: Optional[logging.Logger] = None): + def __init__( + self, + gardenlinux_root: str = ".", + feature_dir_name: str = "features", + logger: Optional[logging.Logger] = None, + ): feature_base_dir = os.path.join(gardenlinux_root, feature_dir_name) if not os.access(feature_base_dir, os.R_OK): - raise ValueError("Feature directory given is invalid: {0}".format(feature_base_dir)) + raise ValueError( + "Feature directory given is invalid: {0}".format(feature_base_dir) + ) if logger is None or not logger.hasHandlers(): logger = LoggerSetup.get_logger("gardenlinux.features") @@ -28,7 +35,9 @@ def __init__(self, gardenlinux_root: str = ".", feature_dir_name: str = "feature self._graph = None self._logger = logger - self._logger.debug("features.Parser initialized for directory: {0}".format(feature_base_dir)) + self._logger.debug( + "features.Parser initialized for directory: {0}".format(feature_base_dir) + ) @property def graph(self) -> networkx.Graph: @@ -49,7 +58,9 @@ def graph(self) -> networkx.Graph: continue for ref in node_features[attr]: - if not os.path.isfile("{0}/{1}/info.yaml".format(self._feature_base_dir, ref)): + if not os.path.isfile( + "{0}/{1}/info.yaml".format(self._feature_base_dir, ref) + ): raise ValueError( f"feature {node} references feature {ref}, but {feature_dir}/{ref}/info.yaml does not exist" ) @@ -67,7 +78,7 @@ def filter( self, cname: str, ignore_excludes: bool = False, - additional_filter_func: Optional[Callable[(str,), bool]] = None + additional_filter_func: Optional[Callable[(str,), bool]] = None, ) -> networkx.Graph: input_features = Parser.get_cname_as_feature_set(cname) filter_set = input_features.copy() @@ -80,9 +91,18 @@ def filter( self.graph.add_node("libc", content=BARE_FLAVOR_LIBC_FEATURE_CONTENT) for feature in input_features: - filter_set.update(networkx.descendants(Parser._get_graph_view_for_attr(self.graph, "include"), feature)) + filter_set.update( + networkx.descendants( + Parser._get_graph_view_for_attr(self.graph, "include"), feature + ) + ) - graph = networkx.subgraph_view(self.graph, filter_node = self._get_filter_set_callable(filter_set, additional_filter_func)) + graph = networkx.subgraph_view( + self.graph, + filter_node=self._get_filter_set_callable( + filter_set, additional_filter_func + ), + ) if not ignore_excludes: Parser._exclude_from_filter_set(graph, input_features, filter_set) @@ -93,13 +113,13 @@ def filter_as_dict( self, cname: str, ignore_excludes: bool = False, - additional_filter_func: Optional[Callable[(str,), bool]] = None + additional_filter_func: Optional[Callable[(str,), bool]] = None, ) -> dict: """ -:param str cname: the target cname to get the feature dict for -:param str gardenlinux_root: path of garden linux src root + :param str cname: the target cname to get the feature dict for + :param str gardenlinux_root: path of garden linux src root -:return: dict with list of features for a given cname, split into platform, element and flag + :return: dict with list of features for a given cname, split into platform, element and flag """ graph = self.filter(cname, ignore_excludes, additional_filter_func) @@ -121,13 +141,13 @@ def filter_as_list( self, cname: str, ignore_excludes: bool = False, - additional_filter_func: Optional[Callable[(str,), bool]] = None + additional_filter_func: Optional[Callable[(str,), bool]] = None, ) -> list: """ -:param str cname: the target cname to get the feature dict for -:param str gardenlinux_root: path of garden linux src root + :param str cname: the target cname to get the feature dict for + :param str gardenlinux_root: path of garden linux src root -:return: list of features for a given cname + :return: list of features for a given cname """ graph = self.filter(cname, ignore_excludes, additional_filter_func) @@ -137,13 +157,13 @@ def filter_as_string( self, cname: str, ignore_excludes: bool = False, - additional_filter_func: Optional[Callable[(str,), bool]] = None + additional_filter_func: Optional[Callable[(str,), bool]] = None, ) -> str: """ -:param str cname: the target cname to get the feature set for -:param str gardenlinux_root: path of garden linux src root + :param str cname: the target cname to get the feature set for + :param str gardenlinux_root: path of garden linux src root -:return: a comma separated string with the expanded feature set for the cname + :return: a comma separated string with the expanded feature set for the cname """ graph = self.filter(cname, ignore_excludes, additional_filter_func) @@ -200,15 +220,17 @@ def get_cname_as_feature_set(cname): @staticmethod def _get_filter_set_callable(filter_set, additional_filter_func): def filter_func(node): - additional_filter_result = True if additional_filter_func is None else additional_filter_func(node) - return (node in filter_set and additional_filter_result) + additional_filter_result = ( + True if additional_filter_func is None else additional_filter_func(node) + ) + return node in filter_set and additional_filter_result return filter_func @staticmethod def _get_graph_view_for_attr(graph, attr): return networkx.subgraph_view( - graph, filter_edge = Parser._get_graph_view_for_attr_callable(graph, attr) + graph, filter_edge=Parser._get_graph_view_for_attr_callable(graph, attr) ) @staticmethod @@ -231,7 +253,7 @@ def key_function(node): return f"{prefix}-{node}" - return list(networkx.lexicographical_topological_sort(graph, key = key_function)) + return list(networkx.lexicographical_topological_sort(graph, key=key_function)) @staticmethod def sort_reversed_graph_nodes(graph): diff --git a/src/gardenlinux/flavors/__main__.py b/src/gardenlinux/flavors/__main__.py index 1dd12a79..bff775be 100644 --- a/src/gardenlinux/flavors/__main__.py +++ b/src/gardenlinux/flavors/__main__.py @@ -17,76 +17,84 @@ def generate_markdown_table(combinations, no_arch): for arch, combination in combinations: platform = combination.split("-")[0] - table += f"| {platform:<10} | {arch:<18} | `{combination}` |\n" + table += ( + f"| {platform:<10} | {arch:<18} | `{combination}` |\n" + ) return table + def parse_args(): parser = ArgumentParser(description="Parse flavors.yaml and generate combinations.") - parser.add_argument("--no-arch", action="store_true", help="Exclude architecture from the flavor output.") + parser.add_argument( + "--no-arch", + action="store_true", + help="Exclude architecture from the flavor output.", + ) parser.add_argument( "--include-only", action="append", default=[], - help="Restrict combinations to those matching wildcard patterns (can be specified multiple times)." + help="Restrict combinations to those matching wildcard patterns (can be specified multiple times).", ) parser.add_argument( "--exclude", action="append", default=[], - help="Exclude combinations based on wildcard patterns (can be specified multiple times)." + help="Exclude combinations based on wildcard patterns (can be specified multiple times).", ) parser.add_argument( "--build", action="store_true", - help="Filter combinations to include only those with build enabled." + help="Filter combinations to include only those with build enabled.", ) parser.add_argument( "--publish", action="store_true", - help="Filter combinations to include only those with publish enabled." + help="Filter combinations to include only those with publish enabled.", ) parser.add_argument( "--test", action="store_true", - help="Filter combinations to include only those with test enabled." + help="Filter combinations to include only those with test enabled.", ) parser.add_argument( "--test-platform", action="store_true", - help="Filter combinations to include only platforms with test-platform: true." + help="Filter combinations to include only platforms with test-platform: true.", ) parser.add_argument( "--category", action="append", default=[], - help="Filter combinations to include only platforms belonging to the specified categories (can be specified multiple times)." + help="Filter combinations to include only platforms belonging to the specified categories (can be specified multiple times).", ) parser.add_argument( "--exclude-category", action="append", default=[], - help="Exclude platforms belonging to the specified categories (can be specified multiple times)." + help="Exclude platforms belonging to the specified categories (can be specified multiple times).", ) parser.add_argument( "--json-by-arch", action="store_true", - help="Output a JSON dictionary where keys are architectures and values are lists of flavors." + help="Output a JSON dictionary where keys are architectures and values are lists of flavors.", ) parser.add_argument( "--markdown-table-by-platform", action="store_true", - help="Generate a markdown table by platform." + help="Generate a markdown table by platform.", ) return parser.parse_args() + def main(): args = parse_args() repo_path = Git(".").rev_parse("--show-superproject-working-tree") - flavors_file = os.path.join(repo_path, 'flavors.yaml') + flavors_file = os.path.join(repo_path, "flavors.yaml") if not os.path.isfile(flavors_file): sys.exit(f"Error: {flavors_file} does not exist.") @@ -103,7 +111,7 @@ def main(): only_test_platform=args.test_platform, only_publish=args.publish, filter_categories=args.category, - exclude_categories=args.exclude_category + exclude_categories=args.exclude_category, ) if args.json_by_arch: diff --git a/src/gardenlinux/flavors/parser.py b/src/gardenlinux/flavors/parser.py index 3f0052c3..fb35b696 100644 --- a/src/gardenlinux/flavors/parser.py +++ b/src/gardenlinux/flavors/parser.py @@ -7,9 +7,10 @@ from ..constants import GL_FLAVORS_SCHEMA from ..logger import LoggerSetup + class Parser(object): - def __init__(self, data, logger = None): - flavors_data = (yaml.safe_load(data) if isinstance(data, str) else data) + def __init__(self, data, logger=None): + flavors_data = yaml.safe_load(data) if isinstance(data, str) else data jsonschema_validate(instance=flavors_data, schema=GL_FLAVORS_SCHEMA) if logger is None or not logger.hasHandlers(): @@ -18,7 +19,9 @@ def __init__(self, data, logger = None): self._flavors_data = flavors_data self._logger = logger - self._logger.debug("flavors.Parser initialized with data: {0!r}".format(flavors_data)) + self._logger.debug( + "flavors.Parser initialized with data: {0!r}".format(flavors_data) + ) def filter( self, @@ -29,16 +32,16 @@ def filter( only_test_platform=False, only_publish=False, filter_categories=[], - exclude_categories=[] + exclude_categories=[], ): """Parses the flavors.yaml file and generates combinations.""" self._logger.debug("flavors.Parser filtering with {0}".format(locals())) combinations = [] # Use a list for consistent order - for target in self._flavors_data['targets']: - name = target['name'] - category = target.get('category', '') + for target in self._flavors_data["targets"]: + name = target["name"] + category = target.get("category", "") # Apply category filters if filter_categories and category not in filter_categories: @@ -46,13 +49,13 @@ def filter( if exclude_categories and category in exclude_categories: continue - for flavor in target['flavors']: - features = flavor.get('features', []) - arch = flavor.get('arch', 'amd64') - build = flavor.get('build', False) - test = flavor.get('test', False) - test_platform = flavor.get('test-platform', False) - publish = flavor.get('publish', False) + for flavor in target["flavors"]: + features = flavor.get("features", []) + arch = flavor.get("arch", "amd64") + build = flavor.get("build", False) + test = flavor.get("test", False) + test_platform = flavor.get("test-platform", False) + publish = flavor.get("publish", False) # Apply flag-specific filters in the order: build, test, test-platform, publish if only_build and not build: @@ -83,7 +86,9 @@ def filter( combinations.append((arch, combination)) - return sorted(combinations, key=lambda platform: platform[1].split("-")[0]) # Sort by platform name + return sorted( + combinations, key=lambda platform: platform[1].split("-")[0] + ) # Sort by platform name @staticmethod def group_by_arch(combinations): @@ -98,7 +103,9 @@ def group_by_arch(combinations): @staticmethod def remove_arch(combinations): """Removes the architecture from combinations.""" - return [combination.replace(f"-{arch}", "") for arch, combination in combinations] + return [ + combination.replace(f"-{arch}", "") for arch, combination in combinations + ] @staticmethod def should_exclude(combination, excludes, wildcard_excludes): @@ -109,7 +116,9 @@ def should_exclude(combination, excludes, wildcard_excludes): if combination in excludes: return True # Exclude if matches any wildcard pattern - return any(fnmatch.fnmatch(combination, pattern) for pattern in wildcard_excludes) + return any( + fnmatch.fnmatch(combination, pattern) for pattern in wildcard_excludes + ) @staticmethod def should_include_only(combination, include_only_patterns): @@ -119,4 +128,6 @@ def should_include_only(combination, include_only_patterns): """ if not include_only_patterns: return True - return any(fnmatch.fnmatch(combination, pattern) for pattern in include_only_patterns) + return any( + fnmatch.fnmatch(combination, pattern) for pattern in include_only_patterns + ) diff --git a/src/python_gardenlinux_lib/__init__.py b/src/python_gardenlinux_lib/__init__.py index fc1d800f..cc733797 100644 --- a/src/python_gardenlinux_lib/__init__.py +++ b/src/python_gardenlinux_lib/__init__.py @@ -1,4 +1,3 @@ -from .git import Git from .version import Version -__all__ = ["Git", "Version"] +__all__ = ["Version"] diff --git a/src/python_gardenlinux_lib/features/parse_features.py b/src/python_gardenlinux_lib/features/parse_features.py index f1e95d57..eca20ff4 100644 --- a/src/python_gardenlinux_lib/features/parse_features.py +++ b/src/python_gardenlinux_lib/features/parse_features.py @@ -36,52 +36,6 @@ def get_gardenlinux_commit(gardenlinux_root: str, limit: Optional[int] = None) - else: return commit_str -def get_features_dict( - cname: str, gardenlinux_root: str, feature_dir_name: str = "features" -) -> dict: - """ - :param str cname: the target cname to get the feature dict for - :param str gardenlinux_root: path of garden linux src root - :return: dict with list of features for a given cname, split into platform, element and flag - - """ - - return Parser(gardenlinux_root, feature_dir_name).filter_as_dict(cname) - -def get_features_graph( - cname: str, gardenlinux_root: str, feature_dir_name: str = "features" -) -> networkx.Graph: - """ - :param str cname: the target cname to get the feature dict for - :param str gardenlinux_root: path of garden linux src root - :return: list of features for a given cname - - """ - - return Parser(gardenlinux_root, feature_dir_name).filter(cname) - -def get_features_list( - cname: str, gardenlinux_root: str, feature_dir_name: str = "features" -) -> list: - """ - :param str cname: the target cname to get the feature dict for - :param str gardenlinux_root: path of garden linux src root - :return: list of features for a given cname - - """ - - return Parser(gardenlinux_root, feature_dir_name).filter_as_list(cname) - -def get_features( - cname: str, gardenlinux_root: str, feature_dir_name: str = "features" -) -> str: - """ - :param str cname: the target cname to get the feature set for - :param str gardenlinux_root: path of garden linux src root - :return: a comma separated string with the expanded feature set for the cname - """ - - return Parser(gardenlinux_root, feature_dir_name).filter_as_string(cname) def construct_layer_metadata( filetype: str, cname: str, version: str, arch: str, commit: str @@ -101,6 +55,7 @@ def construct_layer_metadata( "annotations": {"io.gardenlinux.image.layer.architecture": arch}, } + def construct_layer_metadata_from_filename(filename: str, arch: str) -> dict: """ :param str filename: filename of the blob @@ -114,6 +69,7 @@ def construct_layer_metadata_from_filename(filename: str, arch: str) -> dict: "annotations": {"io.gardenlinux.image.layer.architecture": arch}, } + def get_file_set_from_cname(cname: str, version: str, arch: str, gardenlinux_root: str): """ :param str cname: the target cname of the image @@ -123,7 +79,7 @@ def get_file_set_from_cname(cname: str, version: str, arch: str, gardenlinux_roo :return: set of file names for a given cname """ file_set = set() - features_by_type = get_features_dict(cname, gardenlinux_root) + features_by_type = Parser(gardenlinux_root).filter_as_dict(cname) commit_str = get_gardenlinux_commit(gardenlinux_root, 8) if commit_str == "local": @@ -136,6 +92,7 @@ def get_file_set_from_cname(cname: str, version: str, arch: str, gardenlinux_roo ) return file_set + def get_oci_metadata_from_fileset(fileset: list, arch: str): """ :param str arch: arch of the target image @@ -151,6 +108,7 @@ def get_oci_metadata_from_fileset(fileset: list, arch: str): return oci_layer_metadata_list + def get_oci_metadata(cname: str, version: str, arch: str, gardenlinux_root: str): """ :param str cname: the target cname of the image @@ -173,6 +131,7 @@ def get_oci_metadata(cname: str, version: str, arch: str, gardenlinux_root: str) return oci_layer_metadata_list + def lookup_media_type_for_filetype(filetype: str) -> str: """ :param str filetype: filetype of the target layer @@ -185,6 +144,7 @@ def lookup_media_type_for_filetype(filetype: str) -> str: f"media type for {filetype} is not defined. You may want to add the definition to parse_features_lib" ) + def lookup_media_type_for_file(filename: str) -> str: """ :param str filename: filename of the target layer @@ -198,6 +158,7 @@ def lookup_media_type_for_file(filename: str) -> str: f"media type for {filename} is not defined. You may want to add the definition to parse_features_lib" ) + def deduce_feature_name(feature_dir: str): """ :param str feature_dir: Directory of single Feature @@ -208,6 +169,7 @@ def deduce_feature_name(feature_dir: str): raise ValueError("Expected name from parse_feature_yaml function to be set") return parsed["name"] + def deduce_archive_filetypes(feature_dir): """ :param str feature_dir: Directory of single Feature @@ -215,6 +177,7 @@ def deduce_archive_filetypes(feature_dir): """ return deduce_filetypes_from_string(feature_dir, "image") + def deduce_image_filetypes(feature_dir): """ :param str feature_dir: Directory of single Feature @@ -222,6 +185,7 @@ def deduce_image_filetypes(feature_dir): """ return deduce_filetypes_from_string(feature_dir, "convert") + def deduce_filetypes(feature_dir): """ :param str feature_dir: Directory of single Feature @@ -236,6 +200,7 @@ def deduce_filetypes(feature_dir): image_file_types.extend(archive_file_types) return image_file_types + def deduce_filetypes_from_string(feature_dir: str, script_base_name: str): """ Garden Linux features can optionally have an image. or convert. script, @@ -259,5 +224,6 @@ def deduce_filetypes_from_string(feature_dir: str, script_base_name: str): return sorted(result) + def sort_set(input_set, order_list): - return [item for item in order_list if item in input_set] \ No newline at end of file + return [item for item in order_list if item in input_set] diff --git a/src/python_gardenlinux_lib/flavors/parse_flavors.py b/src/python_gardenlinux_lib/flavors/parse_flavors.py deleted file mode 100755 index 941d3c73..00000000 --- a/src/python_gardenlinux_lib/flavors/parse_flavors.py +++ /dev/null @@ -1,230 +0,0 @@ -#!/usr/bin/env python - -# @TODO: This code is provided for backward compatibility only and deprecated. - -from gardenlinux.flavors.parser import Parser -from git import Git -import base64 -import json -import logging -import os -import sys -import subprocess -import yaml - -from gardenlinux.constants import GL_FLAVORS_SCHEMA - -# Define the schema for validation -SCHEMA = GL_FLAVORS_SCHEMA - - -def find_repo_root(): - """Finds the root directory of the Git repository.""" - - return Git(".").rev_parse("--show-superproject-working-tree") - -def validate_flavors(data): - """Validate the flavors.yaml data against the schema.""" - try: - validate(instance=data, schema=SCHEMA) - except ValidationError as e: - sys.exit(f"Validation Error: {e.message}") - - -def should_exclude(combination, excludes, wildcard_excludes): - """ - Checks if a combination should be excluded based on exact match or wildcard patterns. - """ - return Parser.should_exclude(combination, excludes, wildcard_excludes) - - -def should_include_only(combination, include_only_patterns): - """ - Checks if a combination should be included based on `--include-only` wildcard patterns. - If no patterns are provided, all combinations are included by default. - """ - return Parser.should_include_only(combination, include_only_patterns) - - -def parse_flavors_data( - data, - include_only_patterns=None, - wildcard_excludes=None, - only_build=False, - only_test=False, - only_test_platform=False, - only_publish=False, - filter_categories=None, - exclude_categories=None, -): - """Parse flavors.yaml data and generate combinations.""" - return Parser(data).filter( - include_only_patterns, - wildcard_excludes, - only_build, - only_test, - only_test_platform, - only_publish, - filter_categories, - exclude_categories - ) - -def _get_flavors_from_github(commit): - """Returns the flavors.yaml from GitHub if readable.""" - - # Try flavors.yaml first - api_path = "/repos/gardenlinux/gardenlinux/contents/flavors.yaml" - if commit != "latest": - api_path = f"{api_path}?ref={commit}" - command = ["gh", "api", api_path] - logger.debug(f"Fetching flavors.yaml from GitHub for commit {commit_short}") - result = subprocess.run( - command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True - ) - - if result.returncode == 0: - content_data = json.loads(result.stdout) - return base64.b64decode(content_data["content"]).decode("utf-8") - else: - raise RuntimeError("Failed receiving result from GitHub: {0}".format(result.stderr)) - - -def parse_flavors_commit( - commit=None, - version=None, - query_s3=False, - s3_objects=None, - logger=None, - include_only_patterns=None, - wildcard_excludes=None, - only_build=False, - only_test=False, - only_test_platform=False, - only_publish=False, - filter_categories=None, - exclude_categories=None, -): - """ - Parse flavors for a specific commit, optionally checking S3 artifacts. - - Args: - commit (str): The git commit hash to check - version (dict, optional): Version info with 'major' and optional 'minor' keys - query_s3 (bool): Whether to check S3 artifacts if no flavors.yaml found - s3_objects (dict, optional): Pre-fetched S3 artifacts data - logger (logging.Logger): Logger instance to use - include_only_patterns (list): Restrict combinations to those matching wildcard patterns - wildcard_excludes (list): Exclude combinations based on wildcard patterns - only_build (bool): Filter combinations to include only those with build enabled - only_test (bool): Filter combinations to include only those with test enabled - only_test_platform (bool): Filter combinations to include only platforms with test-platform: true - only_publish (bool): Filter combinations to include only those with publish enabled - filter_categories (list): Filter combinations to include only platforms belonging to specified categories - exclude_categories (list): Exclude platforms belonging to specified categories - - Returns: - list: List of flavor strings, or empty list if no flavors found - """ - - if logger is None: - logger = logging.getLogger("gardenlinux.flavors") - logger.addHandler(logging.NullHandler()) - - version_info = ( - f"{version['major']}.{version.get('minor', 0)}" if version else "unknown" - ) - if commit is None: - commit = "latest" - commit_short = commit[:8] - - logger.debug( - f"Checking flavors for version {version_info} (commit {commit_short})" - ) - - flavors_content = None - - if os.access("./flavors.yaml", os.F_OK | os.R_OK): - with open("./flavors.yaml", "r") as fp: - flavors_content = fp.read() - else: - try: - flavors_content = _get_flavors_from_github() - except Exception as exc: - logger.debug(exc) - - if flavors_content is not None: - # Parse flavors with all filters - combinations = parse_flavors_data( - yaml.safe_load(flavors_content), - include_only_patterns=include_only_patterns or [], - wildcard_excludes=wildcard_excludes or [], - only_build=only_build, - only_test=only_test, - only_test_platform=only_test_platform, - only_publish=only_publish, - filter_categories=filter_categories or [], - exclude_categories=exclude_categories or [], - ) - - all_flavors = set() - for _, combination in combinations: - all_flavors.add(combination) - - if all_flavors: - logger.info(f"Found {len(all_flavors)} flavors in flavors.yaml") - return sorted(all_flavors) - else: - logger.info("No flavors found in flavors.yaml") - elif query_s3 and s3_objects and isinstance(s3_objects, dict): - logger.debug("Checking S3 artifacts") - index = s3_objects.get("index", {}) - artifacts = s3_objects.get("artifacts", []) - - # Try index lookup first - search_key = f"{version_info}-{commit_short}" - if search_key in index: - flavors = index[search_key] - logger.debug(f"Found flavors in S3 index for {search_key}") - else: - # If no index match, search through artifacts - found_flavors = set() - - # Search for artifacts matching version and commit - for key in artifacts: - if version_info in key and commit_short in key: - try: - parts = key.split("/") - if len(parts) >= 2: - flavor_with_version = parts[1] - flavor = flavor_with_version.rsplit( - "-" + version_info, 1 - )[0] - if flavor: - found_flavors.add(flavor) - except Exception as e: - logger.debug(f"Error parsing artifact key {key}: {e}") - continue - - flavors = list(found_flavors) - - # Apply filters to S3 flavors - filtered_flavors = [] - for flavor in flavors: - # Create a dummy combination with amd64 architecture for filtering - combination = ("amd64", flavor) - if should_include_only( - flavor, include_only_patterns or [] - ) and not should_exclude(flavor, wildcard_excludes or [], []): - filtered_flavors.append(flavor) - - if filtered_flavors: - logger.info( - f"Found {len(filtered_flavors)} flavors in S3 artifacts after filtering" - ) - return sorted(filtered_flavors) - else: - logger.info( - f"No flavors found in S3 for version {version_info} and commit {commit_short} after filtering" - ) - - return [] diff --git a/src/python_gardenlinux_lib/version.py b/src/python_gardenlinux_lib/version.py index a789ebb9..949cd2c6 100644 --- a/src/python_gardenlinux_lib/version.py +++ b/src/python_gardenlinux_lib/version.py @@ -4,8 +4,7 @@ import requests from pathlib import Path -from .logger import LoggerSetup -from .features.parse_features import get_features +from gardenlinux.logger import LoggerSetup class Version: diff --git a/tests/test_get_features_dict.py b/tests/test_get_features_dict.py index f6b9c30b..629752b1 100644 --- a/tests/test_get_features_dict.py +++ b/tests/test_get_features_dict.py @@ -1,6 +1,6 @@ import pytest -from python_gardenlinux_lib.features.parse_features import get_features_dict +from gardenlinux.features import Parser from tests.conftest import GL_ROOT_DIR @@ -66,5 +66,5 @@ def test_get_features_dict(input_cname: str, expected_output: dict): features have changed since writing this test. In this case, update the expected output accordingly. You can print the output of get_features_dict so you have the dict in the expected format. """ - features_dict = get_features_dict(input_cname, GL_ROOT_DIR) + features_dict = Parser(GL_ROOT_DIR).filter_as_dict(input_cname) assert features_dict == expected_output diff --git a/tests/test_parse_debsource.py b/tests/test_parse_debsource.py index 0f0e62d1..fbe556fc 100644 --- a/tests/test_parse_debsource.py +++ b/tests/test_parse_debsource.py @@ -1,4 +1,4 @@ -from python_gardenlinux_lib.apt.parse_debsource import DebsrcFile +from gardenlinux.apt import DebsrcFile import io test_data = """Package: vim diff --git a/tests/test_push_image.py b/tests/test_push_image.py index 9fa406bd..4b3bd81c 100644 --- a/tests/test_push_image.py +++ b/tests/test_push_image.py @@ -1,8 +1,9 @@ import pytest import os +from gardenlinux.features import Parser +from python_gardenlinux_lib.features.parse_features import get_oci_metadata from python_gardenlinux_lib.oras.registry import GlociRegistry -from python_gardenlinux_lib.features import parse_features CONTAINER_NAME_ZOT_EXAMPLE = "127.0.0.1:18081/gardenlinux-example" GARDENLINUX_ROOT_DIR_EXAMPLE = "test-data/gardenlinux/" @@ -27,12 +28,11 @@ ], ) def test_push_example(version, cname, arch): - oci_metadata = parse_features.get_oci_metadata( - cname, version, arch, GARDENLINUX_ROOT_DIR_EXAMPLE - ) + oci_metadata = get_oci_metadata(cname, version, arch, GARDENLINUX_ROOT_DIR_EXAMPLE) container_name = f"{CONTAINER_NAME_ZOT_EXAMPLE}:{version}" a_registry = GlociRegistry(container_name=container_name, insecure=True) - features = parse_features.get_features(cname, GARDENLINUX_ROOT_DIR_EXAMPLE) + features = Parser(GARDENLINUX_ROOT_DIR_EXAMPLE).filter_as_string(cname) + a_registry.push_image_manifest( arch, cname,