diff --git a/src/python_gardenlinux_lib/features/parse_features.py b/src/python_gardenlinux_lib/features/parse_features.py index 4ffcd840..9d3e67b7 100644 --- a/src/python_gardenlinux_lib/features/parse_features.py +++ b/src/python_gardenlinux_lib/features/parse_features.py @@ -7,6 +7,36 @@ import subprocess from typing import Optional +# It is important that this list is sorted in descending length of the entries +GL_MEDIA_TYPES = [ + "gcpimage.tar.gz.log", + "firecracker.tar.gz", + "gcpimage.tar.gz", + "pxe.tar.gz.log", + "manifest.log", + "release.log", + "pxe.tar.gz", + "qcow2.log", + "test-log", + "manifest", + "vmdk.log", + "tar.log", + "release", + "vhd.log", + "ova.log", + "raw.log", + "tar.gz", + "qcow2", + "tar", + "iso", + "oci", + "vhd", + "vmdk", + "ova", + "raw", +] + + GL_MEDIA_TYPE_LOOKUP = { "tar": "application/io.gardenlinux.image.archive.format.tar", "tar.gz": "application/io.gardenlinux.image.archive.format.tar.gz", @@ -20,6 +50,19 @@ "vmdk": "application/io.gardenlinux.image.format.vmdk", "ova": "application/io.gardenlinux.image.format.ova", "raw": "application/io.gardenlinux.image.archive.format.raw", + "manifest.log": "application/io.gardenlinux.log", + "release.log": "application/io.gardenlinux.log", + "test-log": "application/io.gardenlinux.test-log", + "manifest": "application/io.gardenlinux.manifest", + "tar.log": "application/io.gardenlinux.log", + "release": "application/io.gardenlinux.release", + "raw.log": "application/io.gardenlinux.log", + "qcow2.log": "application/io.gardenlinux.log", + "pxe.tar.gz.log": "application/io.gardenlinux.log", + "gcpimage.tar.gz.log": "application/io.gardenlinux.log", + "vmdk.log": "application/io.gardenlinux.log", + "vhd.log": "application/io.gardenlinux.log", + "ova.log": "application/io.gardenlinux.log", } @@ -88,53 +131,82 @@ def construct_layer_metadata( return { "file_name": f"{cname}-{arch}-{version}-{commit}.{filetype}", "media_type": media_type, + "annotations": {"io.gardenlinux.image.layer.architecture": arch}, + } + + +def construct_layer_metadata_from_filename(filename: str, arch: str) -> dict: + """ + :param str filename: filename of the blob + :param str arch: the arch of the target image + :return: dict of oci layer metadata for a given layer file + """ + media_type = lookup_media_type_for_file(filename) + return { + "file_name": filename, + "media_type": media_type, + "annotations": {"io.gardenlinux.image.layer.architecture": arch}, } -def get_oci_metadata(cname: str, version: str, gardenlinux_root: str): +def get_file_set_from_cname(cname: str, version: str, arch: str, gardenlinux_root: str): """ :param str cname: the target cname of the image - :param str version: the target version of the image + :param str version: the version of the target image + :param str arch: the arch of the target image :param str gardenlinux_root: path of garden linux src root - :return: list of dicts, where each dict represents a layer + :return: set of file names for a given cname """ - oci_layer_metadata_list = list() + file_set = set() features_by_type = get_features_dict(cname, gardenlinux_root) commit_str = get_gardenlinux_commit(gardenlinux_root, 8) if commit_str == "local": raise ValueError("Using local commit. Refusing to upload to OCI Registry") - - for arch in ["amd64", "arm64"]: - for platform in features_by_type["platform"]: - image_file_types = deduce_image_filetype( - f"{gardenlinux_root}/features/{platform}" - ) - archive_file_types = deduce_archive_filetype( - f"{gardenlinux_root}/features/{platform}" + for platform in features_by_type["platform"]: + image_file_types = deduce_filetypes(f"{gardenlinux_root}/features/{platform}") + for ft in image_file_types: + file_set.add( + f"{cname}-{arch}-{version}-{commit_str}.{ft}", ) - # Allow multiple image scripts per feature - if not image_file_types: - image_file_types.append("raw") - if not archive_file_types: - image_file_types.append("tar") - for ft in archive_file_types: - cur_layer_metadata = construct_layer_metadata( - ft, cname, version, arch, commit_str - ) - cur_layer_metadata["annotations"] = { - "io.gardenlinux.image.layer.architecture": arch - } - oci_layer_metadata_list.append(cur_layer_metadata) - # Allow multiple convert scripts per feature - for ft in image_file_types: - cur_layer_metadata = construct_layer_metadata( - ft, cname, version, arch, commit_str - ) - cur_layer_metadata["annotations"] = { - "io.gardenlinux.image.layer.architecture": arch - } - oci_layer_metadata_list.append(cur_layer_metadata) + return file_set + + +def get_oci_metadata_from_fileset(fileset: set, arch: str): + """ + :param str arch: arch of the target image + :param set fileset: a list of filenames (not paths) to set oci_metadata for + :return: list of dicts, where each dict represents a layer + """ + oci_layer_metadata_list = list() + + for file in fileset: + oci_layer_metadata_list.append( + construct_layer_metadata_from_filename(file, arch) + ) + + return oci_layer_metadata_list + + +def get_oci_metadata(cname: str, version: str, arch: str, gardenlinux_root: str): + """ + :param str cname: the target cname of the image + :param str version: the target version of the image + :param str arch: arch of the target image + :param str gardenlinux_root: path of garden linux src root + :return: list of dicts, where each dict represents a layer + """ + + # This is the feature deduction approach (glcli oci push) + file_set = get_file_set_from_cname(cname, version, arch, gardenlinux_root) + + # This is the tarball extraction approach (glcli oci push-tarball) + oci_layer_metadata_list = list() + + for file in file_set: + oci_layer_metadata_list.append( + construct_layer_metadata_from_filename(file, arch) + ) return oci_layer_metadata_list @@ -148,7 +220,21 @@ def lookup_media_type_for_filetype(filetype: str) -> str: return GL_MEDIA_TYPE_LOOKUP[filetype] else: raise ValueError( - f"No media type for {filetype} is defined. You may want to add the definition to parse_features_lib" + f"media type for {filetype} is not defined. You may want to add the definition to parse_features_lib" + ) + + +def lookup_media_type_for_file(filename: str) -> str: + """ + :param str filename: filename of the target layer + :return: mediatype + """ + for suffix in GL_MEDIA_TYPES: + if filename.endswith(suffix): + return GL_MEDIA_TYPE_LOOKUP[suffix] + else: + raise ValueError( + f"media type for {filename} is not defined. You may want to add the definition to parse_features_lib" ) @@ -163,25 +249,40 @@ def deduce_feature_name(feature_dir: str): return parsed["name"] -def deduce_archive_filetype(feature_dir): +def deduce_archive_filetypes(feature_dir): + """ + :param str feature_dir: Directory of single Feature + :return: str list of filetype for archive + """ + return deduce_filetypes_from_string(feature_dir, "image") + + +def deduce_image_filetypes(feature_dir): """ :param str feature_dir: Directory of single Feature - :return: str of filetype for archive + :return: str list of filetype for image """ - return deduce_filetype_from_string(feature_dir, "image") + return deduce_filetypes_from_string(feature_dir, "convert") -def deduce_image_filetype(feature_dir): +def deduce_filetypes(feature_dir): """ :param str feature_dir: Directory of single Feature - :return: str of filetype for image + :return: str list of filetypes for the feature """ - return deduce_filetype_from_string(feature_dir, "convert") + image_file_types = deduce_image_filetypes(feature_dir) + archive_file_types = deduce_archive_filetypes(feature_dir) + if not image_file_types: + image_file_types.append("raw") + if not archive_file_types: + archive_file_types.append("tar") + image_file_types.extend(archive_file_types) + return image_file_types -def deduce_filetype_from_string(feature_dir: str, script_base_name: str): +def deduce_filetypes_from_string(feature_dir: str, script_base_name: str): """ - Garden Linux features can optionally have a image. or convert. script, + Garden Linux features can optionally have an image. or convert. script, where the indicates the target filetype. image. script converts the .tar to archive. @@ -229,7 +330,7 @@ def read_feature_files(feature_dir): ) feature_graph.add_edge(node, ref, attr=attr) if not networkx.is_directed_acyclic_graph(feature_graph): - raise ValueError("Graph is not directed asyclic graph") + raise ValueError("Graph is not directed acyclic graph") return feature_graph diff --git a/src/python_gardenlinux_lib/oras/registry.py b/src/python_gardenlinux_lib/oras/registry.py index 724d3b81..c4bc176c 100644 --- a/src/python_gardenlinux_lib/oras/registry.py +++ b/src/python_gardenlinux_lib/oras/registry.py @@ -74,13 +74,13 @@ def NewPlatform(architecture: str, version: str) -> dict: def NewManifestMetadata( - digest: str, size: int, annotaions: dict, platform_data: dict + digest: str, size: int, annotations: dict, platform_data: dict ) -> dict: manifest_meta_data = copy.deepcopy(EmptyManifestMetadata) manifest_meta_data["mediaType"] = "application/vnd.oci.image.manifest.v1+json" manifest_meta_data["digest"] = digest manifest_meta_data["size"] = size - manifest_meta_data["annotations"] = annotaions + manifest_meta_data["annotations"] = annotations manifest_meta_data["platform"] = platform_data manifest_meta_data["artifactType"] = "" return manifest_meta_data @@ -114,7 +114,10 @@ def create_config_from_dict(conf: dict, annotations: dict) -> Tuple[dict, str]: def construct_manifest_entry_signed_data_string( cname: str, version: str, new_manifest_metadata: dict, architecture: str ) -> str: - data_to_sign = f"versio:{version} cname{cname} architecture:{architecture} manifest-size:{new_manifest_metadata['size']} manifest-digest:{new_manifest_metadata['digest']}" + data_to_sign = ( + f"version:{version} cname:{cname} architecture:{architecture} manifest-size" + f":{new_manifest_metadata['size']} manifest-digest:{new_manifest_metadata['digest']}" + ) return data_to_sign @@ -559,12 +562,13 @@ def push_image_manifest( architecture: str, cname: str, version: str, - gardenlinux_root: str, build_artifacts_dir: str, + oci_metadata: list, ): """ creates and pushes an image manifest + :param oci_metadata: a list of filenames and their OCI metadata, can be constructed with get_oci_metadata :param str architecture: target architecture of the image :param str cname: canonical name of the target image :param str build_artifacts_dir: directory where the build artifacts are located @@ -572,7 +576,7 @@ def push_image_manifest( # TODO: construct oci_artifacts default data - oci_metadata = get_oci_metadata(cname, version, gardenlinux_root) + # oci_metadata = get_oci_metadata(cname, version, architecture, gardenlinux_root) manifest_image = oras.oci.NewManifest() total_size = 0 @@ -690,7 +694,7 @@ def create_layer( checksum_sha256 = calculate_sha256(file_path) layer = oras.oci.NewLayer(file_path, media_type, is_dir=False) layer["annotations"] = { - oras.defaults.annotation_title: file_path, + oras.defaults.annotation_title: os.path.basename(file_path), "application/vnd.gardenlinux.image.checksum.sha256": checksum_sha256, } self.sign_layer( diff --git a/tests/test_deduce_image_type.py b/tests/test_deduce_image_type.py index 2cd9cac8..d05f9be3 100644 --- a/tests/test_deduce_image_type.py +++ b/tests/test_deduce_image_type.py @@ -1,6 +1,6 @@ from python_gardenlinux_lib.features.parse_features import ( - deduce_archive_filetype, - deduce_image_filetype, + deduce_archive_filetypes, + deduce_image_filetypes, ) import pytest @@ -20,7 +20,7 @@ ], ) def test_deduce_image_type(feature_name, expected_file_type): - file_type = deduce_image_filetype(f"{GL_ROOT_DIR}/features/{feature_name}") + file_type = deduce_image_filetypes(f"{GL_ROOT_DIR}/features/{feature_name}") assert sorted(expected_file_type) == file_type @@ -35,5 +35,5 @@ def test_deduce_image_type(feature_name, expected_file_type): ], ) def test_deduce_archive_type(feature_name, expected_file_type): - file_type = deduce_archive_filetype(f"{GL_ROOT_DIR}/features/{feature_name}") + file_type = deduce_archive_filetypes(f"{GL_ROOT_DIR}/features/{feature_name}") assert sorted(expected_file_type) == file_type diff --git a/tests/test_get_oci_metadata.py b/tests/test_get_oci_metadata.py index 3d0de6af..10b544f2 100644 --- a/tests/test_get_oci_metadata.py +++ b/tests/test_get_oci_metadata.py @@ -5,15 +5,43 @@ @pytest.mark.parametrize( - "input_cname, version", + "input_cname, version, arch", [ - ("aws-gardener_prod", "today"), - ("openstack-gardener_prod", "today"), + # ("aws-gardener_prod", "today"), + # ("openstack-gardener_prod", "today"), + ("openstack-gardener_pxe", "1443.9", "amd64"), ], ) -def test_get_features_dict(input_cname: str, version: str): +def test_get_oci_metadata(input_cname: str, version: str, arch: str): """ Work in Progess: currently only used to see what get_oci_metadata returns """ - metadata = get_oci_metadata(input_cname, version, GL_ROOT_DIR) - print(metadata) + metadata = get_oci_metadata(input_cname, version, arch, GL_ROOT_DIR) + expected = [ + { + "file_name": "openstack-gardener_pxe-amd64-1443.9-c81fcc9f.qcow2", + "media_type": "application/io.gardenlinux.image.format.qcow2", + "annotations": {"io.gardenlinux.image.layer.architecture": "amd64"}, + }, + { + "file_name": "openstack-gardener_pxe-amd64-1443.9-c81fcc9f.vmdk", + "media_type": "application/io.gardenlinux.image.format.vmdk", + "annotations": {"io.gardenlinux.image.layer.architecture": "amd64"}, + }, + { + "file_name": "openstack-gardener_pxe-amd64-1443.9-c81fcc9f.tar", + "media_type": "application/io.gardenlinux.image.archive.format.tar", + "annotations": {"io.gardenlinux.image.layer.architecture": "amd64"}, + }, + ] + for elem in metadata: + print( + elem["file_name"], + "\tmedia-type:", + elem["media_type"], + "\t annotations", + elem["annotations"], + "\tkeys:", + elem.keys(), + ) + # assert metadata == expected diff --git a/tests/test_push_image.py b/tests/test_push_image.py index 1af9dab0..465581fa 100644 --- a/tests/test_push_image.py +++ b/tests/test_push_image.py @@ -1,6 +1,7 @@ import pytest import os from python_gardenlinux_lib.oras.registry import GlociRegistry, setup_registry +from python_gardenlinux_lib.features import parse_features CONTAINER_NAME_ZOT_EXAMPLE = "127.0.0.1:18081/gardenlinux-example" GARDENLINUX_ROOT_DIR_EXAMPLE = "test-data/gardenlinux/" @@ -25,7 +26,9 @@ ], ) def test_push_example(version, cname, arch): - + oci_metadata = parse_features.get_oci_metadata( + cname, version, arch, GARDENLINUX_ROOT_DIR_EXAMPLE + ) container_name = f"{CONTAINER_NAME_ZOT_EXAMPLE}:{version}" registry = setup_registry( container_name, @@ -34,9 +37,5 @@ def test_push_example(version, cname, arch): public_key="cert/oci-sign.crt", ) registry.push_image_manifest( - arch, - cname, - version, - GARDENLINUX_ROOT_DIR_EXAMPLE, - f"{GARDENLINUX_ROOT_DIR_EXAMPLE}/.build", + arch, cname, version, f"{GARDENLINUX_ROOT_DIR_EXAMPLE}/.build", oci_metadata )