list and describe metadata from oci

Author: adwk67

Cargo.lock

@@ -38,7 +38,7 @@ phf = "0.11"
 phf_codegen = "0.11"
 rand = "0.8"
 regex = "1.10"
-reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
+reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
 rstest = "0.22"
 semver = { version = "1.0", features = ["serde"] }
 serde = { version = "1.0", features = ["derive"] }
@@ -54,6 +54,7 @@ tower-http = { version = "0.5", features = ["validate-request"] }
 tracing = "0.1"
 tracing-subscriber = "0.3"
 url = "2.5"
+urlencoding = "2.1.3"
 utoipa = { version = "4.2", features = ["indexmap"] }
 utoipa-swagger-ui = { version = "7.1", features = ["axum"] }
 uuid = { version = "1.10", features = ["v4"] }

Cargo.nix

@@ -32,6 +32,7 @@ tera.workspace = true
 tokio.workspace = true
 tracing.workspace = true
 url.workspace = true
+urlencoding.workspace = true
 utoipa = { workspace = true, optional = true }
 which.workspace = true
 futures.workspace = true

Cargo.toml

@@ -22,6 +22,7 @@ pub const HELM_REPO_NAME_TEST: &str = "stackable-test";
 pub const HELM_REPO_NAME_DEV: &str = "stackable-dev";
 pub const HELM_REPO_INDEX_FILE: &str = "index.yaml";
 
+pub const HELM_OCI_BASE: &str = "oci.stackable.tech";
 pub const HELM_OCI_REGISTRY: &str = "oci://oci.stackable.tech/sdp-charts";
 
 pub const HELM_DEFAULT_CHART_VERSION: &str = ">0.0.0-0";

rust/stackable-cockpit/Cargo.toml

@@ -1,4 +1,3 @@
-use std::collections::HashMap;
 use std::fmt::Display;
 
 use serde::{Deserialize, Serialize};
@@ -7,7 +6,10 @@ use tokio::task::block_in_place;
 use tracing::{debug, error, info, instrument};
 use url::Url;
 
-use crate::constants::{HELM_DEFAULT_CHART_VERSION, HELM_REPO_INDEX_FILE};
+use crate::{
+    constants::{HELM_DEFAULT_CHART_VERSION, HELM_REPO_INDEX_FILE},
+    utils::chartsource::ChartSourceMetadata,
+};
 
 #[derive(Debug, Deserialize, Serialize)]
 #[serde(rename_all = "camelCase")]
@@ -35,17 +37,6 @@ pub struct ChartRepo {
     pub url: String,
 }
 
-#[derive(Clone, Debug, Deserialize)]
-pub struct Repository {
-    pub entries: HashMap<String, Vec<RepositoryEntry>>,
-}
-
-#[derive(Clone, Debug, Deserialize)]
-pub struct RepositoryEntry {
-    pub name: String,
-    pub version: String,
-}
-
 #[derive(Debug, Snafu)]
 pub enum Error {
     #[snafu(display("failed to parse URL"))]
@@ -398,7 +389,7 @@ pub fn add_repo(repository_name: &str, repository_url: &str) -> Result<(), Error
 
 /// Retrieves the Helm index file from the repository URL.
 #[instrument]
-pub async fn get_helm_index<T>(repo_url: T) -> Result<Repository, Error>
+pub async fn get_helm_index<T>(repo_url: T) -> Result<ChartSourceMetadata, Error>
 where
     T: AsRef<str> + std::fmt::Debug,
 {

rust/stackable-cockpit/src/constants.rs

@@ -2,6 +2,7 @@ pub mod common;
 pub mod constants;
 pub mod engine;
 pub mod helm;
+pub mod oci;
 pub mod platform;
 pub mod utils;
 pub mod xfer;

rust/stackable-cockpit/src/helm.rs

@@ -0,0 +1,173 @@
+use std::collections::HashMap;
+
+use serde::Deserialize;
+use snafu::{OptionExt, ResultExt, Snafu};
+use tracing::debug;
+use urlencoding::encode;
+
+use crate::{
+    constants::{HELM_OCI_BASE, HELM_REPO_NAME_DEV, HELM_REPO_NAME_STABLE, HELM_REPO_NAME_TEST},
+    utils::chartsource::{ChartSourceEntry, ChartSourceMetadata},
+};
+
+#[derive(Debug, Snafu)]
+pub enum Error {
+    #[snafu(display("cannot get repositories"))]
+    GetRepositories { source: reqwest::Error },
+
+    #[snafu(display("cannot parse repositories"))]
+    ParseRepositories { source: reqwest::Error },
+
+    #[snafu(display("cannot get artifacts"))]
+    GetArtifacts { source: reqwest::Error },
+
+    #[snafu(display("cannot parse artifacts"))]
+    ParseArtifacts { source: reqwest::Error },
+
+    #[snafu(display("unexpected OCI repository name"))]
+    UnexpectedOciRepositoryName,
+}
+
+#[derive(Deserialize, Debug)]
+pub struct OciRepository {
+    pub name: String,
+}
+
+#[derive(Deserialize, Debug)]
+pub struct Tag {
+    pub name: String,
+}
+
+#[derive(Deserialize, Debug)]
+pub struct Artifact {
+    pub digest: String,
+    pub tags: Option<Vec<Tag>>,
+}
+
+pub async fn get_oci_index<'a>() -> Result<HashMap<&'a str, ChartSourceMetadata>, Error> {
+    let mut source_index_files: HashMap<&str, ChartSourceMetadata> = HashMap::new();
+
+    // initialize map
+    for repo_name in [
+        HELM_REPO_NAME_STABLE,
+        HELM_REPO_NAME_TEST,
+        HELM_REPO_NAME_DEV,
+    ] {
+        source_index_files.insert(
+            repo_name,
+            ChartSourceMetadata {
+                entries: HashMap::new(),
+            },
+        );
+    }
+    let base_url = format!("https://{}/api/v2.0", HELM_OCI_BASE);
+
+    // fetch all operators
+    let url = format!(
+        "{}/repositories?page_size={}&q=name=~sdp-charts/",
+        base_url, 100
+    );
+
+    // reuse connections
+    let client = reqwest::Client::new();
+
+    let repositories: Vec<OciRepository> = client
+        .get(&url)
+        .send()
+        .await
+        .context(GetRepositoriesSnafu)?
+        .json()
+        .await
+        .context(ParseRepositoriesSnafu)?;
+
+    debug!("OCI repos {:?}", repositories);
+
+    for repository in &repositories {
+        // fetch all artifacts pro operator
+        let (project_name, repository_name) = repository
+            .name
+            .split_once('/')
+            .context(UnexpectedOciRepositoryNameSnafu)?;
+
+        debug!("OCI repo parts {} and {}", project_name, repository_name);
+
+        let mut artifacts = Vec::new();
+        let mut page = 1;
+        let page_size = 20;
+
+        while let Ok(artifacts_page) = client
+            .get(format!(
+                "{}/projects/{}/repositories/{}/artifacts?page_size={}&page={}",
+                base_url,
+                encode(project_name),
+                encode(repository_name),
+                page_size,
+                page
+            ))
+            .send()
+            .await
+            .context(GetArtifactsSnafu)?
+            .json::<Vec<Artifact>>()
+            .await
+            .context(ParseArtifactsSnafu)
+        {
+            let count = artifacts_page.len();
+            artifacts.extend(artifacts_page);
+            if count < page_size {
+                break;
+            }
+            page += 1;
+        }
+
+        for artifact in &artifacts {
+            if let Some(release_artifact) =
+                artifact.tags.as_ref().and_then(|tags| tags.iter().next())
+            {
+                let release_version = release_artifact
+                    .name
+                    .replace("-arm64", "")
+                    .replace("-amd64", "");
+
+                debug!(
+                    "OCI resolved artifact {}, {}, {}",
+                    release_version.to_string(),
+                    repository_name.to_string(),
+                    release_artifact.name.to_string()
+                );
+
+                let entry = ChartSourceEntry {
+                    name: repository_name.to_string(),
+                    version: release_version.to_string(),
+                };
+
+                match release_version.as_str() {
+                    "0.0.0-dev" => {
+                        if let Some(repo) = source_index_files.get_mut(HELM_REPO_NAME_DEV) {
+                            repo.entries
+                                .entry(repository_name.to_string())
+                                .or_default()
+                                .push(entry)
+                        }
+                    }
+                    version if version.contains("-pr") => {
+                        if let Some(repo) = source_index_files.get_mut(HELM_REPO_NAME_TEST) {
+                            repo.entries
+                                .entry(repository_name.to_string())
+                                .or_default()
+                                .push(entry)
+                        }
+                    }
+                    _ => {
+                        if let Some(repo) = source_index_files.get_mut(HELM_REPO_NAME_STABLE) {
+                            repo.entries
+                                .entry(repository_name.to_string())
+                                .or_default()
+                                .push(entry)
+                        }
+                    }
+                }
+            }
+        }
+    }
+    Ok(source_index_files)
+}

rust/stackable-cockpit/src/lib.rs

@@ -0,0 +1,14 @@
+use std::collections::HashMap;
+
+use serde::Deserialize;
+
+#[derive(Clone, Debug, Deserialize)]
+pub struct ChartSourceMetadata {
+    pub entries: HashMap<String, Vec<ChartSourceEntry>>,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+pub struct ChartSourceEntry {
+    pub name: String,
+    pub version: String,
+}

rust/stackable-cockpit/src/oci.rs

@@ -1,3 +1,4 @@
+pub mod chartsource;
 pub mod check;
 pub mod k8s;
 pub mod params;