From a04e1a021a48acd675d866f8bae7b5238f4d2df1 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 4 Apr 2025 10:50:54 +0200 Subject: [PATCH 01/15] watch referenced configmaps and inject vector aggregator discovery cm as env --- Cargo.toml | 4 +- rust/operator-binary/src/hbase_controller.rs | 56 +++++----- rust/operator-binary/src/main.rs | 110 ++++++++++++------- rust/operator-binary/src/product_logging.rs | 48 +------- 4 files changed, 105 insertions(+), 113 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d0e6d9cd..07d677ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,8 +10,8 @@ edition = "2021" repository = "https://github.com/stackabletech/hbase-operator" [workspace.dependencies] -stackable-versioned = { git = "https://github.com/stackabletech/operator-rs.git", features = ["k8s"], tag = "stackable-versioned-0.6.0" } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.87.0" } +stackable-versioned = { git = "https://github.com/stackabletech/operator-rs.git", features = ["k8s"], tag = "stackable-versioned-0.7.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.87.5" } product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" } anyhow = "1.0" diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 6bfdfab6..25dca880 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -80,8 +80,7 @@ use crate::{ }, operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, product_logging::{ - extend_role_group_config_map, resolve_vector_aggregator_address, - CONTAINERDEBUG_LOG_DIRECTORY, STACKABLE_LOG_DIR, + extend_role_group_config_map, CONTAINERDEBUG_LOG_DIRECTORY, STACKABLE_LOG_DIR, }, security::{self, opa::HbaseOpaConfig}, zookeeper::{self, ZookeeperConnectionInformation}, @@ -235,10 +234,8 @@ pub enum Error { #[snafu(display("failed to resolve and merge config for role and role group"))] FailedToResolveConfig { source: crate::crd::Error }, - #[snafu(display("failed to resolve the Vector aggregator address"))] - ResolveVectorAggregatorAddress { - source: crate::product_logging::Error, - }, + #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] + VectorAggregatorConfigMapMissing, #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] InvalidLoggingConfig { @@ -350,10 +347,6 @@ pub async fn reconcile_hbase( .await .context(RetrieveZookeeperConnectionInformationSnafu)?; - let vector_aggregator_address = resolve_vector_aggregator_address(hbase, client) - .await - .context(ResolveVectorAggregatorAddressSnafu)?; - let roles = hbase.build_role_properties().context(RolePropertiesSnafu)?; let validated_config = validate_all_roles_and_groups_config( @@ -448,7 +441,6 @@ pub async fn reconcile_hbase( &merged_config, &resolved_product_image, hbase_opa_config.as_ref(), - vector_aggregator_address.as_deref(), )?; let rg_statefulset = build_rolegroup_statefulset( hbase, @@ -576,7 +568,6 @@ fn build_rolegroup_config_map( merged_config: &AnyServiceConfig, resolved_product_image: &ResolvedProductImage, hbase_opa_config: Option<&HbaseOpaConfig>, - vector_aggregator_address: Option<&str>, ) -> Result { let mut hbase_site_xml = String::new(); let mut hbase_env_sh = String::new(); @@ -703,7 +694,6 @@ fn build_rolegroup_config_map( extend_role_group_config_map( rolegroup, - vector_aggregator_address, merged_config.logging(), &mut builder, &resolved_product_image.product_version, @@ -1003,21 +993,31 @@ fn build_rolegroup_statefulset( // Vector sidecar shall be the last container in the list if merged_config.logging().enable_vector_agent { - pod_builder.add_container( - product_logging::framework::vector_container( - resolved_product_image, - "hbase-config", - "log", - merged_config.logging().containers.get(&Container::Vector), - ResourceRequirementsBuilder::new() - .with_cpu_request("250m") - .with_cpu_limit("500m") - .with_memory_request("128Mi") - .with_memory_limit("128Mi") - .build(), - ) - .context(ConfigureLoggingSnafu)?, - ); + if let Some(vector_aggregator_config_map_name) = hbase + .spec + .cluster_config + .vector_aggregator_config_map_name + .to_owned() + { + pod_builder.add_container( + product_logging::framework::vector_container( + resolved_product_image, + "hbase-config", + "log", + merged_config.logging().containers.get(&Container::Vector), + ResourceRequirementsBuilder::new() + .with_cpu_request("250m") + .with_cpu_limit("500m") + .with_memory_request("128Mi") + .with_memory_limit("128Mi") + .build(), + &vector_aggregator_config_map_name, + ) + .context(ConfigureLoggingSnafu)?, + ); + } else { + VectorAggregatorConfigMapMissingSnafu.fail()?; + } } let mut pod_template = pod_builder.build_template(); diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index dc13d535..46f224de 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -5,13 +5,18 @@ use futures::StreamExt; use hbase_controller::FULL_HBASE_CONTROLLER_NAME; use stackable_operator::{ cli::{Command, ProductOperatorRun}, - k8s_openapi::api::{apps::v1::StatefulSet, core::v1::Service}, + k8s_openapi::api::{ + apps::v1::StatefulSet, + core::v1::{ConfigMap, Service}, + }, kube::{ core::DeserializeGuard, runtime::{ events::{Recorder, Reporter}, + reflector::ObjectRef, watcher, Controller, }, + ResourceExt, }, logging::controller::report_controller_reconciled, shared::yaml::SerializeOptions, @@ -88,46 +93,77 @@ async fn main() -> anyhow::Result<()> { }, )); - Controller::new( + let hbase_controller = Controller::new( watch_namespace.get_api::>(&client), watcher::Config::default(), - ) - .owns( - watch_namespace.get_api::(&client), - watcher::Config::default(), - ) - .owns( - watch_namespace.get_api::(&client), - watcher::Config::default(), - ) - .shutdown_on_signal() - .run( - hbase_controller::reconcile_hbase, - hbase_controller::error_policy, - Arc::new(hbase_controller::Ctx { - client: client.clone(), - product_config, - }), - ) - .for_each_concurrent( - 16, // concurrency limit - |result| { - // The event_recorder needs to be shared across all invocations, so that - // events are correctly aggregated - let event_recorder = event_recorder.clone(); - async move { - report_controller_reconciled( - &event_recorder, - FULL_HBASE_CONTROLLER_NAME, - &result, - ) - .await; - } - }, - ) - .await; + ); + let hbase_store_1 = hbase_controller.store(); + hbase_controller + .owns( + watch_namespace.get_api::(&client), + watcher::Config::default(), + ) + .owns( + watch_namespace.get_api::(&client), + watcher::Config::default(), + ) + .shutdown_on_signal() + .watches( + watch_namespace.get_api::>(&client), + watcher::Config::default(), + move |config_map| { + hbase_store_1 + .state() + .into_iter() + .filter(move |hbase| references_config_map(hbase, &config_map)) + .map(|hbase| ObjectRef::from_obj(&*hbase)) + }, + ) + .run( + hbase_controller::reconcile_hbase, + hbase_controller::error_policy, + Arc::new(hbase_controller::Ctx { + client: client.clone(), + product_config, + }), + ) + .for_each_concurrent( + 16, // concurrency limit + |result| { + // The event_recorder needs to be shared across all invocations, so that + // events are correctly aggregated + let event_recorder = event_recorder.clone(); + async move { + report_controller_reconciled( + &event_recorder, + FULL_HBASE_CONTROLLER_NAME, + &result, + ) + .await; + } + }, + ) + .await; } } Ok(()) } + +fn references_config_map( + hbase: &DeserializeGuard, + config_map: &DeserializeGuard, +) -> bool { + let Ok(hbase) = &hbase.0 else { + return false; + }; + + hbase.spec.cluster_config.zookeeper_config_map_name == config_map.name_any() + || hbase.spec.cluster_config.hdfs_config_map_name == config_map.name_any() + || match hbase.spec.cluster_config.authorization.to_owned() { + Some(hbase_authorization) => { + hbase_authorization.opa.config_map_name == config_map.name_any() + } + None => false, + } +} diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs index 29d896ff..7e3cf831 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/product_logging.rs @@ -1,9 +1,6 @@ -use snafu::{OptionExt, ResultExt, Snafu}; +use snafu::Snafu; use stackable_operator::{ builder::configmap::ConfigMapBuilder, - client::Client, - k8s_openapi::api::core::v1::ConfigMap, - kube::ResourceExt, memory::BinaryMultiple, product_logging::{ self, @@ -45,7 +42,6 @@ pub enum Error { type Result = std::result::Result; -const VECTOR_AGGREGATOR_CM_ENTRY: &str = "ADDRESS"; const CONSOLE_CONVERSION_PATTERN: &str = "%d{ISO8601} %-5p [%t] %c{2}: %.1000m%n"; const HBASE_LOG4J_FILE: &str = "hbase.log4j.xml"; const HBASE_LOG4J2_FILE: &str = "hbase.log4j2.xml"; @@ -55,45 +51,9 @@ pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; pub static CONTAINERDEBUG_LOG_DIRECTORY: std::sync::LazyLock = std::sync::LazyLock::new(|| format!("{STACKABLE_LOG_DIR}/containerdebug")); -/// Return the address of the Vector aggregator if the corresponding ConfigMap name is given in the -/// cluster spec -pub async fn resolve_vector_aggregator_address( - hbase: &v1alpha1::HbaseCluster, - client: &Client, -) -> Result> { - let vector_aggregator_address = if let Some(vector_aggregator_config_map_name) = - &hbase.spec.cluster_config.vector_aggregator_config_map_name - { - let vector_aggregator_address = client - .get::( - vector_aggregator_config_map_name, - hbase - .namespace() - .as_deref() - .context(ObjectHasNoNamespaceSnafu)?, - ) - .await - .context(ConfigMapNotFoundSnafu { - cm_name: vector_aggregator_config_map_name.to_string(), - })? - .data - .and_then(|mut data| data.remove(VECTOR_AGGREGATOR_CM_ENTRY)) - .context(MissingConfigMapEntrySnafu { - entry: VECTOR_AGGREGATOR_CM_ENTRY, - cm_name: vector_aggregator_config_map_name.to_string(), - })?; - Some(vector_aggregator_address) - } else { - None - }; - - Ok(vector_aggregator_address) -} - /// Extend the role group ConfigMap with logging and Vector configurations pub fn extend_role_group_config_map( rolegroup: &RoleGroupRef, - vector_aggregator_address: Option<&str>, logging: &Logging, cm_builder: &mut ConfigMapBuilder, hbase_version: &str, @@ -120,11 +80,7 @@ pub fn extend_role_group_config_map( if logging.enable_vector_agent { cm_builder.add_data( product_logging::framework::VECTOR_CONFIG_FILE, - product_logging::framework::create_vector_config( - rolegroup, - vector_aggregator_address.context(MissingVectorAggregatorAddressSnafu)?, - vector_log_config, - ), + product_logging::framework::create_vector_config(rolegroup, vector_log_config), ); } From 70d10434ba851e26e3107ce02fc08a20877e4b24 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 4 Apr 2025 11:19:01 +0200 Subject: [PATCH 02/15] wip: use local stackable-operator version --- Cargo.lock | 118 ++++++++++++++++++++++++++++++++--------------------- Cargo.toml | 4 +- 2 files changed, 73 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 29cdd24b..315f9d77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,7 +27,7 @@ dependencies = [ "getrandom", "once_cell", "version_check", - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -168,14 +168,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] -name = "backoff" -version = "0.4.0" +name = "backon" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +checksum = "970d91570c01a8a5959b36ad7dd1c30642df24b6b3068710066f6809f7033bb7" dependencies = [ - "getrandom", - "instant", - "rand", + "fastrand", + "gloo-timers", + "tokio", ] [[package]] @@ -633,6 +633,12 @@ dependencies = [ "regex-syntax 0.8.5", ] +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "fnv" version = "1.0.7" @@ -802,6 +808,18 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "hashbrown" version = "0.15.2" @@ -1189,15 +1207,6 @@ version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", -] - [[package]] name = "integer-encoding" version = "3.0.4" @@ -1257,9 +1266,9 @@ dependencies = [ [[package]] name = "json-patch" -version = "3.0.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "863726d7afb6bc2590eeff7135d923545e5e964f004c2ccf8716c25e70a86f08" +checksum = "159294d661a039f7644cea7e4d844e6b25aaf71c1ffe9d73a96d768c24b0faf4" dependencies = [ "jsonptr", "serde", @@ -1282,9 +1291,9 @@ dependencies = [ [[package]] name = "jsonptr" -version = "0.6.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dea2b27dd239b2556ed7a25ba842fe47fd602e7fc7433c2a8d6106d4d9edd70" +checksum = "a5a3cc660ba5d72bce0b3bb295bf20847ccbb40fd423f3f05b61273672e561fe" dependencies = [ "serde", "serde_json", @@ -1307,7 +1316,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.2" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.6.0#53ccc1e9eca2a5b35a8618593c548e8687fb150d" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.7.0#a6d8db57a3108a4fb12eb82f9f7941f55ae2c88d" dependencies = [ "darling", "regex", @@ -1316,9 +1325,9 @@ dependencies = [ [[package]] name = "kube" -version = "0.98.0" +version = "0.99.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32053dc495efad4d188c7b33cc7c02ef4a6e43038115348348876efd39a53cba" +checksum = "9a4eb20010536b48abe97fec37d23d43069bcbe9686adcf9932202327bc5ca6e" dependencies = [ "k8s-openapi", "kube-client", @@ -1329,9 +1338,9 @@ dependencies = [ [[package]] name = "kube-client" -version = "0.98.0" +version = "0.99.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d34ad38cdfbd1fa87195d42569f57bb1dda6ba5f260ee32fef9570b7937a0c9" +checksum = "7fc2ed952042df20d15ac2fe9614d0ec14b6118eab89633985d4b36e688dccf1" dependencies = [ "base64 0.22.1", "bytes", @@ -1352,7 +1361,6 @@ dependencies = [ "kube-core", "pem", "rustls", - "rustls-pemfile", "secrecy", "serde", "serde_json", @@ -1367,9 +1375,9 @@ dependencies = [ [[package]] name = "kube-core" -version = "0.98.0" +version = "0.99.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97aa830b288a178a90e784d1b0f1539f2d200d2188c7b4a3146d9dc983d596f3" +checksum = "ff0d0793db58e70ca6d689489183816cb3aa481673e7433dc618cf7e8007c675" dependencies = [ "chrono", "form_urlencoded", @@ -1385,34 +1393,34 @@ dependencies = [ [[package]] name = "kube-derive" -version = "0.98.0" +version = "0.99.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37745d8a4076b77e0b1952e94e358726866c8e14ec94baaca677d47dcdb98658" +checksum = "c562f58dc9f7ca5feac8a6ee5850ca221edd6f04ce0dd2ee873202a88cd494c9" dependencies = [ "darling", "proc-macro2", "quote", + "serde", "serde_json", "syn 2.0.99", ] [[package]] name = "kube-runtime" -version = "0.98.0" +version = "0.99.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a41af186a0fe80c71a13a13994abdc3ebff80859ca6a4b8a6079948328c135b" +checksum = "88f34cfab9b4bd8633062e0e85edb81df23cb09f159f2e31c60b069ae826ffdc" dependencies = [ "ahash", "async-broadcast", "async-stream", "async-trait", - "backoff", + "backon", "educe", "futures 0.3.31", "hashbrown", "hostname", "json-patch", - "jsonptr", "k8s-openapi", "kube-client", "parking_lot", @@ -1798,11 +1806,11 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy", + "zerocopy 0.8.24", ] [[package]] @@ -2383,8 +2391,7 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.87.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.0#53ccc1e9eca2a5b35a8618593c548e8687fb150d" +version = "0.87.5" dependencies = [ "chrono", "clap", @@ -2422,7 +2429,6 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.0#53ccc1e9eca2a5b35a8618593c548e8687fb150d" dependencies = [ "darling", "proc-macro2", @@ -2433,7 +2439,6 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.0.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.87.0#53ccc1e9eca2a5b35a8618593c548e8687fb150d" dependencies = [ "kube", "semver", @@ -2444,16 +2449,16 @@ dependencies = [ [[package]] name = "stackable-versioned" -version = "0.6.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.6.0#53ccc1e9eca2a5b35a8618593c548e8687fb150d" +version = "0.7.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.7.0#a6d8db57a3108a4fb12eb82f9f7941f55ae2c88d" dependencies = [ "stackable-versioned-macros", ] [[package]] name = "stackable-versioned-macros" -version = "0.6.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.6.0#53ccc1e9eca2a5b35a8618593c548e8687fb150d" +version = "0.7.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.7.0#a6d8db57a3108a4fb12eb82f9f7941f55ae2c88d" dependencies = [ "convert_case", "darling", @@ -3250,8 +3255,16 @@ version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ - "byteorder", - "zerocopy-derive", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +dependencies = [ + "zerocopy-derive 0.8.24", ] [[package]] @@ -3265,6 +3278,17 @@ dependencies = [ "syn 2.0.99", ] +[[package]] +name = "zerocopy-derive" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.99", +] + [[package]] name = "zerofrom" version = "0.1.6" diff --git a/Cargo.toml b/Cargo.toml index 07d677ef..8737a7a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,6 @@ strum = { version = "0.27", features = ["derive"] } tokio = { version = "1.40", features = ["full"] } tracing = "0.1" -# [patch."https://github.com/stackabletech/operator-rs.git"] +[patch."https://github.com/stackabletech/operator-rs.git"] # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } -# stackable-operator = { path = "../operator-rs/crates/stackable-operator" } +stackable-operator = { path = "../operator-rs/crates/stackable-operator" } From 2afa1c2a8c721c56385e9c8c7c03ff5fb66dff0f Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 4 Apr 2025 11:40:21 +0200 Subject: [PATCH 03/15] add changelog entry --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4651df61..e77ae7f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## [Unreleased] +### Added + +- Inject the vector aggregator address into the vector config using the env var `VECTOR_AGGREGATOR_ADDRESS` ([#645]). + +### Fixed + +- Fix a bug where changes to ConfigMaps that are referenced in the HbaseCluster spec didn't trigger a reconciliation ([#645]). + +[#645]: https://github.com/stackabletech/airflow-operator/pull/645 + ## [25.3.0] - 2025-03-21 ### Added From 88c553405f8a4cd39e7d0217e7cf3566daa479e6 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 8 Apr 2025 11:40:22 +0200 Subject: [PATCH 04/15] bump operator-rs --- Cargo.lock | 5 +- rust/operator-binary/src/crd/affinity.rs | 74 +++++++++++++----------- rust/operator-binary/src/main.rs | 6 +- 3 files changed, 46 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4bdbff9d..6cd5782b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2654,8 +2654,7 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.89.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.89.1#cd73728af410c52972b9a9a3ba1302bcdb574d04" +version = "0.90.0" dependencies = [ "chrono", "clap", @@ -2690,7 +2689,6 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.89.1#cd73728af410c52972b9a9a3ba1302bcdb574d04" dependencies = [ "darling", "proc-macro2", @@ -2701,7 +2699,6 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.0.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.89.1#cd73728af410c52972b9a9a3ba1302bcdb574d04" dependencies = [ "kube", "semver", diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 3c079faf..cccf320e 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -186,39 +186,45 @@ mod tests { HbaseRole::RestServer => (), }; - assert_eq!(affinity, StackableAffinity { - pod_affinity: Some(PodAffinity { - preferred_during_scheduling_ignored_during_execution: Some(expected_affinities), - required_during_scheduling_ignored_during_execution: None, - }), - pod_anti_affinity: Some(PodAntiAffinity { - preferred_during_scheduling_ignored_during_execution: Some(vec![ - WeightedPodAffinityTerm { - pod_affinity_term: PodAffinityTerm { - label_selector: Some(LabelSelector { - match_expressions: None, - match_labels: Some(BTreeMap::from([ - ("app.kubernetes.io/name".to_string(), "hbase".to_string(),), - ( - "app.kubernetes.io/instance".to_string(), - "simple-hbase".to_string(), - ), - ("app.kubernetes.io/component".to_string(), role.to_string(),) - ])) - }), - match_label_keys: None, - mismatch_label_keys: None, - namespace_selector: None, - namespaces: None, - topology_key: "kubernetes.io/hostname".to_string(), - }, - weight: 70 - } - ]), - required_during_scheduling_ignored_during_execution: None, - }), - node_affinity: None, - node_selector: None, - }); + assert_eq!( + affinity, + StackableAffinity { + pod_affinity: Some(PodAffinity { + preferred_during_scheduling_ignored_during_execution: Some(expected_affinities), + required_during_scheduling_ignored_during_execution: None, + }), + pod_anti_affinity: Some(PodAntiAffinity { + preferred_during_scheduling_ignored_during_execution: Some(vec![ + WeightedPodAffinityTerm { + pod_affinity_term: PodAffinityTerm { + label_selector: Some(LabelSelector { + match_expressions: None, + match_labels: Some(BTreeMap::from([ + ("app.kubernetes.io/name".to_string(), "hbase".to_string(),), + ( + "app.kubernetes.io/instance".to_string(), + "simple-hbase".to_string(), + ), + ( + "app.kubernetes.io/component".to_string(), + role.to_string(), + ) + ])) + }), + match_label_keys: None, + mismatch_label_keys: None, + namespace_selector: None, + namespaces: None, + topology_key: "kubernetes.io/hostname".to_string(), + }, + weight: 70 + } + ]), + required_during_scheduling_ignored_during_execution: None, + }), + node_affinity: None, + node_selector: None, + } + ); } } diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 92fa9635..ed40fbc8 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -6,13 +6,17 @@ use hbase_controller::FULL_HBASE_CONTROLLER_NAME; use stackable_operator::{ YamlSchema, cli::{Command, ProductOperatorRun, RollingPeriod}, - k8s_openapi::api::{apps::v1::StatefulSet, core::v1::Service}, + k8s_openapi::api::{ + apps::v1::StatefulSet, + core::v1::{ConfigMap, Service}, + }, kube::{ ResourceExt, core::DeserializeGuard, runtime::{ Controller, events::{Recorder, Reporter}, + reflector::ObjectRef, watcher, }, }, From e89b2538c0c59d92153790d7c21b8ef5cc1b1630 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 8 Apr 2025 11:41:27 +0200 Subject: [PATCH 05/15] remove local patch for operator-rs --- Cargo.lock | 3 +++ Cargo.toml | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6cd5782b..99e2e35c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2655,6 +2655,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.90.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.90.0#ea063b4595caa20c82d37c595487c76476c9ab10" dependencies = [ "chrono", "clap", @@ -2689,6 +2690,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.90.0#ea063b4595caa20c82d37c595487c76476c9ab10" dependencies = [ "darling", "proc-macro2", @@ -2699,6 +2701,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.0.1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.90.0#ea063b4595caa20c82d37c595487c76476c9ab10" dependencies = [ "kube", "semver", diff --git a/Cargo.toml b/Cargo.toml index 8685c5e9..c7c167d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,6 @@ strum = { version = "0.27", features = ["derive"] } tokio = { version = "1.40", features = ["full"] } tracing = "0.1" -[patch."https://github.com/stackabletech/operator-rs.git"] +# [patch."https://github.com/stackabletech/operator-rs.git"] # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } -stackable-operator = { path = "../operator-rs/crates/stackable-operator" } +# stackable-operator = { path = "../operator-rs/crates/stackable-operator" } From abcda67e2feed8cd7e76832bf340b12f9fce0c24 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 8 Apr 2025 12:33:14 +0200 Subject: [PATCH 06/15] chore: Update nix files --- Cargo.nix | 20 ++++++++++---------- crate-hashes.json | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index cec128e4..ea61e7e1 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -7178,10 +7178,10 @@ rec { }; "ring" = rec { crateName = "ring"; - version = "0.17.11"; + version = "0.17.14"; edition = "2021"; - links = "ring_core_0_17_11_"; - sha256 = "0wzyhdbf71ndd14kkpyj2a6nvczvli2mndzv2al7r26k4yp4jlys"; + links = "ring_core_0_17_14_"; + sha256 = "1dw32gv19ccq4hsx3ribhpdzri1vnrlcfqb2vj41xn4l49n9ws54"; dependencies = [ { name = "cfg-if"; @@ -8580,13 +8580,13 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.89.1"; + version = "0.90.0"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "cd73728af410c52972b9a9a3ba1302bcdb574d04"; - sha256 = "1hrxrybc6197ibx0m2wfxlg5pdg4hanf6xvslzrhsp77a04pb0y9"; + rev = "ea063b4595caa20c82d37c595487c76476c9ab10"; + sha256 = "0fclvpxhchykqd7bl8hscr4v06mbs2v5vjp0xv27nvqr94j63xs2"; }; libName = "stackable_operator"; authors = [ @@ -8731,8 +8731,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "cd73728af410c52972b9a9a3ba1302bcdb574d04"; - sha256 = "1hrxrybc6197ibx0m2wfxlg5pdg4hanf6xvslzrhsp77a04pb0y9"; + rev = "ea063b4595caa20c82d37c595487c76476c9ab10"; + sha256 = "0fclvpxhchykqd7bl8hscr4v06mbs2v5vjp0xv27nvqr94j63xs2"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -8766,8 +8766,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "cd73728af410c52972b9a9a3ba1302bcdb574d04"; - sha256 = "1hrxrybc6197ibx0m2wfxlg5pdg4hanf6xvslzrhsp77a04pb0y9"; + rev = "ea063b4595caa20c82d37c595487c76476c9ab10"; + sha256 = "0fclvpxhchykqd7bl8hscr4v06mbs2v5vjp0xv27nvqr94j63xs2"; }; libName = "stackable_shared"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index 0460dbd5..c8a9703a 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,7 +1,7 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.89.1#stackable-operator-derive@0.3.1": "1hrxrybc6197ibx0m2wfxlg5pdg4hanf6xvslzrhsp77a04pb0y9", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.89.1#stackable-operator@0.89.1": "1hrxrybc6197ibx0m2wfxlg5pdg4hanf6xvslzrhsp77a04pb0y9", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.89.1#stackable-shared@0.0.1": "1hrxrybc6197ibx0m2wfxlg5pdg4hanf6xvslzrhsp77a04pb0y9", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.90.0#stackable-operator-derive@0.3.1": "0fclvpxhchykqd7bl8hscr4v06mbs2v5vjp0xv27nvqr94j63xs2", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.90.0#stackable-operator@0.90.0": "0fclvpxhchykqd7bl8hscr4v06mbs2v5vjp0xv27nvqr94j63xs2", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.90.0#stackable-shared@0.0.1": "0fclvpxhchykqd7bl8hscr4v06mbs2v5vjp0xv27nvqr94j63xs2", "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-telemetry-0.4.0#stackable-telemetry@0.4.0": "0hcm64fb2ngyalq8rci5lrr881prg023pq9cd1sfr79iynbr6a26", "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.7.1#k8s-version@0.1.2": "16klfwx3kz3ys7afwjicfj8msws9a718izx09jspwwpff3rl6wsi", "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.7.1#stackable-versioned-macros@0.7.1": "16klfwx3kz3ys7afwjicfj8msws9a718izx09jspwwpff3rl6wsi", From ff3e85cf266a4dd01caf553386bf1a138bf3e728 Mon Sep 17 00:00:00 2001 From: Stacky McStackface <95074132+stackable-bot@users.noreply.github.com> Date: Wed, 9 Apr 2025 09:40:30 +0200 Subject: [PATCH 07/15] chore: Generated commit to update templated files since the last template run up to stackabletech/operator-templating@8ad056844d17e91e62f8574da12e885fafdfc230 (#646) Reference-to: stackabletech/operator-templating@8ad0568 (Update nixpkgs) --- Cargo.nix | 905 ++++++++++++++++++++++++++--------------------- nix/README.md | 27 ++ nix/sources.json | 20 +- 3 files changed, 547 insertions(+), 405 deletions(-) create mode 100644 nix/README.md diff --git a/Cargo.nix b/Cargo.nix index ea61e7e1..222887f6 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5,6 +5,7 @@ { nixpkgs ? , pkgs ? import nixpkgs { config = {}; } +, fetchurl ? pkgs.fetchurl , lib ? pkgs.lib , stdenv ? pkgs.stdenv , buildRustCrateForPkgs ? pkgs: pkgs.buildRustCrate @@ -61,6 +62,8 @@ rec { }; }; + + # A derivation that joins the outputs of all workspace members together. allWorkspaceMembers = pkgs.symlinkJoin { name = "all-workspace-members"; @@ -971,14 +974,14 @@ rec { { name = "tokio"; packageId = "tokio"; - target = {target, features}: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!("wasm32" == target."arch" or null)); features = [ "time" "rt" "macros" "sync" "rt-multi-thread" ]; } { name = "tokio"; packageId = "tokio"; usesDefaultFeatures = false; - target = {target, features}: ("wasm32" == target."arch" or null); + target = { target, features }: ("wasm32" == target."arch" or null); features = [ "macros" "rt" "sync" ]; } ]; @@ -1649,7 +1652,7 @@ rec { name = "libc"; packageId = "libc"; usesDefaultFeatures = false; - target = { target, features }: (stdenv.hostPlatform.rust.rustcTarget == "aarch64-linux-android"); + target = { target, features }: (target.name == "aarch64-linux-android"); } { name = "libc"; @@ -7098,33 +7101,33 @@ rec { name = "futures-util"; packageId = "futures-util"; usesDefaultFeatures = false; - target = {target, features}: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!("wasm32" == target."arch" or null)); features = [ "std" "alloc" ]; } { name = "hyper"; packageId = "hyper"; usesDefaultFeatures = false; - target = {target, features}: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!("wasm32" == target."arch" or null)); features = [ "http1" "http2" "client" "server" ]; } { name = "hyper-util"; packageId = "hyper-util"; - target = {target, features}: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!("wasm32" == target."arch" or null)); features = [ "http1" "http2" "client" "client-legacy" "server-auto" "tokio" ]; } { name = "serde"; packageId = "serde"; - target = {target, features}: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!("wasm32" == target."arch" or null)); features = [ "derive" ]; } { name = "tokio"; packageId = "tokio"; usesDefaultFeatures = false; - target = {target, features}: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!("wasm32" == target."arch" or null)); features = [ "macros" "rt-multi-thread" ]; } { @@ -7136,7 +7139,7 @@ rec { { name = "wasm-bindgen"; packageId = "wasm-bindgen"; - target = {target, features}: ("wasm32" == target."arch" or null); + target = { target, features }: ("wasm32" == target."arch" or null); features = [ "serde-serialize" ]; } ]; @@ -7227,7 +7230,7 @@ rec { name = "libc"; packageId = "libc"; usesDefaultFeatures = false; - target = {target, features}: ((target."unix" or false) || (target."windows" or false) || ("wasi" == target."os" or null)); + target = { target, features }: ((target."unix" or false) || (target."windows" or false) || ("wasi" == target."os" or null)); } ]; features = { @@ -9516,17 +9519,17 @@ rec { { name = "libc"; packageId = "libc"; - target = {target, features}: (target."unix" or false); + target = { target, features }: (target."unix" or false); } { name = "socket2"; packageId = "socket2"; - target = {target, features}: (!(builtins.elem "wasm" target."family")); + target = { target, features }: (!(builtins.elem "wasm" target."family")); } { name = "windows-sys"; packageId = "windows-sys 0.52.0"; - target = {target, features}: (target."windows" or false); + target = { target, features }: (target."windows" or false); features = [ "Win32_Foundation" "Win32_Security_Authorization" ]; } ]; @@ -11679,12 +11682,12 @@ rec { { name = "winapi-i686-pc-windows-gnu"; packageId = "winapi-i686-pc-windows-gnu"; - target = { target, features }: (stdenv.hostPlatform.rust.rustcTarget == "i686-pc-windows-gnu"); + target = { target, features }: (target.name == "i686-pc-windows-gnu"); } { name = "winapi-x86_64-pc-windows-gnu"; packageId = "winapi-x86_64-pc-windows-gnu"; - target = { target, features }: (stdenv.hostPlatform.rust.rustcTarget == "x86_64-pc-windows-gnu"); + target = { target, features }: (target.name == "x86_64-pc-windows-gnu"); } ]; features = { @@ -13016,7 +13019,7 @@ rec { { name = "windows_aarch64_gnullvm"; packageId = "windows_aarch64_gnullvm 0.52.6"; - target = { target, features }: (stdenv.hostPlatform.rust.rustcTarget == "aarch64-pc-windows-gnullvm"); + target = { target, features }: (target.name == "aarch64-pc-windows-gnullvm"); } { name = "windows_aarch64_msvc"; @@ -13031,7 +13034,7 @@ rec { { name = "windows_i686_gnullvm"; packageId = "windows_i686_gnullvm 0.52.6"; - target = { target, features }: (stdenv.hostPlatform.rust.rustcTarget == "i686-pc-windows-gnullvm"); + target = { target, features }: (target.name == "i686-pc-windows-gnullvm"); } { name = "windows_i686_msvc"; @@ -13046,7 +13049,7 @@ rec { { name = "windows_x86_64_gnullvm"; packageId = "windows_x86_64_gnullvm 0.52.6"; - target = { target, features }: (stdenv.hostPlatform.rust.rustcTarget == "x86_64-pc-windows-gnullvm"); + target = { target, features }: (target.name == "x86_64-pc-windows-gnullvm"); } { name = "windows_x86_64_msvc"; @@ -13069,7 +13072,7 @@ rec { { name = "windows_aarch64_gnullvm"; packageId = "windows_aarch64_gnullvm 0.53.0"; - target = { target, features }: (stdenv.hostPlatform.rust.rustcTarget == "aarch64-pc-windows-gnullvm"); + target = { target, features }: (target.name == "aarch64-pc-windows-gnullvm"); } { name = "windows_aarch64_msvc"; @@ -13084,7 +13087,7 @@ rec { { name = "windows_i686_gnullvm"; packageId = "windows_i686_gnullvm 0.53.0"; - target = { target, features }: (stdenv.hostPlatform.rust.rustcTarget == "i686-pc-windows-gnullvm"); + target = { target, features }: (target.name == "i686-pc-windows-gnullvm"); } { name = "windows_i686_msvc"; @@ -13099,7 +13102,7 @@ rec { { name = "windows_x86_64_gnullvm"; packageId = "windows_x86_64_gnullvm 0.53.0"; - target = { target, features }: (stdenv.hostPlatform.rust.rustcTarget == "x86_64-pc-windows-gnullvm"); + target = { target, features }: (target.name == "x86_64-pc-windows-gnullvm"); } { name = "windows_x86_64_msvc"; @@ -13671,10 +13674,13 @@ rec { # crate2nix/default.nix (excerpt start) # - /* Target (platform) data for conditional dependencies. + /* + Target (platform) data for conditional dependencies. This corresponds roughly to what buildRustCrate is setting. */ makeDefaultTarget = platform: { + name = platform.rust.rustcTarget; + unix = platform.isUnix; windows = platform.isWindows; fuchsia = true; @@ -13683,30 +13689,70 @@ rec { inherit (platform.rust.platform) arch os - vendor; + vendor + ; family = platform.rust.platform.target-family; env = "gnu"; - endian = - if platform.parsed.cpu.significantByte.name == "littleEndian" - then "little" else "big"; + endian = if platform.parsed.cpu.significantByte.name == "littleEndian" then "little" else "big"; pointer_width = toString platform.parsed.cpu.bits; debug_assertions = false; }; - /* Filters common temp files and build files. */ + registryUrl = + { registries + , url + , crate + , version + , sha256 + , + }: + let + dl = registries.${url}.dl; + tmpl = [ + "{crate}" + "{version}" + "{prefix}" + "{lowerprefix}" + "{sha256-checksum}" + ]; + in + with lib.strings; + if lib.lists.any (i: hasInfix "{}" dl) tmpl then + let + prefix = + if builtins.stringLength crate == 1 then + "1" + else if builtins.stringLength crate == 2 then + "2" + else + "${builtins.substring 0 2 crate}/${builtins.substring 2 (builtins.stringLength crate - 2) crate}"; + in + builtins.replaceStrings tmpl [ + crate + version + prefix + (lib.strings.toLower prefix) + sha256 + ] + else + "${dl}/${crate}/${version}/download"; + + # Filters common temp files and build files. # TODO(pkolloch): Substitute with gitignore filter - sourceFilter = name: type: + sourceFilter = + name: type: let baseName = builtins.baseNameOf (builtins.toString name); in - ! ( + !( # Filter out git baseName == ".gitignore" || (type == "directory" && baseName == ".git") # Filter out build results || ( - type == "directory" && ( + type == "directory" + && ( baseName == "target" || baseName == "_site" || baseName == ".sass-cache" @@ -13716,16 +13762,11 @@ rec { ) # Filter out nix-build result symlinks - || ( - type == "symlink" && lib.hasPrefix "result" baseName - ) + || (type == "symlink" && lib.hasPrefix "result" baseName) # Filter out IDE config - || ( - type == "directory" && ( - baseName == ".idea" || baseName == ".vscode" - ) - ) || lib.hasSuffix ".iml" baseName + || (type == "directory" && (baseName == ".idea" || baseName == ".vscode")) + || lib.hasSuffix ".iml" baseName # Filter out nix build files || baseName == "Cargo.nix" @@ -13739,90 +13780,100 @@ rec { || baseName == "tests.nix" ); - /* Returns a crate which depends on successful test execution + /* + Returns a crate which depends on successful test execution of crate given as the second argument. testCrateFlags: list of flags to pass to the test exectuable testInputs: list of packages that should be available during test execution */ - crateWithTest = { crate, testCrate, testCrateFlags, testInputs, testPreRun, testPostRun }: - assert builtins.typeOf testCrateFlags == "list"; - assert builtins.typeOf testInputs == "list"; - assert builtins.typeOf testPreRun == "string"; - assert builtins.typeOf testPostRun == "string"; - let - # override the `crate` so that it will build and execute tests instead of - # building the actual lib and bin targets We just have to pass `--test` - # to rustc and it will do the right thing. We execute the tests and copy - # their log and the test executables to $out for later inspection. - test = - let - drv = testCrate.override - ( - _: { - buildTests = true; - } + crateWithTest = + { crate + , testCrate + , testCrateFlags + , testInputs + , testPreRun + , testPostRun + , + }: + assert builtins.typeOf testCrateFlags == "list"; + assert builtins.typeOf testInputs == "list"; + assert builtins.typeOf testPreRun == "string"; + assert builtins.typeOf testPostRun == "string"; + let + # override the `crate` so that it will build and execute tests instead of + # building the actual lib and bin targets We just have to pass `--test` + # to rustc and it will do the right thing. We execute the tests and copy + # their log and the test executables to $out for later inspection. + test = + let + drv = testCrate.override (_: { + buildTests = true; + }); + # If the user hasn't set any pre/post commands, we don't want to + # insert empty lines. This means that any existing users of crate2nix + # don't get a spurious rebuild unless they set these explicitly. + testCommand = pkgs.lib.concatStringsSep "\n" ( + pkgs.lib.filter (s: s != "") [ + testPreRun + "$f $testCrateFlags 2>&1 | tee -a $out" + testPostRun + ] ); - # If the user hasn't set any pre/post commands, we don't want to - # insert empty lines. This means that any existing users of crate2nix - # don't get a spurious rebuild unless they set these explicitly. - testCommand = pkgs.lib.concatStringsSep "\n" - (pkgs.lib.filter (s: s != "") [ - testPreRun - "$f $testCrateFlags 2>&1 | tee -a $out" - testPostRun - ]); - in - pkgs.stdenvNoCC.mkDerivation { - name = "run-tests-${testCrate.name}"; - - inherit (crate) src; - - inherit testCrateFlags; - - buildInputs = testInputs; - - buildPhase = '' - set -e - export RUST_BACKTRACE=1 - - # build outputs - testRoot=target/debug - mkdir -p $testRoot - - # executables of the crate - # we copy to prevent std::env::current_exe() to resolve to a store location - for i in ${crate}/bin/*; do - cp "$i" "$testRoot" - done - chmod +w -R . - - # test harness executables are suffixed with a hash, like cargo does - # this allows to prevent name collision with the main - # executables of the crate - hash=$(basename $out) - for file in ${drv}/tests/*; do - f=$testRoot/$(basename $file)-$hash - cp $file $f - ${testCommand} - done - ''; - }; - in - pkgs.runCommand "${crate.name}-linked" - { - inherit (crate) outputs crateName; - passthru = (crate.passthru or { }) // { - inherit test; - }; - } - (lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) '' - echo tested by ${test} - '' + '' - ${lib.concatMapStringsSep "\n" (output: "ln -s ${crate.${output}} ${"$"}${output}") crate.outputs} - ''); - - /* A restricted overridable version of builtRustCratesWithFeatures. */ + in + pkgs.stdenvNoCC.mkDerivation { + name = "run-tests-${testCrate.name}"; + + inherit (crate) src; + + inherit testCrateFlags; + + buildInputs = testInputs; + + buildPhase = '' + set -e + export RUST_BACKTRACE=1 + + # build outputs + testRoot=target/debug + mkdir -p $testRoot + + # executables of the crate + # we copy to prevent std::env::current_exe() to resolve to a store location + for i in ${crate}/bin/*; do + cp "$i" "$testRoot" + done + chmod +w -R . + + # test harness executables are suffixed with a hash, like cargo does + # this allows to prevent name collision with the main + # executables of the crate + hash=$(basename $out) + for file in ${drv}/tests/*; do + f=$testRoot/$(basename $file)-$hash + cp $file $f + ${testCommand} + done + ''; + }; + in + pkgs.runCommand "${crate.name}-linked" + { + inherit (crate) outputs crateName; + passthru = (crate.passthru or { }) // { + inherit test; + }; + } + ( + lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) '' + echo tested by ${test} + '' + + '' + ${lib.concatMapStringsSep "\n" (output: "ln -s ${crate.${output}} ${"$"}${output}") crate.outputs} + '' + ); + + # A restricted overridable version of builtRustCratesWithFeatures. buildRustCrateWithFeatures = { packageId , features ? rootFeatures @@ -13831,10 +13882,11 @@ rec { , runTests ? false , testCrateFlags ? [ ] , testInputs ? [ ] - # Any command to run immediatelly before a test is executed. - , testPreRun ? "" - # Any command run immediatelly after a test is executed. - , testPostRun ? "" + , # Any command to run immediatelly before a test is executed. + testPreRun ? "" + , # Any command run immediatelly after a test is executed. + testPostRun ? "" + , }: lib.makeOverridable ( @@ -13845,17 +13897,19 @@ rec { , testInputs , testPreRun , testPostRun + , }: let buildRustCrateForPkgsFuncOverriden = - if buildRustCrateForPkgsFunc != null - then buildRustCrateForPkgsFunc + if buildRustCrateForPkgsFunc != null then + buildRustCrateForPkgsFunc else ( - if crateOverrides == pkgs.defaultCrateOverrides - then buildRustCrateForPkgs + if crateOverrides == pkgs.defaultCrateOverrides then + buildRustCrateForPkgs else - pkgs: (buildRustCrateForPkgs pkgs).override { + pkgs: + (buildRustCrateForPkgs pkgs).override { defaultCrateOverrides = crateOverrides; } ); @@ -13877,15 +13931,32 @@ rec { { crate = drv; testCrate = testDrv; - inherit testCrateFlags testInputs testPreRun testPostRun; + inherit + testCrateFlags + testInputs + testPreRun + testPostRun + ; } - else drv; + else + drv; in derivation ) - { inherit features crateOverrides runTests testCrateFlags testInputs testPreRun testPostRun; }; + { + inherit + features + crateOverrides + runTests + testCrateFlags + testInputs + testPreRun + testPostRun + ; + }; - /* Returns an attr set with packageId mapped to the result of buildRustCrateForPkgsFunc + /* + Returns an attr set with packageId mapped to the result of buildRustCrateForPkgsFunc for the corresponding crate. */ builtRustCratesWithFeatures = @@ -13895,7 +13966,8 @@ rec { , buildRustCrateForPkgsFunc , runTests , makeTarget ? makeDefaultTarget - } @ args: + , + }@args: assert (builtins.isAttrs crateConfigs); assert (builtins.isString packageId); assert (builtins.isList features); @@ -13903,55 +13975,61 @@ rec { assert (builtins.isBool runTests); let rootPackageId = packageId; - mergedFeatures = mergePackageFeatures - ( - args // { - inherit rootPackageId; - target = makeTarget stdenv.hostPlatform // { test = runTests; }; - } - ); + mergedFeatures = mergePackageFeatures ( + args + // { + inherit rootPackageId; + target = makeTarget stdenv.hostPlatform // { + test = runTests; + }; + } + ); # Memoize built packages so that reappearing packages are only built once. builtByPackageIdByPkgs = mkBuiltByPackageIdByPkgs pkgs; - mkBuiltByPackageIdByPkgs = pkgs: + mkBuiltByPackageIdByPkgs = + pkgs: let self = { - crates = lib.mapAttrs (packageId: value: buildByPackageIdForPkgsImpl self pkgs packageId) crateConfigs; - target = makeTarget stdenv.hostPlatform; + crates = lib.mapAttrs + ( + packageId: value: buildByPackageIdForPkgsImpl self pkgs packageId + ) + crateConfigs; + target = makeTarget pkgs.stdenv.hostPlatform; build = mkBuiltByPackageIdByPkgs pkgs.buildPackages; }; in self; - buildByPackageIdForPkgsImpl = self: pkgs: packageId: + buildByPackageIdForPkgsImpl = + self: pkgs: packageId: let features = mergedFeatures."${packageId}" or [ ]; crateConfig' = crateConfigs."${packageId}"; - crateConfig = - builtins.removeAttrs crateConfig' [ "resolvedDefaultFeatures" "devDependencies" ]; - devDependencies = - lib.optionals - (runTests && packageId == rootPackageId) - (crateConfig'.devDependencies or [ ]); - dependencies = - dependencyDerivations { - inherit features; - inherit (self) target; - buildByPackageId = depPackageId: - # proc_macro crates must be compiled for the build architecture - if crateConfigs.${depPackageId}.procMacro or false - then self.build.crates.${depPackageId} - else self.crates.${depPackageId}; - dependencies = - (crateConfig.dependencies or [ ]) - ++ devDependencies; - }; - buildDependencies = - dependencyDerivations { - inherit features; - inherit (self.build) target; - buildByPackageId = depPackageId: - self.build.crates.${depPackageId}; - dependencies = crateConfig.buildDependencies or [ ]; - }; + crateConfig = builtins.removeAttrs crateConfig' [ + "resolvedDefaultFeatures" + "devDependencies" + ]; + devDependencies = lib.optionals (runTests && packageId == rootPackageId) ( + crateConfig'.devDependencies or [ ] + ); + dependencies = dependencyDerivations { + inherit features; + inherit (self) target; + buildByPackageId = + depPackageId: + # proc_macro crates must be compiled for the build architecture + if crateConfigs.${depPackageId}.procMacro or false then + self.build.crates.${depPackageId} + else + self.crates.${depPackageId}; + dependencies = (crateConfig.dependencies or [ ]) ++ devDependencies; + }; + buildDependencies = dependencyDerivations { + inherit features; + inherit (self.build) target; + buildByPackageId = depPackageId: self.build.crates.${depPackageId}; + dependencies = crateConfig.buildDependencies or [ ]; + }; dependenciesWithRenames = let buildDeps = filterEnabledDependencies { @@ -13976,45 +14054,54 @@ rec { # } crateRenames = let - grouped = - lib.groupBy - (dependency: dependency.name) - dependenciesWithRenames; - versionAndRename = dep: + grouped = lib.groupBy (dependency: dependency.name) dependenciesWithRenames; + versionAndRename = + dep: let package = crateConfigs."${dep.packageId}"; in - { inherit (dep) rename; inherit (package) version; }; + { + inherit (dep) rename; + inherit (package) version; + }; in lib.mapAttrs (name: builtins.map versionAndRename) grouped; in - buildRustCrateForPkgsFunc pkgs - ( - crateConfig // { - src = crateConfig.src or ( - pkgs.fetchurl rec { - name = "${crateConfig.crateName}-${crateConfig.version}.tar.gz"; - # https://www.pietroalbini.org/blog/downloading-crates-io/ - # Not rate-limited, CDN URL. - url = "https://static.crates.io/crates/${crateConfig.crateName}/${crateConfig.crateName}-${crateConfig.version}.crate"; - sha256 = - assert (lib.assertMsg (crateConfig ? sha256) "Missing sha256 for ${name}"); - crateConfig.sha256; - } - ); - extraRustcOpts = lib.lists.optional (targetFeatures != [ ]) "-C target-feature=${lib.concatMapStringsSep "," (x: "+${x}") targetFeatures}"; - inherit features dependencies buildDependencies crateRenames release; - } - ); + buildRustCrateForPkgsFunc pkgs ( + crateConfig + // { + src = + crateConfig.src or (pkgs.fetchurl rec { + name = "${crateConfig.crateName}-${crateConfig.version}.tar.gz"; + # https://www.pietroalbini.org/blog/downloading-crates-io/ + # Not rate-limited, CDN URL. + url = "https://static.crates.io/crates/${crateConfig.crateName}/${crateConfig.crateName}-${crateConfig.version}.crate"; + sha256 = + assert (lib.assertMsg (crateConfig ? sha256) "Missing sha256 for ${name}"); + crateConfig.sha256; + }); + extraRustcOpts = + lib.lists.optional (targetFeatures != [ ]) + "-C target-feature=${lib.concatMapStringsSep "," (x: "+${x}") targetFeatures}"; + inherit + features + dependencies + buildDependencies + crateRenames + release + ; + } + ); in builtByPackageIdByPkgs; - /* Returns the actual derivations for the given dependencies. */ + # Returns the actual derivations for the given dependencies. dependencyDerivations = { buildByPackageId , features , dependencies , target + , }: assert (builtins.isList features); assert (builtins.isList dependencies); @@ -14027,52 +14114,59 @@ rec { in map depDerivation enabledDependencies; - /* Returns a sanitized version of val with all values substituted that cannot + /* + Returns a sanitized version of val with all values substituted that cannot be serialized as JSON. */ - sanitizeForJson = val: - if builtins.isAttrs val - then lib.mapAttrs (n: sanitizeForJson) val - else if builtins.isList val - then builtins.map sanitizeForJson val - else if builtins.isFunction val - then "function" - else val; - - /* Returns various tools to debug a crate. */ - debugCrate = { packageId, target ? makeDefaultTarget stdenv.hostPlatform }: - assert (builtins.isString packageId); - let - debug = rec { - # The built tree as passed to buildRustCrate. - buildTree = buildRustCrateWithFeatures { - buildRustCrateForPkgsFunc = _: lib.id; - inherit packageId; - }; - sanitizedBuildTree = sanitizeForJson buildTree; - dependencyTree = sanitizeForJson - ( - buildRustCrateWithFeatures { - buildRustCrateForPkgsFunc = _: crate: { - "01_crateName" = crate.crateName or false; - "02_features" = crate.features or [ ]; - "03_dependencies" = crate.dependencies or [ ]; - }; - inherit packageId; - } - ); - mergedPackageFeatures = mergePackageFeatures { - features = rootFeatures; - inherit packageId target; - }; - diffedDefaultPackageFeatures = diffDefaultPackageFeatures { - inherit packageId target; + sanitizeForJson = + val: + if builtins.isAttrs val then + lib.mapAttrs (n: sanitizeForJson) val + else if builtins.isList val then + builtins.map sanitizeForJson val + else if builtins.isFunction val then + "function" + else + val; + + # Returns various tools to debug a crate. + debugCrate = + { packageId + , target ? makeDefaultTarget stdenv.hostPlatform + , + }: + assert (builtins.isString packageId); + let + debug = rec { + # The built tree as passed to buildRustCrate. + buildTree = buildRustCrateWithFeatures { + buildRustCrateForPkgsFunc = _: lib.id; + inherit packageId; + }; + sanitizedBuildTree = sanitizeForJson buildTree; + dependencyTree = sanitizeForJson (buildRustCrateWithFeatures { + buildRustCrateForPkgsFunc = _: crate: { + "01_crateName" = crate.crateName or false; + "02_features" = crate.features or [ ]; + "03_dependencies" = crate.dependencies or [ ]; + }; + inherit packageId; + }); + mergedPackageFeatures = mergePackageFeatures { + features = rootFeatures; + inherit packageId target; + }; + diffedDefaultPackageFeatures = diffDefaultPackageFeatures { + inherit packageId target; + }; }; + in + { + internal = debug; }; - in - { internal = debug; }; - /* Returns differences between cargo default features and crate2nix default + /* + Returns differences between cargo default features and crate2nix default features. This is useful for verifying the feature resolution in crate2nix. @@ -14081,22 +14175,26 @@ rec { { crateConfigs ? crates , packageId , target + , }: assert (builtins.isAttrs crateConfigs); let prefixValues = prefix: lib.mapAttrs (n: v: { "${prefix}" = v; }); - mergedFeatures = - prefixValues - "crate2nix" - (mergePackageFeatures { inherit crateConfigs packageId target; features = [ "default" ]; }); + mergedFeatures = prefixValues "crate2nix" (mergePackageFeatures { + inherit crateConfigs packageId target; + features = [ "default" ]; + }); configs = prefixValues "cargo" crateConfigs; - combined = lib.foldAttrs (a: b: a // b) { } [ mergedFeatures configs ]; - onlyInCargo = - builtins.attrNames - (lib.filterAttrs (n: v: !(v ? "crate2nix") && (v ? "cargo")) combined); - onlyInCrate2Nix = - builtins.attrNames - (lib.filterAttrs (n: v: (v ? "crate2nix") && !(v ? "cargo")) combined); + combined = lib.foldAttrs (a: b: a // b) { } [ + mergedFeatures + configs + ]; + onlyInCargo = builtins.attrNames ( + lib.filterAttrs (n: v: !(v ? "crate2nix") && (v ? "cargo")) combined + ); + onlyInCrate2Nix = builtins.attrNames ( + lib.filterAttrs (n: v: (v ? "crate2nix") && !(v ? "cargo")) combined + ); differentFeatures = lib.filterAttrs ( n: v: @@ -14110,7 +14208,8 @@ rec { inherit onlyInCargo onlyInCrate2Nix differentFeatures; }; - /* Returns an attrset mapping packageId to the list of enabled features. + /* + Returns an attrset mapping packageId to the list of enabled features. If multiple paths to a dependency enable different features, the corresponding feature sets are merged. Features in rust are additive. @@ -14123,10 +14222,10 @@ rec { , dependencyPath ? [ crates.${packageId}.crateName ] , featuresByPackageId ? { } , target - # Adds devDependencies to the crate with rootPackageId. - , runTests ? false + , # Adds devDependencies to the crate with rootPackageId. + runTests ? false , ... - } @ args: + }@args: assert (builtins.isAttrs crateConfigs); assert (builtins.isString packageId); assert (builtins.isString rootPackageId); @@ -14139,84 +14238,93 @@ rec { crateConfig = crateConfigs."${packageId}" or (builtins.throw "Package not found: ${packageId}"); expandedFeatures = expandFeatures (crateConfig.features or { }) features; enabledFeatures = enableFeatures (crateConfig.dependencies or [ ]) expandedFeatures; - depWithResolvedFeatures = dependency: + depWithResolvedFeatures = + dependency: let inherit (dependency) packageId; features = dependencyFeatures enabledFeatures dependency; in - { inherit packageId features; }; - resolveDependencies = cache: path: dependencies: - assert (builtins.isAttrs cache); - assert (builtins.isList dependencies); - let - enabledDependencies = filterEnabledDependencies { - inherit dependencies target; - features = enabledFeatures; - }; - directDependencies = map depWithResolvedFeatures enabledDependencies; - foldOverCache = op: lib.foldl op cache directDependencies; - in - foldOverCache - ( - cache: { packageId, features }: - let - cacheFeatures = cache.${packageId} or [ ]; - combinedFeatures = sortedUnique (cacheFeatures ++ features); - in - if cache ? ${packageId} && cache.${packageId} == combinedFeatures - then cache - else - mergePackageFeatures { - features = combinedFeatures; - featuresByPackageId = cache; - inherit crateConfigs packageId target runTests rootPackageId; - } + { + inherit packageId features; + }; + resolveDependencies = + cache: path: dependencies: + assert (builtins.isAttrs cache); + assert (builtins.isList dependencies); + let + enabledDependencies = filterEnabledDependencies { + inherit dependencies target; + features = enabledFeatures; + }; + directDependencies = map depWithResolvedFeatures enabledDependencies; + foldOverCache = op: lib.foldl op cache directDependencies; + in + foldOverCache ( + cache: + { packageId, features }: + let + cacheFeatures = cache.${packageId} or [ ]; + combinedFeatures = sortedUnique (cacheFeatures ++ features); + in + if cache ? ${packageId} && cache.${packageId} == combinedFeatures then + cache + else + mergePackageFeatures { + features = combinedFeatures; + featuresByPackageId = cache; + inherit + crateConfigs + packageId + target + runTests + rootPackageId + ; + } ); cacheWithSelf = let cacheFeatures = featuresByPackageId.${packageId} or [ ]; combinedFeatures = sortedUnique (cacheFeatures ++ enabledFeatures); in - featuresByPackageId // { + featuresByPackageId + // { "${packageId}" = combinedFeatures; }; - cacheWithDependencies = - resolveDependencies cacheWithSelf "dep" - ( - crateConfig.dependencies or [ ] - ++ lib.optionals - (runTests && packageId == rootPackageId) - (crateConfig.devDependencies or [ ]) - ); - cacheWithAll = - resolveDependencies - cacheWithDependencies "build" - (crateConfig.buildDependencies or [ ]); + cacheWithDependencies = resolveDependencies cacheWithSelf "dep" ( + crateConfig.dependencies or [ ] + ++ lib.optionals (runTests && packageId == rootPackageId) (crateConfig.devDependencies or [ ]) + ); + cacheWithAll = resolveDependencies cacheWithDependencies "build" ( + crateConfig.buildDependencies or [ ] + ); in cacheWithAll; - /* Returns the enabled dependencies given the enabled features. */ - filterEnabledDependencies = { dependencies, features, target }: - assert (builtins.isList dependencies); - assert (builtins.isList features); - assert (builtins.isAttrs target); + # Returns the enabled dependencies given the enabled features. + filterEnabledDependencies = + { dependencies + , features + , target + , + }: + assert (builtins.isList dependencies); + assert (builtins.isList features); + assert (builtins.isAttrs target); - lib.filter - ( - dep: - let - targetFunc = dep.target or (features: true); - in - targetFunc { inherit features target; } - && ( - !(dep.optional or false) - || builtins.any (doesFeatureEnableDependency dep) features + lib.filter + ( + dep: + let + targetFunc = dep.target or (features: true); + in + targetFunc { inherit features target; } + && (!(dep.optional or false) || builtins.any (doesFeatureEnableDependency dep) features) ) - ) - dependencies; + dependencies; - /* Returns whether the given feature should enable the given dependency. */ - doesFeatureEnableDependency = dependency: feature: + # Returns whether the given feature should enable the given dependency. + doesFeatureEnableDependency = + dependency: feature: let name = dependency.rename or dependency.name; prefix = "${name}/"; @@ -14225,109 +14333,116 @@ rec { in feature == name || feature == "dep:" + name || startsWithPrefix; - /* Returns the expanded features for the given inputFeatures by applying the + /* + Returns the expanded features for the given inputFeatures by applying the rules in featureMap. featureMap is an attribute set which maps feature names to lists of further feature names to enable in case this feature is selected. */ - expandFeatures = featureMap: inputFeatures: - assert (builtins.isAttrs featureMap); - assert (builtins.isList inputFeatures); - let - expandFeaturesNoCycle = oldSeen: inputFeatures: - if inputFeatures != [ ] - then - let - # The feature we're currently expanding. - feature = builtins.head inputFeatures; - # All the features we've seen/expanded so far, including the one - # we're currently processing. - seen = oldSeen // { ${feature} = 1; }; - # Expand the feature but be careful to not re-introduce a feature - # that we've already seen: this can easily cause a cycle, see issue - # #209. - enables = builtins.filter (f: !(seen ? "${f}")) (featureMap."${feature}" or [ ]); - in - [ feature ] ++ (expandFeaturesNoCycle seen (builtins.tail inputFeatures ++ enables)) - # No more features left, nothing to expand to. - else [ ]; - outFeatures = expandFeaturesNoCycle { } inputFeatures; - in - sortedUnique outFeatures; + expandFeatures = + featureMap: inputFeatures: + assert (builtins.isAttrs featureMap); + assert (builtins.isList inputFeatures); + let + expandFeaturesNoCycle = + oldSeen: inputFeatures: + if inputFeatures != [ ] then + let + # The feature we're currently expanding. + feature = builtins.head inputFeatures; + # All the features we've seen/expanded so far, including the one + # we're currently processing. + seen = oldSeen // { + ${feature} = 1; + }; + # Expand the feature but be careful to not re-introduce a feature + # that we've already seen: this can easily cause a cycle, see issue + # #209. + enables = builtins.filter (f: !(seen ? "${f}")) (featureMap."${feature}" or [ ]); + in + [ feature ] ++ (expandFeaturesNoCycle seen (builtins.tail inputFeatures ++ enables)) + # No more features left, nothing to expand to. + else + [ ]; + outFeatures = expandFeaturesNoCycle { } inputFeatures; + in + sortedUnique outFeatures; - /* This function adds optional dependencies as features if they are enabled + /* + This function adds optional dependencies as features if they are enabled indirectly by dependency features. This function mimics Cargo's behavior described in a note at: https://doc.rust-lang.org/nightly/cargo/reference/features.html#dependency-features */ - enableFeatures = dependencies: features: - assert (builtins.isList features); - assert (builtins.isList dependencies); - let - additionalFeatures = lib.concatMap - ( - dependency: - assert (builtins.isAttrs dependency); - let - enabled = builtins.any (doesFeatureEnableDependency dependency) features; - in - if (dependency.optional or false) && enabled - then [ (dependency.rename or dependency.name) ] - else [ ] - ) - dependencies; - in - sortedUnique (features ++ additionalFeatures); + enableFeatures = + dependencies: features: + assert (builtins.isList features); + assert (builtins.isList dependencies); + let + additionalFeatures = lib.concatMap + ( + dependency: + assert (builtins.isAttrs dependency); + let + enabled = builtins.any (doesFeatureEnableDependency dependency) features; + in + if (dependency.optional or false) && enabled then + [ (dependency.rename or dependency.name) ] + else + [ ] + ) + dependencies; + in + sortedUnique (features ++ additionalFeatures); /* Returns the actual features for the given dependency. features: The features of the crate that refers this dependency. */ - dependencyFeatures = features: dependency: - assert (builtins.isList features); - assert (builtins.isAttrs dependency); - let - defaultOrNil = - if dependency.usesDefaultFeatures or true - then [ "default" ] - else [ ]; - explicitFeatures = dependency.features or [ ]; - additionalDependencyFeatures = - let - name = dependency.rename or dependency.name; - stripPrefixMatch = prefix: s: - if lib.hasPrefix prefix s - then lib.removePrefix prefix s - else null; - extractFeature = feature: lib.findFirst - (f: f != null) - null - (map (prefix: stripPrefixMatch prefix feature) [ - (name + "/") - (name + "?/") - ]); - dependencyFeatures = lib.filter (f: f != null) (map extractFeature features); - in - dependencyFeatures; - in - defaultOrNil ++ explicitFeatures ++ additionalDependencyFeatures; - - /* Sorts and removes duplicates from a list of strings. */ - sortedUnique = features: - assert (builtins.isList features); - assert (builtins.all builtins.isString features); - let - outFeaturesSet = lib.foldl (set: feature: set // { "${feature}" = 1; }) { } features; - outFeaturesUnique = builtins.attrNames outFeaturesSet; - in - builtins.sort (a: b: a < b) outFeaturesUnique; + dependencyFeatures = + features: dependency: + assert (builtins.isList features); + assert (builtins.isAttrs dependency); + let + defaultOrNil = if dependency.usesDefaultFeatures or true then [ "default" ] else [ ]; + explicitFeatures = dependency.features or [ ]; + additionalDependencyFeatures = + let + name = dependency.rename or dependency.name; + stripPrefixMatch = prefix: s: if lib.hasPrefix prefix s then lib.removePrefix prefix s else null; + extractFeature = + feature: + lib.findFirst (f: f != null) null ( + map (prefix: stripPrefixMatch prefix feature) [ + (name + "/") + (name + "?/") + ] + ); + dependencyFeatures = lib.filter (f: f != null) (map extractFeature features); + in + dependencyFeatures; + in + defaultOrNil ++ explicitFeatures ++ additionalDependencyFeatures; - deprecationWarning = message: value: - if strictDeprecation - then builtins.throw "strictDeprecation enabled, aborting: ${message}" - else builtins.trace message value; + # Sorts and removes duplicates from a list of strings. + sortedUnique = + features: + assert (builtins.isList features); + assert (builtins.all builtins.isString features); + let + outFeaturesSet = lib.foldl (set: feature: set // { "${feature}" = 1; }) { } features; + outFeaturesUnique = builtins.attrNames outFeaturesSet; + in + builtins.sort (a: b: a < b) outFeaturesUnique; + + deprecationWarning = + message: value: + if strictDeprecation then + builtins.throw "strictDeprecation enabled, aborting: ${message}" + else + builtins.trace message value; # # crate2nix/default.nix (excerpt end) diff --git a/nix/README.md b/nix/README.md new file mode 100644 index 00000000..d3245031 --- /dev/null +++ b/nix/README.md @@ -0,0 +1,27 @@ + + +# Updating nix dependencies + +## Run the following for an operator + +> [!NOTE] +> We track the `master` branch of crate2nix as that is relatively up to date, but the releases are infrequent. + +```shell +niv update crate2nix +niv update nixpkgs +niv update beky.py -b X.Y.Z # Using the release tag +``` + +### Test + +- Run make `regenerate-nix` to ensure crate2nix works +- Run a smoke test to ensure beku.py works. +- Run `make run-dev` to ensure nixpkgs are fine. + +## Update operator-templating + +Do the same as above, but from `template/` diff --git a/nix/sources.json b/nix/sources.json index 9a26de8f..78d7121b 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -1,14 +1,14 @@ { "beku.py": { - "branch": "main", + "branch": "0.0.10", "description": "Test suite expander for Stackable Kuttl tests.", "homepage": null, "owner": "stackabletech", "repo": "beku.py", - "rev": "1ebc9e7b70fb8ee11dfb569ae45b3bcd63666d0e", - "sha256": "1zg24h5wdis7cysa08r8vvbw2rpyx6fgv148i1lg54dwd3sa0h0d", + "rev": "fc75202a38529a4ac6776dd8a5dfee278d927f58", + "sha256": "152yary0p11h87yabv74jnwkghsal7lx16az0qlzrzdrs6n5v8id", "type": "tarball", - "url": "https://github.com/stackabletech/beku.py/archive/1ebc9e7b70fb8ee11dfb569ae45b3bcd63666d0e.tar.gz", + "url": "https://github.com/stackabletech/beku.py/archive/fc75202a38529a4ac6776dd8a5dfee278d927f58.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "crate2nix": { @@ -17,10 +17,10 @@ "homepage": "", "owner": "kolloch", "repo": "crate2nix", - "rev": "236f6addfd452a48be805819e3216af79e988fd5", - "sha256": "1cnq84c1bhhbn3blm31scrqsxw2bl1w67v6gpav01m0s2509klf5", + "rev": "be31feae9a82c225c0fd1bdf978565dc452a483a", + "sha256": "14d0ymlrwk7dynv35qcw4xn0dylfpwjmf6f8znflbk2l6fk23l12", "type": "tarball", - "url": "https://github.com/kolloch/crate2nix/archive/236f6addfd452a48be805819e3216af79e988fd5.tar.gz", + "url": "https://github.com/kolloch/crate2nix/archive/be31feae9a82c225c0fd1bdf978565dc452a483a.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "nixpkgs": { @@ -29,10 +29,10 @@ "homepage": "", "owner": "NixOS", "repo": "nixpkgs", - "rev": "aa6ae0afa6adeb5c202a168e51eda1d3da571117", - "sha256": "1kbg6limdl7f21vr36g7qlrimm8lxr97b6kvxkz91yfdffn942p9", + "rev": "b0b4b5f8f621bfe213b8b21694bab52ecfcbf30b", + "sha256": "1y8kwbb5b0r1m88nk871ai56qi2drygvibjgc2swp48jfyp5ya99", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/aa6ae0afa6adeb5c202a168e51eda1d3da571117.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/b0b4b5f8f621bfe213b8b21694bab52ecfcbf30b.tar.gz", "url_template": "https://github.com///archive/.tar.gz" } } From dae97e34b6ff2a03c478a0a2c51f2daeeae20151 Mon Sep 17 00:00:00 2001 From: Nick <10092581+NickLarsenNZ@users.noreply.github.com> Date: Wed, 9 Apr 2025 12:10:36 +0200 Subject: [PATCH 08/15] fix: Use `json` file extension for log files (#647) * fix: Use `json` file extension for log files * chore: Update changelog --- CHANGELOG.md | 5 +++++ rust/operator-binary/src/main.rs | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 505ada54..cceb9f0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,8 +15,13 @@ - Fix a bug where changes to ConfigMaps that are referenced in the HbaseCluster spec didn't trigger a reconciliation ([#645]). +### Fixed + +- Use `json` file extension for log files ([#647]). + [#640]: https://github.com/stackabletech/hbase-operator/pull/640 [#645]: https://github.com/stackabletech/airflow-operator/pull/645 +[#647]: https://github.com/stackabletech/hbase-operator/pull/647 ## [25.3.0] - 2025-03-21 diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index ed40fbc8..fce1b615 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -91,7 +91,7 @@ async fn main() -> anyhow::Result<()> { // TODO (@Techassi): Change to CONSOLE_LOG or FILE_LOG, create constant .with_environment_variable(ENV_VAR_CONSOLE_LOG) .with_default_level(LevelFilter::INFO) - .file_log_settings_builder(log_directory, "tracing-rs.log") + .file_log_settings_builder(log_directory, "tracing-rs.json") .with_rotation_period(rotation_period) .build() })) From b2430ab6523ddd9157db4bc0531f7ed585a25dfc Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 9 Apr 2025 15:07:19 +0200 Subject: [PATCH 09/15] fix formatting in changelog --- CHANGELOG.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cceb9f0a..e50c19e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,11 +13,8 @@ ### Fixed -- Fix a bug where changes to ConfigMaps that are referenced in the HbaseCluster spec didn't trigger a reconciliation ([#645]). - -### Fixed - - Use `json` file extension for log files ([#647]). +- Fix a bug where changes to ConfigMaps that are referenced in the HbaseCluster spec didn't trigger a reconciliation ([#645]). [#640]: https://github.com/stackabletech/hbase-operator/pull/640 [#645]: https://github.com/stackabletech/airflow-operator/pull/645 From 6e02e3b7049cb444faf8aa5ffdfe65072fff11e4 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 9 Apr 2025 15:08:09 +0200 Subject: [PATCH 10/15] fix typo in changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e50c19e1..9352d342 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ - Fix a bug where changes to ConfigMaps that are referenced in the HbaseCluster spec didn't trigger a reconciliation ([#645]). [#640]: https://github.com/stackabletech/hbase-operator/pull/640 -[#645]: https://github.com/stackabletech/airflow-operator/pull/645 +[#645]: https://github.com/stackabletech/hbase-operator/pull/645 [#647]: https://github.com/stackabletech/hbase-operator/pull/647 ## [25.3.0] - 2025-03-21 From 5e08d1dc2916a9aad355a438d00ced3954ea85f6 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Wed, 9 Apr 2025 17:30:34 +0200 Subject: [PATCH 11/15] chore: Rename store variable --- rust/operator-binary/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index fce1b615..91581873 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -140,7 +140,7 @@ async fn main() -> anyhow::Result<()> { watch_namespace.get_api::>(&client), watcher::Config::default(), ); - let hbase_store_1 = hbase_controller.store(); + let config_map_store = hbase_controller.store(); hbase_controller .owns( watch_namespace.get_api::(&client), @@ -155,7 +155,7 @@ async fn main() -> anyhow::Result<()> { watch_namespace.get_api::>(&client), watcher::Config::default(), move |config_map| { - hbase_store_1 + config_map_store .state() .into_iter() .filter(move |hbase| references_config_map(hbase, &config_map)) From b56b0cf86337f41c8d78d4b2a5590fdfb4b42a07 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Wed, 9 Apr 2025 17:31:01 +0200 Subject: [PATCH 12/15] chore: Use borrows --- rust/operator-binary/src/hbase_controller.rs | 9 +++------ rust/operator-binary/src/main.rs | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index e837c359..376036c6 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -993,11 +993,8 @@ fn build_rolegroup_statefulset( // Vector sidecar shall be the last container in the list if merged_config.logging().enable_vector_agent { - if let Some(vector_aggregator_config_map_name) = hbase - .spec - .cluster_config - .vector_aggregator_config_map_name - .to_owned() + if let Some(vector_aggregator_config_map_name) = + &hbase.spec.cluster_config.vector_aggregator_config_map_name { pod_builder.add_container( product_logging::framework::vector_container( @@ -1011,7 +1008,7 @@ fn build_rolegroup_statefulset( .with_memory_request("128Mi") .with_memory_limit("128Mi") .build(), - &vector_aggregator_config_map_name, + vector_aggregator_config_map_name, ) .context(ConfigureLoggingSnafu)?, ); diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 91581873..f903d90f 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -203,7 +203,7 @@ fn references_config_map( hbase.spec.cluster_config.zookeeper_config_map_name == config_map.name_any() || hbase.spec.cluster_config.hdfs_config_map_name == config_map.name_any() - || match hbase.spec.cluster_config.authorization.to_owned() { + || match &hbase.spec.cluster_config.authorization { Some(hbase_authorization) => { hbase_authorization.opa.config_map_name == config_map.name_any() } From 37eefed7bc56c4c06f53f997fbade88b3c4772db Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Wed, 9 Apr 2025 17:31:42 +0200 Subject: [PATCH 13/15] chore: Formatting --- rust/operator-binary/src/main.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index f903d90f..d720ae74 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -128,13 +128,10 @@ async fn main() -> anyhow::Result<()> { ) .await?; - let event_recorder = Arc::new(Recorder::new( - client.as_kube_client(), - Reporter { - controller: FULL_HBASE_CONTROLLER_NAME.to_string(), - instance: None, - }, - )); + let event_recorder = Arc::new(Recorder::new(client.as_kube_client(), Reporter { + controller: FULL_HBASE_CONTROLLER_NAME.to_string(), + instance: None, + })); let hbase_controller = Controller::new( watch_namespace.get_api::>(&client), From f768d69588a56b3e98821e3fe70db6cf2db3c8a4 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Wed, 9 Apr 2025 17:33:32 +0200 Subject: [PATCH 14/15] chore: Formatting --- rust/operator-binary/src/crd/affinity.rs | 74 +++++++++++------------- 1 file changed, 34 insertions(+), 40 deletions(-) diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index cccf320e..3c079faf 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -186,45 +186,39 @@ mod tests { HbaseRole::RestServer => (), }; - assert_eq!( - affinity, - StackableAffinity { - pod_affinity: Some(PodAffinity { - preferred_during_scheduling_ignored_during_execution: Some(expected_affinities), - required_during_scheduling_ignored_during_execution: None, - }), - pod_anti_affinity: Some(PodAntiAffinity { - preferred_during_scheduling_ignored_during_execution: Some(vec![ - WeightedPodAffinityTerm { - pod_affinity_term: PodAffinityTerm { - label_selector: Some(LabelSelector { - match_expressions: None, - match_labels: Some(BTreeMap::from([ - ("app.kubernetes.io/name".to_string(), "hbase".to_string(),), - ( - "app.kubernetes.io/instance".to_string(), - "simple-hbase".to_string(), - ), - ( - "app.kubernetes.io/component".to_string(), - role.to_string(), - ) - ])) - }), - match_label_keys: None, - mismatch_label_keys: None, - namespace_selector: None, - namespaces: None, - topology_key: "kubernetes.io/hostname".to_string(), - }, - weight: 70 - } - ]), - required_during_scheduling_ignored_during_execution: None, - }), - node_affinity: None, - node_selector: None, - } - ); + assert_eq!(affinity, StackableAffinity { + pod_affinity: Some(PodAffinity { + preferred_during_scheduling_ignored_during_execution: Some(expected_affinities), + required_during_scheduling_ignored_during_execution: None, + }), + pod_anti_affinity: Some(PodAntiAffinity { + preferred_during_scheduling_ignored_during_execution: Some(vec![ + WeightedPodAffinityTerm { + pod_affinity_term: PodAffinityTerm { + label_selector: Some(LabelSelector { + match_expressions: None, + match_labels: Some(BTreeMap::from([ + ("app.kubernetes.io/name".to_string(), "hbase".to_string(),), + ( + "app.kubernetes.io/instance".to_string(), + "simple-hbase".to_string(), + ), + ("app.kubernetes.io/component".to_string(), role.to_string(),) + ])) + }), + match_label_keys: None, + mismatch_label_keys: None, + namespace_selector: None, + namespaces: None, + topology_key: "kubernetes.io/hostname".to_string(), + }, + weight: 70 + } + ]), + required_during_scheduling_ignored_during_execution: None, + }), + node_affinity: None, + node_selector: None, + }); } } From 24b642fa328c3c5a6f1315a1bf841503d23bee17 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Wed, 9 Apr 2025 18:21:43 +0200 Subject: [PATCH 15/15] noop