diff --git a/Cargo.lock b/Cargo.lock index ee5992e2..49b5c22c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -220,6 +220,12 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.8.0" @@ -368,6 +374,15 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -624,6 +639,15 @@ dependencies = [ "regex-syntax 0.8.5", ] +[[package]] +name = "fluent-uri" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17c704e9dbe1ddd863da1e6ff3567795087b1eb201ce80d8fa81162e1516500d" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "fnv" version = "1.0.7" @@ -780,7 +804,7 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b903b73e45dc0c6c596f2d37eccece7c1c8bb6e4407b001096387c63d0d93724" dependencies = [ - "bitflags", + "bitflags 2.8.0", "libc", "libgit2-sys", "log", @@ -793,6 +817,16 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + [[package]] name = "hashbrown" version = "0.15.2" @@ -1171,7 +1205,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.15.2", ] [[package]] @@ -1201,6 +1235,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.14" @@ -1237,18 +1280,45 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json-patch" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b1fb8864823fad91877e6caea0baca82e49e8db50f8e5c9f9a453e27d3330fc" +dependencies = [ + "jsonptr 0.4.7", + "serde", + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "json-patch" version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "863726d7afb6bc2590eeff7135d923545e5e964f004c2ccf8716c25e70a86f08" dependencies = [ - "jsonptr", + "jsonptr 0.6.3", "serde", "serde_json", "thiserror 1.0.69", ] +[[package]] +name = "jsonpath-rust" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d8fe85bd70ff715f31ce8c739194b423d79811a19602115d611a3ec85d6200" +dependencies = [ + "lazy_static", + "once_cell", + "pest", + "pest_derive", + "regex", + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "jsonpath-rust" version = "0.7.5" @@ -1262,6 +1332,17 @@ dependencies = [ "thiserror 2.0.11", ] +[[package]] +name = "jsonptr" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c6e529149475ca0b2820835d3dce8fcc41c6b943ca608d32f35b449255e4627" +dependencies = [ + "fluent-uri", + "serde", + "serde_json", +] + [[package]] name = "jsonptr" version = "0.6.3" @@ -1272,6 +1353,20 @@ dependencies = [ "serde_json", ] +[[package]] +name = "k8s-openapi" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8847402328d8301354c94d605481f25a6bdc1ed65471fd96af8eca71141b13" +dependencies = [ + "base64 0.22.1", + "chrono", + "schemars", + "serde", + "serde-value", + "serde_json", +] + [[package]] name = "k8s-openapi" version = "0.24.0" @@ -1286,17 +1381,78 @@ dependencies = [ "serde_json", ] +[[package]] +name = "k8s-version" +version = "0.1.2" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.5.0#048c7d8befddc2f2c6414444006871c95412d67c" +dependencies = [ + "darling", + "regex", + "snafu 0.8.5", +] + +[[package]] +name = "kube" +version = "0.96.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efffeb3df0bd4ef3e5d65044573499c0e4889b988070b08c50b25b1329289a1f" +dependencies = [ + "k8s-openapi 0.23.0", + "kube-client 0.96.0", + "kube-core 0.96.0", + "kube-derive 0.96.0", + "kube-runtime 0.96.0", +] + [[package]] name = "kube" version = "0.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32053dc495efad4d188c7b33cc7c02ef4a6e43038115348348876efd39a53cba" dependencies = [ - "k8s-openapi", - "kube-client", - "kube-core", - "kube-derive", - "kube-runtime", + "k8s-openapi 0.24.0", + "kube-client 0.98.0", + "kube-core 0.98.0", + "kube-derive 0.98.0", + "kube-runtime 0.98.0", +] + +[[package]] +name = "kube-client" +version = "0.96.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf471ece8ff8d24735ce78dac4d091e9fcb8d74811aeb6b75de4d1c3f5de0f1" +dependencies = [ + "base64 0.22.1", + "bytes", + "chrono", + "either", + "futures 0.3.31", + "home", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-http-proxy", + "hyper-rustls", + "hyper-timeout", + "hyper-util", + "jsonpath-rust 0.5.1", + "k8s-openapi 0.23.0", + "kube-core 0.96.0", + "pem", + "rustls", + "rustls-pemfile", + "secrecy", + "serde", + "serde_json", + "serde_yaml", + "thiserror 1.0.69", + "tokio", + "tokio-util", + "tower", + "tower-http", + "tracing", ] [[package]] @@ -1319,9 +1475,9 @@ dependencies = [ "hyper-rustls", "hyper-timeout", "hyper-util", - "jsonpath-rust", - "k8s-openapi", - "kube-core", + "jsonpath-rust 0.7.5", + "k8s-openapi 0.24.0", + "kube-core 0.98.0", "pem", "rustls", "rustls-pemfile", @@ -1337,6 +1493,24 @@ dependencies = [ "tracing", ] +[[package]] +name = "kube-core" +version = "0.96.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f42346d30bb34d1d7adc5c549b691bce7aa3a1e60254e68fab7e2d7b26fe3d77" +dependencies = [ + "chrono", + "form_urlencoded", + "http", + "json-patch 2.0.0", + "k8s-openapi 0.23.0", + "schemars", + "serde", + "serde-value", + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "kube-core" version = "0.98.0" @@ -1346,8 +1520,8 @@ dependencies = [ "chrono", "form_urlencoded", "http", - "json-patch", - "k8s-openapi", + "json-patch 3.0.1", + "k8s-openapi 0.24.0", "schemars", "serde", "serde-value", @@ -1355,6 +1529,19 @@ dependencies = [ "thiserror 2.0.11", ] +[[package]] +name = "kube-derive" +version = "0.96.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9364e04cc5e0482136c6ee8b7fb7551812da25802249f35b3def7aaa31e82ad" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "serde_json", + "syn 2.0.96", +] + [[package]] name = "kube-derive" version = "0.98.0" @@ -1368,6 +1555,34 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "kube-runtime" +version = "0.96.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fbf1f6ffa98e65f1d2a9a69338bb60605d46be7edf00237784b89e62c9bd44" +dependencies = [ + "ahash", + "async-broadcast", + "async-stream", + "async-trait", + "backoff", + "educe", + "futures 0.3.31", + "hashbrown 0.14.5", + "json-patch 2.0.0", + "jsonptr 0.4.7", + "k8s-openapi 0.23.0", + "kube-client 0.96.0", + "parking_lot", + "pin-project", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "kube-runtime" version = "0.98.0" @@ -1381,12 +1596,12 @@ dependencies = [ "backoff", "educe", "futures 0.3.31", - "hashbrown", + "hashbrown 0.15.2", "hostname", - "json-patch", - "jsonptr", - "k8s-openapi", - "kube-client", + "json-patch 3.0.1", + "jsonptr 0.6.3", + "k8s-openapi 0.24.0", + "kube-client 0.98.0", "parking_lot", "pin-project", "serde", @@ -1856,7 +2071,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags", + "bitflags 2.8.0", ] [[package]] @@ -2102,7 +2317,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags", + "bitflags 2.8.0", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -2115,7 +2330,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ - "bitflags", + "bitflags 2.8.0", "core-foundation 0.10.0", "core-foundation-sys", "libc", @@ -2329,23 +2544,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" -[[package]] -name = "stackable-airflow-crd" -version = "0.0.0-dev" -dependencies = [ - "indoc", - "product-config", - "rstest", - "serde", - "serde_json", - "serde_yaml", - "snafu 0.8.5", - "stackable-operator", - "strum", - "tokio", - "tracing", -] - [[package]] name = "stackable-airflow-operator" version = "0.0.0-dev" @@ -2360,10 +2558,11 @@ dependencies = [ "product-config", "rstest", "serde", + "serde_json", "serde_yaml", "snafu 0.8.5", - "stackable-airflow-crd", "stackable-operator", + "stackable-versioned", "strum", "tokio", "tracing", @@ -2383,9 +2582,9 @@ dependencies = [ "either", "futures 0.3.31", "indexmap", - "json-patch", - "k8s-openapi", - "kube", + "json-patch 3.0.1", + "k8s-openapi 0.24.0", + "kube 0.98.0", "opentelemetry-jaeger", "opentelemetry_sdk", "product-config", @@ -2423,13 +2622,37 @@ name = "stackable-shared" version = "0.0.1" source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#59506c6202778889a27b6ae8153457e60a49c68d" dependencies = [ - "kube", + "kube 0.98.0", "semver", "serde", "serde_yaml", "snafu 0.8.5", ] +[[package]] +name = "stackable-versioned" +version = "0.5.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.5.0#048c7d8befddc2f2c6414444006871c95412d67c" +dependencies = [ + "stackable-versioned-macros", +] + +[[package]] +name = "stackable-versioned-macros" +version = "0.5.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.5.0#048c7d8befddc2f2c6414444006871c95412d67c" +dependencies = [ + "convert_case", + "darling", + "itertools", + "k8s-openapi 0.23.0", + "k8s-version", + "kube 0.96.0", + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "strsim" version = "0.11.1" @@ -2721,7 +2944,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" dependencies = [ "base64 0.22.1", - "bitflags", + "bitflags 2.8.0", "bytes", "http", "http-body", @@ -2873,6 +3096,12 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + [[package]] name = "unicode-xid" version = "0.2.6" diff --git a/Cargo.nix b/Cargo.nix index 691d50bf..20ff9d07 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -32,21 +32,23 @@ rec { # "public" attributes that we attempt to keep stable with new versions of crate2nix. # + rootCrate = rec { + packageId = "stackable-airflow-operator"; + # Use this attribute to refer to the derivation building your root crate package. + # You can override the features with rootCrate.build.override { features = [ "default" "feature1" ... ]; }. + build = internal.buildRustCrateWithFeatures { + inherit packageId; + }; + + # Debug support which might change between releases. + # File a bug if you depend on any for non-debug work! + debug = internal.debugCrate { inherit packageId; }; + }; # Refer your crate build derivation by name here. # You can override the features with # workspaceMembers."${crateName}".build.override { features = [ "default" "feature1" ... ]; }. workspaceMembers = { - "stackable-airflow-crd" = rec { - packageId = "stackable-airflow-crd"; - build = internal.buildRustCrateWithFeatures { - packageId = "stackable-airflow-crd"; - }; - - # Debug support which might change between releases. - # File a bug if you depend on any for non-debug work! - debug = internal.debugCrate { inherit packageId; }; - }; "stackable-airflow-operator" = rec { packageId = "stackable-airflow-operator"; build = internal.buildRustCrateWithFeatures { @@ -655,7 +657,22 @@ rec { }; resolvedDefaultFeatures = [ "std" ]; }; - "bitflags" = rec { + "bitflags 1.3.2" = rec { + crateName = "bitflags"; + version = "1.3.2"; + edition = "2018"; + sha256 = "12ki6w8gn1ldq7yz9y680llwk5gmrhrzszaa17g1sbrw2r2qvwxy"; + authors = [ + "The Rust Project Developers" + ]; + features = { + "compiler_builtins" = [ "dep:compiler_builtins" ]; + "core" = [ "dep:core" ]; + "rustc-dep-of-std" = [ "core" "compiler_builtins" ]; + }; + resolvedDefaultFeatures = [ "default" ]; + }; + "bitflags 2.8.0" = rec { crateName = "bitflags"; version = "2.8.0"; edition = "2021"; @@ -1086,6 +1103,25 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; + "convert_case" = rec { + crateName = "convert_case"; + version = "0.6.0"; + edition = "2018"; + sha256 = "1jn1pq6fp3rri88zyw6jlhwwgf6qiyc08d6gjv0qypgkl862n67c"; + authors = [ + "Rutrum " + ]; + dependencies = [ + { + name = "unicode-segmentation"; + packageId = "unicode-segmentation"; + } + ]; + features = { + "rand" = [ "dep:rand" ]; + "random" = [ "rand" ]; + }; + }; "core-foundation 0.10.0" = rec { crateName = "core-foundation"; version = "0.10.0"; @@ -1767,6 +1803,26 @@ rec { }; resolvedDefaultFeatures = [ "default" "perf" "std" "unicode" ]; }; + "fluent-uri" = rec { + crateName = "fluent-uri"; + version = "0.1.4"; + edition = "2021"; + sha256 = "03ah2qajw5l1zbc81kh1n8g7n24mfxbg6vqyv9ixipg1vglh9iqp"; + libName = "fluent_uri"; + authors = [ + "Scallop Ye " + ]; + dependencies = [ + { + name = "bitflags"; + packageId = "bitflags 1.3.2"; + } + ]; + features = { + "default" = [ "std" ]; + }; + resolvedDefaultFeatures = [ "std" ]; + }; "fnv" = rec { crateName = "fnv"; version = "1.0.7"; @@ -2220,7 +2276,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags"; + packageId = "bitflags 2.8.0"; } { name = "libc"; @@ -2261,7 +2317,46 @@ rec { ]; }; - "hashbrown" = rec { + "hashbrown 0.14.5" = rec { + crateName = "hashbrown"; + version = "0.14.5"; + edition = "2021"; + sha256 = "1wa1vy1xs3mp11bn3z9dv0jricgr6a2j0zkf1g19yz3vw4il89z5"; + authors = [ + "Amanieu d'Antras " + ]; + dependencies = [ + { + name = "ahash"; + packageId = "ahash"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "allocator-api2"; + packageId = "allocator-api2"; + optional = true; + usesDefaultFeatures = false; + features = [ "alloc" ]; + } + ]; + features = { + "ahash" = [ "dep:ahash" ]; + "alloc" = [ "dep:alloc" ]; + "allocator-api2" = [ "dep:allocator-api2" ]; + "compiler_builtins" = [ "dep:compiler_builtins" ]; + "core" = [ "dep:core" ]; + "default" = [ "ahash" "inline-more" "allocator-api2" ]; + "equivalent" = [ "dep:equivalent" ]; + "nightly" = [ "allocator-api2?/nightly" "bumpalo/allocator_api" ]; + "rayon" = [ "dep:rayon" ]; + "rkyv" = [ "dep:rkyv" ]; + "rustc-dep-of-std" = [ "nightly" "core" "compiler_builtins" "alloc" "rustc-internal-api" ]; + "serde" = [ "dep:serde" ]; + }; + resolvedDefaultFeatures = [ "ahash" "allocator-api2" "default" "inline-more" ]; + }; + "hashbrown 0.15.2" = rec { crateName = "hashbrown"; version = "0.15.2"; edition = "2021"; @@ -3531,7 +3626,7 @@ rec { } { name = "hashbrown"; - packageId = "hashbrown"; + packageId = "hashbrown 0.15.2"; usesDefaultFeatures = false; } ]; @@ -3605,6 +3700,27 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; + "itertools" = rec { + crateName = "itertools"; + version = "0.13.0"; + edition = "2018"; + sha256 = "11hiy3qzl643zcigknclh446qb9zlg4dpdzfkjaa9q9fqpgyfgj1"; + authors = [ + "bluss" + ]; + dependencies = [ + { + name = "either"; + packageId = "either"; + usesDefaultFeatures = false; + } + ]; + features = { + "default" = [ "use_std" ]; + "use_std" = [ "use_alloc" "either/use_std" ]; + }; + resolvedDefaultFeatures = [ "default" "use_alloc" "use_std" ]; + }; "itoa" = rec { crateName = "itoa"; version = "1.0.14"; @@ -3686,7 +3802,48 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; - "json-patch" = rec { + "json-patch 2.0.0" = rec { + crateName = "json-patch"; + version = "2.0.0"; + edition = "2021"; + sha256 = "1z1h6dyy4lx4z74yby2hvgl4jbm8mh5ymjp6fwcdkyi3923bh7sv"; + libName = "json_patch"; + authors = [ + "Ivan Dubrov " + ]; + dependencies = [ + { + name = "jsonptr"; + packageId = "jsonptr 0.4.7"; + } + { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "thiserror"; + packageId = "thiserror 1.0.69"; + } + ]; + devDependencies = [ + { + name = "serde_json"; + packageId = "serde_json"; + features = [ "preserve_order" ]; + } + ]; + features = { + "default" = [ "diff" ]; + "utoipa" = [ "dep:utoipa" ]; + }; + resolvedDefaultFeatures = [ "default" "diff" ]; + }; + "json-patch 3.0.1" = rec { crateName = "json-patch"; version = "3.0.1"; edition = "2021"; @@ -3698,7 +3855,7 @@ rec { dependencies = [ { name = "jsonptr"; - packageId = "jsonptr"; + packageId = "jsonptr 0.6.3"; } { name = "serde"; @@ -3727,7 +3884,48 @@ rec { }; resolvedDefaultFeatures = [ "default" "diff" ]; }; - "jsonpath-rust" = rec { + "jsonpath-rust 0.5.1" = rec { + crateName = "jsonpath-rust"; + version = "0.5.1"; + edition = "2018"; + sha256 = "0032bp43w6k1bl8h55m126cdf8xljj8p736f65gp3zvhpn2zxn0r"; + libName = "jsonpath_rust"; + authors = [ + "BorisZhguchev " + ]; + dependencies = [ + { + name = "lazy_static"; + packageId = "lazy_static"; + } + { + name = "once_cell"; + packageId = "once_cell"; + } + { + name = "pest"; + packageId = "pest"; + } + { + name = "pest_derive"; + packageId = "pest_derive"; + } + { + name = "regex"; + packageId = "regex"; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "thiserror"; + packageId = "thiserror 1.0.69"; + } + ]; + + }; + "jsonpath-rust 0.7.5" = rec { crateName = "jsonpath-rust"; version = "0.7.5"; edition = "2021"; @@ -3760,7 +3958,44 @@ rec { ]; }; - "jsonptr" = rec { + "jsonptr 0.4.7" = rec { + crateName = "jsonptr"; + version = "0.4.7"; + edition = "2021"; + sha256 = "09s6bqjlkd1m5z9hi9iwjimiri7wx3fd6d88hara0p27968m4vhw"; + authors = [ + "chance dinkins" + ]; + dependencies = [ + { + name = "fluent-uri"; + packageId = "fluent-uri"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "serde"; + packageId = "serde"; + usesDefaultFeatures = false; + features = [ "alloc" ]; + } + { + name = "serde_json"; + packageId = "serde_json"; + usesDefaultFeatures = false; + features = [ "alloc" ]; + } + ]; + features = { + "default" = [ "std" ]; + "fluent-uri" = [ "dep:fluent-uri" ]; + "std" = [ "serde/std" "serde_json/std" "fluent-uri?/std" ]; + "uniresid" = [ "dep:uniresid" ]; + "url" = [ "dep:url" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; + "jsonptr 0.6.3" = rec { crateName = "jsonptr"; version = "0.6.3"; edition = "2021"; @@ -3794,12 +4029,12 @@ rec { }; resolvedDefaultFeatures = [ "assign" "default" "delete" "json" "resolve" "serde" "std" ]; }; - "k8s-openapi" = rec { + "k8s-openapi 0.23.0" = rec { crateName = "k8s-openapi"; - version = "0.24.0"; + version = "0.23.0"; edition = "2021"; - links = "k8s-openapi-0.24.0"; - sha256 = "1m8ahw59g44kp9p4yd4ar0px15m2nyvhc5krbvqvw2ag6a8bjx9c"; + links = "k8s-openapi-0.23.0"; + sha256 = "04qv2iqwm3mgjvyp2m6n3vf6nnpjh5a60kf9ah9k1n184d04g24w"; libName = "k8s_openapi"; authors = [ "Arnav Singh " @@ -3841,62 +4076,149 @@ rec { } ]; features = { - "earliest" = [ "v1_28" ]; - "latest" = [ "v1_32" ]; + "earliest" = [ "v1_26" ]; + "latest" = [ "v1_31" ]; "schemars" = [ "dep:schemars" ]; }; - resolvedDefaultFeatures = [ "schemars" "v1_32" ]; + resolvedDefaultFeatures = [ "schemars" "v1_31" ]; }; - "kube" = rec { - crateName = "kube"; - version = "0.98.0"; + "k8s-openapi 0.24.0" = rec { + crateName = "k8s-openapi"; + version = "0.24.0"; edition = "2021"; - sha256 = "1fiwllwzsvl7921k85c10d1nwjpg09ycqcvvihc4vbggjp23s19j"; + links = "k8s-openapi-0.24.0"; + sha256 = "1m8ahw59g44kp9p4yd4ar0px15m2nyvhc5krbvqvw2ag6a8bjx9c"; + libName = "k8s_openapi"; authors = [ - "clux " - "Natalie Klestrup Röijezon " - "kazk " + "Arnav Singh " ]; dependencies = [ { - name = "k8s-openapi"; - packageId = "k8s-openapi"; + name = "base64"; + packageId = "base64 0.22.1"; usesDefaultFeatures = false; + features = [ "alloc" ]; } { - name = "kube-client"; - packageId = "kube-client"; - optional = true; + name = "chrono"; + packageId = "chrono"; usesDefaultFeatures = false; + features = [ "alloc" "serde" ]; } { - name = "kube-core"; - packageId = "kube-core"; + name = "schemars"; + packageId = "schemars"; + optional = true; + usesDefaultFeatures = false; } { - name = "kube-derive"; - packageId = "kube-derive"; - optional = true; + name = "serde"; + packageId = "serde"; + usesDefaultFeatures = false; } { - name = "kube-runtime"; - packageId = "kube-runtime"; - optional = true; + name = "serde-value"; + packageId = "serde-value"; + usesDefaultFeatures = false; } - ]; - devDependencies = [ { - name = "k8s-openapi"; - packageId = "k8s-openapi"; + name = "serde_json"; + packageId = "serde_json"; usesDefaultFeatures = false; - features = [ "latest" ]; + features = [ "alloc" ]; } ]; features = { - "admission" = [ "kube-core/admission" ]; - "aws-lc-rs" = [ "kube-client?/aws-lc-rs" ]; - "client" = [ "kube-client/client" "config" ]; - "config" = [ "kube-client/config" ]; + "earliest" = [ "v1_28" ]; + "latest" = [ "v1_32" ]; + "schemars" = [ "dep:schemars" ]; + }; + resolvedDefaultFeatures = [ "schemars" "v1_32" ]; + }; + "k8s-version" = rec { + crateName = "k8s-version"; + version = "0.1.2"; + edition = "2021"; + workspace_member = null; + src = pkgs.fetchgit { + url = "https://github.com/stackabletech/operator-rs.git"; + rev = "048c7d8befddc2f2c6414444006871c95412d67c"; + sha256 = "1x2pfibrsysmkkmajyj30qkwsjf3rzmc3dxsd09jb9r4x7va6mr6"; + }; + libName = "k8s_version"; + authors = [ + "Stackable GmbH " + ]; + dependencies = [ + { + name = "darling"; + packageId = "darling"; + optional = true; + } + { + name = "regex"; + packageId = "regex"; + } + { + name = "snafu"; + packageId = "snafu 0.8.5"; + } + ]; + features = { + "darling" = [ "dep:darling" ]; + }; + resolvedDefaultFeatures = [ "darling" ]; + }; + "kube 0.96.0" = rec { + crateName = "kube"; + version = "0.96.0"; + edition = "2021"; + sha256 = "07ws50li6nxja26b0w40k2dqir60k4s5fi2hsvjz6kmxy0yypzzg"; + authors = [ + "clux " + "Natalie Klestrup Röijezon " + "kazk " + ]; + dependencies = [ + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.23.0"; + usesDefaultFeatures = false; + } + { + name = "kube-client"; + packageId = "kube-client 0.96.0"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "kube-core"; + packageId = "kube-core 0.96.0"; + } + { + name = "kube-derive"; + packageId = "kube-derive 0.96.0"; + optional = true; + } + { + name = "kube-runtime"; + packageId = "kube-runtime 0.96.0"; + optional = true; + } + ]; + devDependencies = [ + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.23.0"; + usesDefaultFeatures = false; + features = [ "latest" ]; + } + ]; + features = { + "admission" = [ "kube-core/admission" ]; + "aws-lc-rs" = [ "kube-client?/aws-lc-rs" ]; + "client" = [ "kube-client/client" "config" ]; + "config" = [ "kube-client/config" ]; "default" = [ "client" "rustls-tls" ]; "derive" = [ "kube-derive" "kube-core/schema" ]; "gzip" = [ "kube-client/gzip" "client" ]; @@ -3919,11 +4241,83 @@ rec { }; resolvedDefaultFeatures = [ "client" "config" "derive" "jsonpatch" "kube-client" "kube-derive" "kube-runtime" "runtime" "rustls-tls" ]; }; - "kube-client" = rec { - crateName = "kube-client"; + "kube 0.98.0" = rec { + crateName = "kube"; version = "0.98.0"; edition = "2021"; - sha256 = "1jd06xwhnmzrzqrfwq7jlmmxl7dvaygmchjx363zmlgvrlwasd4x"; + sha256 = "1fiwllwzsvl7921k85c10d1nwjpg09ycqcvvihc4vbggjp23s19j"; + authors = [ + "clux " + "Natalie Klestrup Röijezon " + "kazk " + ]; + dependencies = [ + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.24.0"; + usesDefaultFeatures = false; + } + { + name = "kube-client"; + packageId = "kube-client 0.98.0"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "kube-core"; + packageId = "kube-core 0.98.0"; + } + { + name = "kube-derive"; + packageId = "kube-derive 0.98.0"; + optional = true; + } + { + name = "kube-runtime"; + packageId = "kube-runtime 0.98.0"; + optional = true; + } + ]; + devDependencies = [ + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.24.0"; + usesDefaultFeatures = false; + features = [ "latest" ]; + } + ]; + features = { + "admission" = [ "kube-core/admission" ]; + "aws-lc-rs" = [ "kube-client?/aws-lc-rs" ]; + "client" = [ "kube-client/client" "config" ]; + "config" = [ "kube-client/config" ]; + "default" = [ "client" "rustls-tls" ]; + "derive" = [ "kube-derive" "kube-core/schema" ]; + "gzip" = [ "kube-client/gzip" "client" ]; + "http-proxy" = [ "kube-client/http-proxy" "client" ]; + "jsonpatch" = [ "kube-core/jsonpatch" ]; + "kube-client" = [ "dep:kube-client" ]; + "kube-derive" = [ "dep:kube-derive" ]; + "kube-runtime" = [ "dep:kube-runtime" ]; + "kubelet-debug" = [ "kube-client/kubelet-debug" "kube-core/kubelet-debug" ]; + "oauth" = [ "kube-client/oauth" "client" ]; + "oidc" = [ "kube-client/oidc" "client" ]; + "openssl-tls" = [ "kube-client/openssl-tls" "client" ]; + "runtime" = [ "kube-runtime" ]; + "rustls-tls" = [ "kube-client/rustls-tls" "client" ]; + "socks5" = [ "kube-client/socks5" "client" ]; + "unstable-client" = [ "kube-client/unstable-client" "client" ]; + "unstable-runtime" = [ "kube-runtime/unstable-runtime" "runtime" ]; + "webpki-roots" = [ "kube-client/webpki-roots" "client" ]; + "ws" = [ "kube-client/ws" "kube-core/ws" ]; + }; + resolvedDefaultFeatures = [ "client" "config" "derive" "jsonpatch" "kube-client" "kube-derive" "kube-runtime" "runtime" "rustls-tls" ]; + }; + "kube-client 0.96.0" = rec { + crateName = "kube-client"; + version = "0.96.0"; + edition = "2021"; + sha256 = "1wg0blziqkfyfmmyn6l1fj6wp7qy156sr3g7birj93gzx3n73x4b"; libName = "kube_client"; authors = [ "clux " @@ -4010,17 +4404,17 @@ rec { } { name = "jsonpath-rust"; - packageId = "jsonpath-rust"; + packageId = "jsonpath-rust 0.5.1"; optional = true; } { name = "k8s-openapi"; - packageId = "k8s-openapi"; + packageId = "k8s-openapi 0.23.0"; usesDefaultFeatures = false; } { name = "kube-core"; - packageId = "kube-core"; + packageId = "kube-core 0.96.0"; } { name = "pem"; @@ -4058,7 +4452,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.11"; + packageId = "thiserror 1.0.69"; } { name = "tokio"; @@ -4098,14 +4492,9 @@ rec { usesDefaultFeatures = false; features = [ "async-await" ]; } - { - name = "hyper"; - packageId = "hyper"; - features = [ "server" ]; - } { name = "k8s-openapi"; - packageId = "k8s-openapi"; + packageId = "k8s-openapi 0.23.0"; usesDefaultFeatures = false; features = [ "latest" ]; } @@ -4166,12 +4555,12 @@ rec { }; resolvedDefaultFeatures = [ "__non_core" "base64" "bytes" "chrono" "client" "config" "either" "futures" "home" "http-body" "http-body-util" "hyper" "hyper-rustls" "hyper-timeout" "hyper-util" "jsonpatch" "jsonpath-rust" "pem" "rustls" "rustls-pemfile" "rustls-tls" "serde_yaml" "tokio" "tokio-util" "tower" "tower-http" "tracing" ]; }; - "kube-core" = rec { - crateName = "kube-core"; + "kube-client 0.98.0" = rec { + crateName = "kube-client"; version = "0.98.0"; edition = "2021"; - sha256 = "1wwnsn1wk7bd2jiv9iw8446j0bczagqv1lc4wy88l5wa505q7alp"; - libName = "kube_core"; + sha256 = "1jd06xwhnmzrzqrfwq7jlmmxl7dvaygmchjx363zmlgvrlwasd4x"; + libName = "kube_client"; authors = [ "clux " "Natalie Klestrup Röijezon " @@ -4179,42 +4568,546 @@ rec { ]; dependencies = [ { - name = "chrono"; - packageId = "chrono"; + name = "base64"; + packageId = "base64 0.22.1"; + optional = true; + } + { + name = "bytes"; + packageId = "bytes"; + optional = true; + } + { + name = "chrono"; + packageId = "chrono"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "either"; + packageId = "either"; + optional = true; + } + { + name = "futures"; + packageId = "futures 0.3.31"; + optional = true; + usesDefaultFeatures = false; + features = [ "std" ]; + } + { + name = "home"; + packageId = "home"; + optional = true; + } + { + name = "http"; + packageId = "http"; + } + { + name = "http-body"; + packageId = "http-body"; + optional = true; + } + { + name = "http-body-util"; + packageId = "http-body-util"; + optional = true; + } + { + name = "hyper"; + packageId = "hyper"; + optional = true; + features = [ "client" "http1" ]; + } + { + name = "hyper-http-proxy"; + packageId = "hyper-http-proxy"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "hyper-rustls"; + packageId = "hyper-rustls"; + optional = true; + usesDefaultFeatures = false; + features = [ "http1" "logging" "native-tokio" "ring" "tls12" ]; + } + { + name = "hyper-timeout"; + packageId = "hyper-timeout"; + optional = true; + } + { + name = "hyper-util"; + packageId = "hyper-util"; + optional = true; + features = [ "client" "client-legacy" "http1" "tokio" ]; + } + { + name = "jsonpath-rust"; + packageId = "jsonpath-rust 0.7.5"; + optional = true; + } + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.24.0"; + usesDefaultFeatures = false; + } + { + name = "kube-core"; + packageId = "kube-core 0.98.0"; + } + { + name = "pem"; + packageId = "pem"; + optional = true; + } + { + name = "rustls"; + packageId = "rustls"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "rustls-pemfile"; + packageId = "rustls-pemfile"; + optional = true; + } + { + name = "secrecy"; + packageId = "secrecy"; + } + { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "serde_yaml"; + packageId = "serde_yaml"; + optional = true; + } + { + name = "thiserror"; + packageId = "thiserror 2.0.11"; + } + { + name = "tokio"; + packageId = "tokio"; + optional = true; + features = [ "time" "signal" "sync" ]; + } + { + name = "tokio-util"; + packageId = "tokio-util"; + optional = true; + features = [ "io" "codec" ]; + } + { + name = "tower"; + packageId = "tower"; + optional = true; + features = [ "buffer" "filter" "util" ]; + } + { + name = "tower-http"; + packageId = "tower-http"; + optional = true; + features = [ "auth" "map-response-body" "trace" ]; + } + { + name = "tracing"; + packageId = "tracing"; + optional = true; + features = [ "log" ]; + } + ]; + devDependencies = [ + { + name = "futures"; + packageId = "futures 0.3.31"; + usesDefaultFeatures = false; + features = [ "async-await" ]; + } + { + name = "hyper"; + packageId = "hyper"; + features = [ "server" ]; + } + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.24.0"; + usesDefaultFeatures = false; + features = [ "latest" ]; + } + { + name = "tokio"; + packageId = "tokio"; + features = [ "full" ]; + } + ]; + features = { + "__non_core" = [ "tracing" "serde_yaml" "base64" ]; + "admission" = [ "kube-core/admission" ]; + "aws-lc-rs" = [ "rustls?/aws-lc-rs" ]; + "base64" = [ "dep:base64" ]; + "bytes" = [ "dep:bytes" ]; + "chrono" = [ "dep:chrono" ]; + "client" = [ "config" "__non_core" "hyper" "hyper-util" "http-body" "http-body-util" "tower" "tower-http" "hyper-timeout" "chrono" "jsonpath-rust" "bytes" "futures" "tokio" "tokio-util" "either" ]; + "config" = [ "__non_core" "pem" "home" ]; + "default" = [ "client" ]; + "either" = [ "dep:either" ]; + "form_urlencoded" = [ "dep:form_urlencoded" ]; + "futures" = [ "dep:futures" ]; + "gzip" = [ "client" "tower-http/decompression-gzip" ]; + "home" = [ "dep:home" ]; + "http-body" = [ "dep:http-body" ]; + "http-body-util" = [ "dep:http-body-util" ]; + "http-proxy" = [ "hyper-http-proxy" ]; + "hyper" = [ "dep:hyper" ]; + "hyper-http-proxy" = [ "dep:hyper-http-proxy" ]; + "hyper-openssl" = [ "dep:hyper-openssl" ]; + "hyper-rustls" = [ "dep:hyper-rustls" ]; + "hyper-socks2" = [ "dep:hyper-socks2" ]; + "hyper-timeout" = [ "dep:hyper-timeout" ]; + "hyper-util" = [ "dep:hyper-util" ]; + "jsonpatch" = [ "kube-core/jsonpatch" ]; + "jsonpath-rust" = [ "dep:jsonpath-rust" ]; + "kubelet-debug" = [ "ws" "kube-core/kubelet-debug" ]; + "oauth" = [ "client" "tame-oauth" ]; + "oidc" = [ "client" "form_urlencoded" ]; + "openssl" = [ "dep:openssl" ]; + "openssl-tls" = [ "openssl" "hyper-openssl" ]; + "pem" = [ "dep:pem" ]; + "rand" = [ "dep:rand" ]; + "rustls" = [ "dep:rustls" ]; + "rustls-pemfile" = [ "dep:rustls-pemfile" ]; + "rustls-tls" = [ "rustls" "rustls-pemfile" "hyper-rustls" "hyper-http-proxy?/rustls-tls-native-roots" ]; + "serde_yaml" = [ "dep:serde_yaml" ]; + "socks5" = [ "hyper-socks2" ]; + "tame-oauth" = [ "dep:tame-oauth" ]; + "tokio" = [ "dep:tokio" ]; + "tokio-tungstenite" = [ "dep:tokio-tungstenite" ]; + "tokio-util" = [ "dep:tokio-util" ]; + "tower" = [ "dep:tower" ]; + "tower-http" = [ "dep:tower-http" ]; + "tracing" = [ "dep:tracing" ]; + "webpki-roots" = [ "hyper-rustls/webpki-roots" ]; + "ws" = [ "client" "tokio-tungstenite" "rand" "kube-core/ws" "tokio/macros" ]; + }; + resolvedDefaultFeatures = [ "__non_core" "base64" "bytes" "chrono" "client" "config" "either" "futures" "home" "http-body" "http-body-util" "hyper" "hyper-rustls" "hyper-timeout" "hyper-util" "jsonpatch" "jsonpath-rust" "pem" "rustls" "rustls-pemfile" "rustls-tls" "serde_yaml" "tokio" "tokio-util" "tower" "tower-http" "tracing" ]; + }; + "kube-core 0.96.0" = rec { + crateName = "kube-core"; + version = "0.96.0"; + edition = "2021"; + sha256 = "0xrxzqk7nbbymf7ycm02wshs6ynf3dlrnm2wvix1skdk1g9lc8zl"; + libName = "kube_core"; + authors = [ + "clux " + "Natalie Klestrup Röijezon " + "kazk " + ]; + dependencies = [ + { + name = "chrono"; + packageId = "chrono"; + usesDefaultFeatures = false; + features = [ "now" ]; + } + { + name = "form_urlencoded"; + packageId = "form_urlencoded"; + } + { + name = "http"; + packageId = "http"; + } + { + name = "json-patch"; + packageId = "json-patch 2.0.0"; + optional = true; + } + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.23.0"; + usesDefaultFeatures = false; + } + { + name = "schemars"; + packageId = "schemars"; + optional = true; + } + { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + { + name = "serde-value"; + packageId = "serde-value"; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "thiserror"; + packageId = "thiserror 1.0.69"; + } + ]; + devDependencies = [ + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.23.0"; + usesDefaultFeatures = false; + features = [ "latest" ]; + } + ]; + features = { + "admission" = [ "json-patch" ]; + "json-patch" = [ "dep:json-patch" ]; + "jsonpatch" = [ "json-patch" ]; + "kubelet-debug" = [ "ws" ]; + "schema" = [ "schemars" ]; + "schemars" = [ "dep:schemars" ]; + }; + resolvedDefaultFeatures = [ "json-patch" "jsonpatch" "schema" "schemars" ]; + }; + "kube-core 0.98.0" = rec { + crateName = "kube-core"; + version = "0.98.0"; + edition = "2021"; + sha256 = "1wwnsn1wk7bd2jiv9iw8446j0bczagqv1lc4wy88l5wa505q7alp"; + libName = "kube_core"; + authors = [ + "clux " + "Natalie Klestrup Röijezon " + "kazk " + ]; + dependencies = [ + { + name = "chrono"; + packageId = "chrono"; + usesDefaultFeatures = false; + features = [ "now" ]; + } + { + name = "form_urlencoded"; + packageId = "form_urlencoded"; + } + { + name = "http"; + packageId = "http"; + } + { + name = "json-patch"; + packageId = "json-patch 3.0.1"; + optional = true; + } + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.24.0"; + usesDefaultFeatures = false; + } + { + name = "schemars"; + packageId = "schemars"; + optional = true; + } + { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + { + name = "serde-value"; + packageId = "serde-value"; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "thiserror"; + packageId = "thiserror 2.0.11"; + } + ]; + devDependencies = [ + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.24.0"; + usesDefaultFeatures = false; + features = [ "latest" ]; + } + ]; + features = { + "admission" = [ "json-patch" ]; + "json-patch" = [ "dep:json-patch" ]; + "jsonpatch" = [ "json-patch" ]; + "kubelet-debug" = [ "ws" ]; + "schema" = [ "schemars" ]; + "schemars" = [ "dep:schemars" ]; + }; + resolvedDefaultFeatures = [ "json-patch" "jsonpatch" "schema" "schemars" ]; + }; + "kube-derive 0.96.0" = rec { + crateName = "kube-derive"; + version = "0.96.0"; + edition = "2021"; + sha256 = "1bc23sismxyyncsry902b2i2v0aifpxvgs3fdh9q412yrh24wdpr"; + procMacro = true; + libName = "kube_derive"; + authors = [ + "clux " + "Natalie Klestrup Röijezon " + "kazk " + ]; + dependencies = [ + { + name = "darling"; + packageId = "darling"; + } + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "syn"; + packageId = "syn 2.0.96"; + features = [ "extra-traits" ]; + } + ]; + + }; + "kube-derive 0.98.0" = rec { + crateName = "kube-derive"; + version = "0.98.0"; + edition = "2021"; + sha256 = "0n46p76pvm3plsnbm57c2j76r1i6hwslxsaj345pxdvn8255sx1p"; + procMacro = true; + libName = "kube_derive"; + authors = [ + "clux " + "Natalie Klestrup Röijezon " + "kazk " + ]; + dependencies = [ + { + name = "darling"; + packageId = "darling"; + } + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "syn"; + packageId = "syn 2.0.96"; + features = [ "extra-traits" ]; + } + ]; + + }; + "kube-runtime 0.96.0" = rec { + crateName = "kube-runtime"; + version = "0.96.0"; + edition = "2021"; + sha256 = "0i5xr5i9xf44fwih1pvypr35sq30pcw979m9sbqnb3m9zzvg3yyk"; + libName = "kube_runtime"; + authors = [ + "clux " + "Natalie Klestrup Röijezon " + "kazk " + ]; + dependencies = [ + { + name = "ahash"; + packageId = "ahash"; + } + { + name = "async-broadcast"; + packageId = "async-broadcast"; + } + { + name = "async-stream"; + packageId = "async-stream"; + } + { + name = "async-trait"; + packageId = "async-trait"; + } + { + name = "backoff"; + packageId = "backoff"; + } + { + name = "educe"; + packageId = "educe"; usesDefaultFeatures = false; - features = [ "now" ]; + features = [ "Clone" "Debug" "Hash" "PartialEq" ]; } { - name = "form_urlencoded"; - packageId = "form_urlencoded"; + name = "futures"; + packageId = "futures 0.3.31"; + usesDefaultFeatures = false; + features = [ "async-await" ]; } { - name = "http"; - packageId = "http"; + name = "hashbrown"; + packageId = "hashbrown 0.14.5"; } { name = "json-patch"; - packageId = "json-patch"; - optional = true; + packageId = "json-patch 2.0.0"; + } + { + name = "jsonptr"; + packageId = "jsonptr 0.4.7"; } { name = "k8s-openapi"; - packageId = "k8s-openapi"; + packageId = "k8s-openapi 0.23.0"; usesDefaultFeatures = false; } { - name = "schemars"; - packageId = "schemars"; - optional = true; + name = "kube-client"; + packageId = "kube-client 0.96.0"; + usesDefaultFeatures = false; + features = [ "jsonpatch" "client" ]; } { - name = "serde"; - packageId = "serde"; - features = [ "derive" ]; + name = "parking_lot"; + packageId = "parking_lot"; } { - name = "serde-value"; - packageId = "serde-value"; + name = "pin-project"; + packageId = "pin-project"; + } + { + name = "serde"; + packageId = "serde"; } { name = "serde_json"; @@ -4222,65 +5115,45 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.11"; + packageId = "thiserror 1.0.69"; } - ]; - devDependencies = [ { - name = "k8s-openapi"; - packageId = "k8s-openapi"; - usesDefaultFeatures = false; - features = [ "latest" ]; + name = "tokio"; + packageId = "tokio"; + features = [ "time" ]; } - ]; - features = { - "admission" = [ "json-patch" ]; - "json-patch" = [ "dep:json-patch" ]; - "jsonpatch" = [ "json-patch" ]; - "kubelet-debug" = [ "ws" ]; - "schema" = [ "schemars" ]; - "schemars" = [ "dep:schemars" ]; - }; - resolvedDefaultFeatures = [ "json-patch" "jsonpatch" "schema" "schemars" ]; - }; - "kube-derive" = rec { - crateName = "kube-derive"; - version = "0.98.0"; - edition = "2021"; - sha256 = "0n46p76pvm3plsnbm57c2j76r1i6hwslxsaj345pxdvn8255sx1p"; - procMacro = true; - libName = "kube_derive"; - authors = [ - "clux " - "Natalie Klestrup Röijezon " - "kazk " - ]; - dependencies = [ { - name = "darling"; - packageId = "darling"; + name = "tokio-util"; + packageId = "tokio-util"; + features = [ "time" ]; } { - name = "proc-macro2"; - packageId = "proc-macro2"; + name = "tracing"; + packageId = "tracing"; } + ]; + devDependencies = [ { - name = "quote"; - packageId = "quote"; + name = "k8s-openapi"; + packageId = "k8s-openapi 0.23.0"; + usesDefaultFeatures = false; + features = [ "latest" ]; } { name = "serde_json"; packageId = "serde_json"; } { - name = "syn"; - packageId = "syn 2.0.96"; - features = [ "extra-traits" ]; + name = "tokio"; + packageId = "tokio"; + features = [ "full" "test-util" ]; } ]; - + features = { + "unstable-runtime" = [ "unstable-runtime-subscribe" "unstable-runtime-stream-control" "unstable-runtime-reconcile-on" ]; + }; }; - "kube-runtime" = rec { + "kube-runtime 0.98.0" = rec { crateName = "kube-runtime"; version = "0.98.0"; edition = "2021"; @@ -4326,7 +5199,7 @@ rec { } { name = "hashbrown"; - packageId = "hashbrown"; + packageId = "hashbrown 0.15.2"; } { name = "hostname"; @@ -4334,20 +5207,20 @@ rec { } { name = "json-patch"; - packageId = "json-patch"; + packageId = "json-patch 3.0.1"; } { name = "jsonptr"; - packageId = "jsonptr"; + packageId = "jsonptr 0.6.3"; } { name = "k8s-openapi"; - packageId = "k8s-openapi"; + packageId = "k8s-openapi 0.24.0"; usesDefaultFeatures = false; } { name = "kube-client"; - packageId = "kube-client"; + packageId = "kube-client 0.98.0"; usesDefaultFeatures = false; features = [ "jsonpatch" "client" ]; } @@ -4389,7 +5262,7 @@ rec { devDependencies = [ { name = "k8s-openapi"; - packageId = "k8s-openapi"; + packageId = "k8s-openapi 0.24.0"; usesDefaultFeatures = false; features = [ "latest" ]; } @@ -5766,7 +6639,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags"; + packageId = "bitflags 2.8.0"; } ]; features = { @@ -6547,7 +7420,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags"; + packageId = "bitflags 2.8.0"; } { name = "core-foundation"; @@ -6594,7 +7467,7 @@ rec { dependencies = [ { name = "bitflags"; - packageId = "bitflags"; + packageId = "bitflags 2.8.0"; } { name = "core-foundation"; @@ -7227,68 +8100,6 @@ rec { }; resolvedDefaultFeatures = [ "alloc" ]; }; - "stackable-airflow-crd" = rec { - crateName = "stackable-airflow-crd"; - version = "0.0.0-dev"; - edition = "2021"; - src = lib.cleanSourceWith { filter = sourceFilter; src = ./rust/crd; }; - libName = "stackable_airflow_crd"; - authors = [ - "Stackable GmbH " - ]; - dependencies = [ - { - name = "indoc"; - packageId = "indoc"; - } - { - name = "product-config"; - packageId = "product-config"; - } - { - name = "serde"; - packageId = "serde"; - features = [ "derive" ]; - } - { - name = "serde_json"; - packageId = "serde_json"; - } - { - name = "snafu"; - packageId = "snafu 0.8.5"; - } - { - name = "stackable-operator"; - packageId = "stackable-operator"; - } - { - name = "strum"; - packageId = "strum"; - features = [ "derive" ]; - } - { - name = "tokio"; - packageId = "tokio"; - features = [ "full" ]; - } - { - name = "tracing"; - packageId = "tracing"; - } - ]; - devDependencies = [ - { - name = "rstest"; - packageId = "rstest"; - } - { - name = "serde_yaml"; - packageId = "serde_yaml"; - } - ]; - - }; "stackable-airflow-operator" = rec { crateName = "stackable-airflow-operator"; version = "0.0.0-dev"; @@ -7339,6 +8150,10 @@ rec { packageId = "serde"; features = [ "derive" ]; } + { + name = "serde_json"; + packageId = "serde_json"; + } { name = "serde_yaml"; packageId = "serde_yaml"; @@ -7347,14 +8162,15 @@ rec { name = "snafu"; packageId = "snafu 0.8.5"; } - { - name = "stackable-airflow-crd"; - packageId = "stackable-airflow-crd"; - } { name = "stackable-operator"; packageId = "stackable-operator"; } + { + name = "stackable-versioned"; + packageId = "stackable-versioned"; + features = [ "k8s" ]; + } { name = "strum"; packageId = "strum"; @@ -7446,17 +8262,17 @@ rec { } { name = "json-patch"; - packageId = "json-patch"; + packageId = "json-patch 3.0.1"; } { name = "k8s-openapi"; - packageId = "k8s-openapi"; + packageId = "k8s-openapi 0.24.0"; usesDefaultFeatures = false; features = [ "schemars" "v1_32" ]; } { name = "kube"; - packageId = "kube"; + packageId = "kube 0.98.0"; usesDefaultFeatures = false; features = [ "client" "jsonpatch" "runtime" "derive" "rustls-tls" ]; } @@ -7601,7 +8417,7 @@ rec { dependencies = [ { name = "kube"; - packageId = "kube"; + packageId = "kube 0.98.0"; usesDefaultFeatures = false; features = [ "client" "jsonpatch" "runtime" "derive" "rustls-tls" ]; } @@ -7625,6 +8441,106 @@ rec { ]; }; + "stackable-versioned" = rec { + crateName = "stackable-versioned"; + version = "0.5.0"; + edition = "2021"; + workspace_member = null; + src = pkgs.fetchgit { + url = "https://github.com/stackabletech/operator-rs.git"; + rev = "048c7d8befddc2f2c6414444006871c95412d67c"; + sha256 = "1x2pfibrsysmkkmajyj30qkwsjf3rzmc3dxsd09jb9r4x7va6mr6"; + }; + libName = "stackable_versioned"; + authors = [ + "Stackable GmbH " + ]; + dependencies = [ + { + name = "stackable-versioned-macros"; + packageId = "stackable-versioned-macros"; + } + ]; + features = { + "full" = [ "k8s" ]; + "k8s" = [ "stackable-versioned-macros/k8s" ]; + }; + resolvedDefaultFeatures = [ "k8s" ]; + }; + "stackable-versioned-macros" = rec { + crateName = "stackable-versioned-macros"; + version = "0.5.0"; + edition = "2021"; + workspace_member = null; + src = pkgs.fetchgit { + url = "https://github.com/stackabletech/operator-rs.git"; + rev = "048c7d8befddc2f2c6414444006871c95412d67c"; + sha256 = "1x2pfibrsysmkkmajyj30qkwsjf3rzmc3dxsd09jb9r4x7va6mr6"; + }; + procMacro = true; + libName = "stackable_versioned_macros"; + authors = [ + "Stackable GmbH " + ]; + dependencies = [ + { + name = "convert_case"; + packageId = "convert_case"; + } + { + name = "darling"; + packageId = "darling"; + } + { + name = "itertools"; + packageId = "itertools"; + } + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.23.0"; + optional = true; + usesDefaultFeatures = false; + features = [ "schemars" "v1_31" ]; + } + { + name = "k8s-version"; + packageId = "k8s-version"; + features = [ "darling" ]; + } + { + name = "kube"; + packageId = "kube 0.96.0"; + optional = true; + usesDefaultFeatures = false; + features = [ "client" "jsonpatch" "runtime" "derive" "rustls-tls" ]; + } + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "syn"; + packageId = "syn 2.0.96"; + } + ]; + devDependencies = [ + { + name = "k8s-openapi"; + packageId = "k8s-openapi 0.23.0"; + usesDefaultFeatures = false; + features = [ "schemars" "v1_31" ]; + } + ]; + features = { + "full" = [ "k8s" ]; + "k8s" = [ "dep:kube" "dep:k8s-openapi" ]; + }; + resolvedDefaultFeatures = [ "k8s" ]; + }; "strsim" = rec { crateName = "strsim"; version = "0.11.1"; @@ -8578,7 +9494,7 @@ rec { } { name = "bitflags"; - packageId = "bitflags"; + packageId = "bitflags 2.8.0"; } { name = "bytes"; @@ -9170,6 +10086,19 @@ rec { ]; }; + "unicode-segmentation" = rec { + crateName = "unicode-segmentation"; + version = "1.12.0"; + edition = "2018"; + sha256 = "14qla2jfx74yyb9ds3d2mpwpa4l4lzb9z57c6d2ba511458z5k7n"; + libName = "unicode_segmentation"; + authors = [ + "kwantam " + "Manish Goregaokar " + ]; + features = { + }; + }; "unicode-xid" = rec { crateName = "unicode-xid"; version = "0.2.6"; diff --git a/Cargo.toml b/Cargo.toml index f26d8fc6..14a3f20f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["rust/crd", "rust/operator-binary"] +members = ["rust/operator-binary"] resolver = "2" [workspace.package] @@ -10,6 +10,10 @@ edition = "2021" repository = "https://github.com/stackabletech/airflow-operator" [workspace.dependencies] +stackable-versioned = { git = "https://github.com/stackabletech/operator-rs.git", features = ["k8s"], tag = "stackable-versioned-0.5.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.85.0" } +product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" } + anyhow = "1.0" built = { version = "0.7", features = ["chrono", "git2"] } clap = "4.5" @@ -17,14 +21,12 @@ const_format = "0.2" fnv = "1.0" futures = { version = "0.3", features = ["compat"] } indoc = "2.0" -product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" } rstest = "0.24" semver = "1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.8" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.85.0" } strum = { version = "0.26", features = ["derive"] } tokio = { version = "1.40", features = ["full"] } tracing = "0.1" diff --git a/crate-hashes.json b/crate-hashes.json index 290d87f2..c7d32c3a 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -2,5 +2,8 @@ "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#stackable-operator-derive@0.3.1": "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5", "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#stackable-operator@0.85.0": "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5", "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.85.0#stackable-shared@0.0.1": "0rh476rmn5850yj85hq8znwmlfhd7l5bkxz0n5i9m4cddxhi2cl5", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.5.0#k8s-version@0.1.2": "1x2pfibrsysmkkmajyj30qkwsjf3rzmc3dxsd09jb9r4x7va6mr6", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.5.0#stackable-versioned-macros@0.5.0": "1x2pfibrsysmkkmajyj30qkwsjf3rzmc3dxsd09jb9r4x7va6mr6", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-versioned-0.5.0#stackable-versioned@0.5.0": "1x2pfibrsysmkkmajyj30qkwsjf3rzmc3dxsd09jb9r4x7va6mr6", "git+https://github.com/stackabletech/product-config.git?tag=0.7.0#product-config@0.7.0": "0gjsm80g6r75pm3824dcyiz4ysq1ka4c1if6k1mjm9cnd5ym0gny" } \ No newline at end of file diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml deleted file mode 100644 index 42cdff8c..00000000 --- a/rust/crd/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -name = "stackable-airflow-crd" -description = "Contains the Apache Airflow CRD structs and utilities" -version.workspace = true -authors.workspace = true -license.workspace = true -edition.workspace = true -repository.workspace = true -publish = false - -[dependencies] -indoc.workspace = true -product-config.workspace = true -serde.workspace = true -serde_json.workspace = true -snafu.workspace = true -stackable-operator.workspace = true -strum.workspace = true -tokio.workspace = true -tracing.workspace = true - -[dev-dependencies] -serde_yaml.workspace = true -rstest.workspace = true diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 6db61b19..59716524 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -9,7 +9,9 @@ repository.workspace = true publish = false [dependencies] -stackable-airflow-crd = { path = "../crd" } +stackable-versioned.workspace = true +stackable-operator.workspace = true +product-config.workspace = true anyhow.workspace = true clap.workspace = true @@ -17,10 +19,9 @@ const_format.workspace = true fnv.workspace = true futures.workspace = true serde.workspace = true +serde_json.workspace = true serde_yaml.workspace = true snafu.workspace = true -stackable-operator.workspace = true -product-config.workspace = true strum.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 8cdf1d37..6811a319 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -1,4 +1,4 @@ -//! Ensures that `Pod`s are configured and running for each [`AirflowCluster`] +//! Ensures that `Pod`s are configured and running for each [`v1alpha1::AirflowCluster`] use std::{ collections::{BTreeMap, BTreeSet, HashMap}, io::Write, @@ -13,18 +13,6 @@ use product_config::{ ProductConfigManager, }; use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_airflow_crd::{ - authentication::{ - AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, - }, - build_recommended_labels, - git_sync::GitSync, - AirflowCluster, AirflowClusterStatus, AirflowConfig, AirflowConfigOptions, AirflowExecutor, - AirflowRole, Container, ExecutorConfig, ExecutorConfigFragment, AIRFLOW_CONFIG_FILENAME, - AIRFLOW_UID, APP_NAME, CONFIG_PATH, GIT_CONTENT, GIT_ROOT, GIT_SYNC_NAME, LOG_CONFIG_DIR, - OPERATOR_NAME, STACKABLE_LOG_DIR, TEMPLATE_CONFIGMAP_NAME, TEMPLATE_LOCATION, TEMPLATE_NAME, - TEMPLATE_VOLUME_NAME, -}; use stackable_operator::{ builder::{ self, @@ -86,6 +74,18 @@ use strum::{EnumDiscriminants, IntoEnumIterator, IntoStaticStr}; use crate::{ config::{self, PYTHON_IMPORTS}, controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, + crd::{ + self, + authentication::{ + AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, + }, + build_recommended_labels, + git_sync::{GitSync, GIT_SYNC_CONTENT, GIT_SYNC_NAME, GIT_SYNC_ROOT}, + v1alpha1, AirflowClusterStatus, AirflowConfig, AirflowConfigOptions, AirflowExecutor, + AirflowRole, Container, ExecutorConfig, ExecutorConfigFragment, AIRFLOW_CONFIG_FILENAME, + AIRFLOW_UID, APP_NAME, CONFIG_PATH, LOG_CONFIG_DIR, OPERATOR_NAME, STACKABLE_LOG_DIR, + TEMPLATE_CONFIGMAP_NAME, TEMPLATE_LOCATION, TEMPLATE_NAME, TEMPLATE_VOLUME_NAME, + }, env_vars::{ self, build_airflow_template_envs, build_gitsync_statefulset_envs, build_gitsync_template, }, @@ -113,7 +113,6 @@ pub struct Ctx { #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] -#[allow(clippy::enum_variant_names)] pub enum Error { #[snafu(display("object has no namespace"))] ObjectHasNoNamespace, @@ -129,19 +128,19 @@ pub enum Error { #[snafu(display("failed to apply Service for {rolegroup}"))] ApplyRoleGroupService { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, + rolegroup: RoleGroupRef, }, #[snafu(display("failed to apply ConfigMap for {rolegroup}"))] ApplyRoleGroupConfig { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, + rolegroup: RoleGroupRef, }, #[snafu(display("failed to apply StatefulSet for {rolegroup}"))] ApplyRoleGroupStatefulSet { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, + rolegroup: RoleGroupRef, }, #[snafu(display("invalid product config"))] @@ -192,19 +191,17 @@ pub enum Error { #[snafu(display("failed to build config file for {rolegroup}"))] BuildRoleGroupConfigFile { source: FlaskAppConfigWriterError, - rolegroup: RoleGroupRef, + rolegroup: RoleGroupRef, }, #[snafu(display("failed to build ConfigMap for {rolegroup}"))] BuildRoleGroupConfig { source: stackable_operator::builder::configmap::Error, - rolegroup: RoleGroupRef, + rolegroup: RoleGroupRef, }, #[snafu(display("failed to resolve and merge config for role and role group"))] - FailedToResolveConfig { - source: stackable_airflow_crd::Error, - }, + FailedToResolveConfig { source: crd::Error }, #[snafu(display("could not parse Airflow role [{role}]"))] UnidentifiedAirflowRole { @@ -249,9 +246,7 @@ pub enum Error { }, #[snafu(display("failed to apply authentication configuration"))] - InvalidAuthenticationConfig { - source: stackable_airflow_crd::authentication::Error, - }, + InvalidAuthenticationConfig { source: crd::authentication::Error }, #[snafu(display("pod template serialization"))] PodTemplateSerde { source: serde_yaml::Error }, @@ -336,7 +331,7 @@ impl ReconcilerError for Error { } pub async fn reconcile_airflow( - airflow: Arc>, + airflow: Arc>, ctx: Arc, ) -> Result { tracing::info!("Starting reconcile"); @@ -558,7 +553,7 @@ pub async fn reconcile_airflow( #[allow(clippy::too_many_arguments)] async fn build_executor_template( - airflow: &AirflowCluster, + airflow: &v1alpha1::AirflowCluster, common_config: &CommonConfiguration, resolved_product_image: &ResolvedProductImage, authentication_config: &AirflowClientAuthenticationDetailsResolved, @@ -612,7 +607,7 @@ async fn build_executor_template( /// The server-role service is the primary endpoint that should be used by clients that do not perform internal load balancing, /// including targets outside the cluster. fn build_role_service( - airflow: &AirflowCluster, + airflow: &v1alpha1::AirflowCluster, resolved_product_image: &ResolvedProductImage, role_name: &str, port: u16, @@ -674,9 +669,9 @@ fn role_port(role_name: &str) -> Option { /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator #[allow(clippy::too_many_arguments)] fn build_rolegroup_config_map( - airflow: &AirflowCluster, + airflow: &v1alpha1::AirflowCluster, resolved_product_image: &ResolvedProductImage, - rolegroup: &RoleGroupRef, + rolegroup: &RoleGroupRef, rolegroup_config: &HashMap>, authentication_config: &AirflowClientAuthenticationDetailsResolved, logging: &Logging, @@ -760,9 +755,9 @@ fn build_rolegroup_config_map( /// /// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. fn build_rolegroup_service( - airflow: &AirflowCluster, + airflow: &v1alpha1::AirflowCluster, resolved_product_image: &ResolvedProductImage, - rolegroup: &RoleGroupRef, + rolegroup: &RoleGroupRef, ) -> Result { let mut ports = vec![ServicePort { name: Some(METRICS_PORT_NAME.into()), @@ -807,9 +802,9 @@ fn build_rolegroup_service( } fn build_rolegroup_metadata( - airflow: &AirflowCluster, + airflow: &v1alpha1::AirflowCluster, resolved_product_image: &&ResolvedProductImage, - rolegroup: &&RoleGroupRef, + rolegroup: &&RoleGroupRef, prometheus_label: Label, ) -> Result { let metadata = ObjectMetaBuilder::new() @@ -835,10 +830,10 @@ fn build_rolegroup_metadata( /// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the corresponding [`Service`] (from [`build_rolegroup_service`]). #[allow(clippy::too_many_arguments)] fn build_server_rolegroup_statefulset( - airflow: &AirflowCluster, + airflow: &v1alpha1::AirflowCluster, resolved_product_image: &ResolvedProductImage, airflow_role: &AirflowRole, - rolegroup_ref: &RoleGroupRef, + rolegroup_ref: &RoleGroupRef, rolegroup_config: &HashMap>, authentication_config: &AirflowClientAuthenticationDetailsResolved, service_account: &ServiceAccount, @@ -1008,7 +1003,7 @@ fn build_server_rolegroup_statefulset( )?; pb.add_volume( - VolumeBuilder::new(GIT_CONTENT) + VolumeBuilder::new(GIT_SYNC_CONTENT) .empty_dir(EmptyDirVolumeSource::default()) .build(), ) @@ -1116,14 +1111,14 @@ fn build_logging_container( #[allow(clippy::too_many_arguments)] fn build_executor_template_config_map( - airflow: &AirflowCluster, + airflow: &v1alpha1::AirflowCluster, resolved_product_image: &ResolvedProductImage, authentication_config: &AirflowClientAuthenticationDetailsResolved, sa_name: &str, merged_executor_config: &ExecutorConfig, env_overrides: &HashMap, pod_overrides: &PodTemplateSpec, - rolegroup_ref: &RoleGroupRef, + rolegroup_ref: &RoleGroupRef, ) -> Result { let mut pb = PodBuilder::new(); let pb_metadata = ObjectMetaBuilder::new() @@ -1203,7 +1198,7 @@ fn build_executor_template_config_map( airflow.volume_mounts(), )?; pb.add_volume( - VolumeBuilder::new(GIT_CONTENT) + VolumeBuilder::new(GIT_SYNC_CONTENT) .empty_dir(EmptyDirVolumeSource::default()) .build(), ) @@ -1275,7 +1270,7 @@ fn build_gitsync_container( "-c".to_string(), ]) .args(vec![gitsync.get_args(one_time).join("\n")]) - .add_volume_mount(GIT_CONTENT, GIT_ROOT) + .add_volume_mount(GIT_SYNC_CONTENT, GIT_SYNC_ROOT) .context(AddVolumeMountSnafu)? .add_volume_mounts(volume_mounts) .context(AddVolumeMountSnafu)? @@ -1292,7 +1287,7 @@ fn build_gitsync_container( } pub fn error_policy( - _obj: Arc>, + _obj: Arc>, error: &Error, _ctx: Arc, ) -> Action { diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index 60f2d122..4bb571d0 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -2,17 +2,18 @@ use std::collections::BTreeMap; use indoc::formatdoc; use snafu::{ResultExt, Snafu}; -use stackable_airflow_crd::{ +use stackable_operator::commons::{ + authentication::{ldap::AuthenticationProvider, oidc}, + tls_verification::TlsVerification, +}; + +use crate::crd::{ authentication::{ AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, FlaskRolesSyncMoment, DEFAULT_OIDC_PROVIDER, }, AirflowConfigOptions, }; -use stackable_operator::commons::{ - authentication::{ldap::AuthenticationProvider, oidc}, - tls_verification::TlsVerification, -}; pub const PYTHON_IMPORTS: &[&str] = &[ "import os", @@ -280,13 +281,15 @@ mod tests { use indoc::formatdoc; use rstest::rstest; - use stackable_airflow_crd::authentication::{ - default_sync_roles_at, default_user_registration, AirflowAuthenticationClassResolved, - AirflowClientAuthenticationDetailsResolved, FlaskRolesSyncMoment, - }; use stackable_operator::commons::authentication::{ldap, oidc}; - use crate::config::add_airflow_config; + use crate::{ + config::add_airflow_config, + crd::authentication::{ + default_user_registration, AirflowAuthenticationClassResolved, + AirflowClientAuthenticationDetailsResolved, FlaskRolesSyncMoment, + }, + }; #[test] fn test_auth_db_config() { @@ -421,7 +424,7 @@ mod tests { ], user_registration: default_user_registration(), user_registration_role: "Admin".to_string(), - sync_roles_at: default_sync_roles_at(), + sync_roles_at: FlaskRolesSyncMoment::Registration, }; let mut result = BTreeMap::new(); diff --git a/rust/operator-binary/src/controller_commons.rs b/rust/operator-binary/src/controller_commons.rs index 78f746fd..7d16c41b 100644 --- a/rust/operator-binary/src/controller_commons.rs +++ b/rust/operator-binary/src/controller_commons.rs @@ -1,4 +1,3 @@ -use stackable_airflow_crd::MAX_LOG_FILES_SIZE; use stackable_operator::{ builder::pod::volume::VolumeBuilder, k8s_openapi::api::core::v1::{ConfigMapVolumeSource, EmptyDirVolumeSource, Volume}, @@ -11,6 +10,8 @@ use stackable_operator::{ }, }; +use crate::crd::MAX_LOG_FILES_SIZE; + pub const CONFIG_VOLUME_NAME: &str = "config"; pub const LOG_CONFIG_VOLUME_NAME: &str = "log-config"; pub const LOG_VOLUME_NAME: &str = "log"; diff --git a/rust/crd/src/affinity.rs b/rust/operator-binary/src/crd/affinity.rs similarity index 96% rename from rust/crd/src/affinity.rs rename to rust/operator-binary/src/crd/affinity.rs index a17d3a9d..a8a0bb66 100644 --- a/rust/crd/src/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -5,7 +5,7 @@ use stackable_operator::{ k8s_openapi::api::core::v1::{PodAffinity, PodAntiAffinity}, }; -use crate::{AirflowRole, APP_NAME}; +use crate::crd::{AirflowRole, APP_NAME}; /// Used for all [`AirflowRole`]s besides executors. pub fn get_affinity(cluster_name: &str, role: &AirflowRole) -> StackableAffinityFragment { @@ -40,7 +40,6 @@ fn get_affinity_for_role(cluster_name: &str, role: &str) -> StackableAffinityFra } #[cfg(test)] mod tests { - use std::collections::BTreeMap; use rstest::rstest; @@ -56,7 +55,7 @@ mod tests { role_utils::RoleGroupRef, }; - use crate::{AirflowCluster, AirflowRole}; + use crate::crd::{v1alpha1, AirflowExecutor, AirflowRole}; #[rstest] #[case(AirflowRole::Worker)] @@ -88,7 +87,7 @@ mod tests { "; let deserializer = serde_yaml::Deserializer::from_str(cluster); - let airflow: AirflowCluster = + let airflow: v1alpha1::AirflowCluster = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); let rolegroup_ref = RoleGroupRef { @@ -179,7 +178,7 @@ mod tests { "; let deserializer = serde_yaml::Deserializer::from_str(cluster); - let airflow: AirflowCluster = + let airflow: v1alpha1::AirflowCluster = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); let expected: StackableAffinity = StackableAffinity { @@ -236,8 +235,8 @@ mod tests { }; let executor_config = match &airflow.spec.executor { - crate::AirflowExecutor::CeleryExecutor { .. } => unreachable!(), - crate::AirflowExecutor::KubernetesExecutor { + AirflowExecutor::CeleryExecutor { .. } => unreachable!(), + AirflowExecutor::KubernetesExecutor { common_configuration, } => &common_configuration.config, }; diff --git a/rust/crd/src/authentication.rs b/rust/operator-binary/src/crd/authentication.rs similarity index 99% rename from rust/crd/src/authentication.rs rename to rust/operator-binary/src/crd/authentication.rs index d6889953..297dabf7 100644 --- a/rust/crd/src/authentication.rs +++ b/rust/operator-binary/src/crd/authentication.rs @@ -110,11 +110,6 @@ pub fn default_user_registration_role() -> String { "Public".to_string() } -/// Matches Flask's default mode of syncing at registration -pub fn default_sync_roles_at() -> FlaskRolesSyncMoment { - FlaskRolesSyncMoment::Registration -} - #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize, Default)] pub enum FlaskRolesSyncMoment { #[default] diff --git a/rust/crd/src/git_sync.rs b/rust/operator-binary/src/crd/git_sync.rs similarity index 93% rename from rust/crd/src/git_sync.rs rename to rust/operator-binary/src/crd/git_sync.rs index 3187e55c..951c99c6 100644 --- a/rust/crd/src/git_sync.rs +++ b/rust/operator-binary/src/crd/git_sync.rs @@ -6,7 +6,15 @@ use stackable_operator::{ utils::COMMON_BASH_TRAP_FUNCTIONS, }; -use crate::{GIT_LINK, GIT_ROOT, GIT_SAFE_DIR, GIT_SYNC_DEPTH, GIT_SYNC_PERIOD_SECONDS}; +pub const GIT_SYNC_CONTENT: &str = "content-from-git"; +pub const GIT_SYNC_SAFE_DIR: &str = "safe.directory"; +pub const GIT_SYNC_DIR: &str = "/stackable/app/git"; +pub const GIT_SYNC_ROOT: &str = "/tmp/git"; +pub const GIT_SYNC_LINK: &str = "current"; +pub const GIT_SYNC_NAME: &str = "gitsync"; + +const GIT_SYNC_PERIOD_SECONDS: u16 = 20u16; +const GIT_SYNC_DEPTH: u8 = 1u8; #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] #[serde(rename_all = "camelCase")] @@ -38,7 +46,7 @@ impl GitSync { /// a pod template and is terminated by airflow itself). The `one_time` parameter is used /// to indicate this. pub fn get_args(&self, one_time: bool) -> Vec { - let mut git_config = format!("{GIT_SAFE_DIR}:{GIT_ROOT}"); + let mut git_config = format!("{GIT_SYNC_SAFE_DIR}:{GIT_SYNC_ROOT}"); let mut git_sync_command = vec![ "/stackable/git-sync".to_string(), format!("--repo={}", self.repo.clone()), @@ -48,8 +56,8 @@ impl GitSync { ), format!("--depth={}", self.depth.unwrap_or(GIT_SYNC_DEPTH)), format!("--period={}s", self.wait.unwrap_or(GIT_SYNC_PERIOD_SECONDS)), - format!("--link={GIT_LINK}"), - format!("--root={GIT_ROOT}"), + format!("--link={GIT_SYNC_LINK}"), + format!("--root={GIT_SYNC_ROOT}"), ]; if let Some(git_sync_conf) = self.git_sync_conf.as_ref() { for (key, value) in git_sync_conf { @@ -60,8 +68,8 @@ impl GitSync { } else { // both "-git-config" and "--gitconfig" are recognized by gitsync if key.to_lowercase().ends_with("-git-config") { - if value.to_lowercase().contains(GIT_SAFE_DIR) { - tracing::warn!("Config option {value:?} contains a value for {GIT_SAFE_DIR} that overrides + if value.to_lowercase().contains(GIT_SYNC_SAFE_DIR) { + tracing::warn!("Config option {value:?} contains a value for {GIT_SYNC_SAFE_DIR} that overrides the value of this operator. Git-sync functionality will probably not work as expected!"); } git_config = format!("{git_config},{value}"); @@ -99,7 +107,7 @@ impl GitSync { mod tests { use rstest::rstest; - use crate::AirflowCluster; + use crate::v1alpha1::AirflowCluster; #[test] fn test_git_sync() { diff --git a/rust/crd/src/lib.rs b/rust/operator-binary/src/crd/mod.rs similarity index 81% rename from rust/crd/src/lib.rs rename to rust/operator-binary/src/crd/mod.rs index c430bd68..828db134 100644 --- a/rust/crd/src/lib.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,9 +1,5 @@ use std::collections::{BTreeMap, BTreeSet}; -use authentication::{ - AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, -}; -use git_sync::GitSync; use product_config::flask_app_config_writer::{FlaskAppConfigOptions, PythonType}; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt, Snafu}; @@ -43,11 +39,16 @@ use stackable_operator::{ time::Duration, utils::{crds::raw_object_list_schema, COMMON_BASH_TRAP_FUNCTIONS}, }; +use stackable_versioned::versioned; use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; -use crate::{ +use crate::crd::{ affinity::{get_affinity, get_executor_affinity}, - authentication::AirflowClientAuthenticationDetails, + authentication::{ + AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetails, + AirflowClientAuthenticationDetailsResolved, + }, + git_sync::{GitSync, GIT_SYNC_CONTENT, GIT_SYNC_DIR}, }; pub mod affinity; @@ -62,21 +63,12 @@ pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; pub const LOG_CONFIG_DIR: &str = "/stackable/app/log_config"; pub const AIRFLOW_HOME: &str = "/stackable/airflow"; pub const AIRFLOW_CONFIG_FILENAME: &str = "webserver_config.py"; -pub const GIT_SYNC_DIR: &str = "/stackable/app/git"; -pub const GIT_CONTENT: &str = "content-from-git"; -pub const GIT_ROOT: &str = "/tmp/git"; -pub const GIT_LINK: &str = "current"; -pub const GIT_SYNC_NAME: &str = "gitsync"; -pub const GIT_SAFE_DIR: &str = "safe.directory"; pub const TEMPLATE_VOLUME_NAME: &str = "airflow-executor-pod-template"; pub const TEMPLATE_CONFIGMAP_NAME: &str = "airflow-executor-pod-template"; pub const TEMPLATE_LOCATION: &str = "/templates"; pub const TEMPLATE_NAME: &str = "airflow_executor_pod_template.yaml"; -const GIT_SYNC_DEPTH: u8 = 1u8; -const GIT_SYNC_PERIOD_SECONDS: u16 = 20u16; - const DEFAULT_AIRFLOW_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(2); const DEFAULT_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(5); @@ -147,110 +139,268 @@ impl FlaskAppConfigOptions for AirflowConfigOptions { } } -/// An Airflow cluster stacklet. This resource is managed by the Stackable operator for Apache Airflow. -/// Find more information on how to use it and the resources that the operator generates in the -/// [operator documentation](DOCS_BASE_URL_PLACEHOLDER/airflow/). -/// -/// The CRD contains three roles: webserver, scheduler and worker/celeryExecutor. -/// You can use either the celeryExecutor or the kubernetesExecutor. -#[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] -#[kube( - group = "airflow.stackable.tech", - version = "v1alpha1", - kind = "AirflowCluster", - plural = "airflowclusters", - shortname = "airflow", - status = "AirflowClusterStatus", - namespaced, - crates( - kube_core = "stackable_operator::kube::core", - k8s_openapi = "stackable_operator::k8s_openapi", - schemars = "stackable_operator::schemars" - ) -)] -#[serde(rename_all = "camelCase")] -pub struct AirflowClusterSpec { - // no doc string - See ProductImage struct - pub image: ProductImage, +#[versioned(version(name = "v1alpha1"))] +pub mod versioned { + /// An Airflow cluster stacklet. This resource is managed by the Stackable operator for Apache Airflow. + /// Find more information on how to use it and the resources that the operator generates in the + /// [operator documentation](DOCS_BASE_URL_PLACEHOLDER/airflow/). + /// + /// The CRD contains three roles: webserver, scheduler and worker/celeryExecutor. + /// You can use either the celeryExecutor or the kubernetesExecutor. + #[versioned(k8s( + group = "airflow.stackable.tech", + kind = "AirflowCluster", + plural = "airflowclusters", + shortname = "airflow", + status = "AirflowClusterStatus", + namespaced, + crates( + kube_core = "stackable_operator::kube::core", + k8s_openapi = "stackable_operator::k8s_openapi", + schemars = "stackable_operator::schemars" + ) + ))] + #[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct AirflowClusterSpec { + // no doc string - See ProductImage struct + pub image: ProductImage, + + /// Configuration that applies to all roles and role groups. + /// This includes settings for authentication, git sync, service exposition and volumes, among other things. + pub cluster_config: v1alpha1::AirflowClusterConfig, + + // no doc string - See ClusterOperation struct + #[serde(default)] + pub cluster_operation: ClusterOperation, + + /// The `webserver` role provides the main UI for user interaction. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub webservers: Option>, + + /// The `scheduler` is responsible for triggering jobs and persisting their metadata to the backend database. + /// Jobs are scheduled on the workers/executors. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schedulers: Option>, - /// Configuration that applies to all roles and role groups. - /// This includes settings for authentication, git sync, service exposition and volumes, among other things. - pub cluster_config: AirflowClusterConfig, + #[serde(flatten)] + pub executor: AirflowExecutor, + } + + #[derive(Clone, Deserialize, Debug, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct AirflowClusterConfig { + #[serde(default)] + pub authentication: Vec, + + /// The name of the Secret object containing the admin user credentials and database connection details. + /// Read the + /// [getting started guide first steps](DOCS_BASE_URL_PLACEHOLDER/airflow/getting_started/first_steps) + /// to find out more. + pub credentials_secret: String, + + /// The `gitSync` settings allow configuring DAGs to mount via `git-sync`. + /// Learn more in the + /// [mounting DAGs documentation](DOCS_BASE_URL_PLACEHOLDER/airflow/usage-guide/mounting-dags#_via_git_sync). + #[serde(default)] + pub dags_git_sync: Vec, + + /// for internal use only - not for production use. + #[serde(default)] + pub expose_config: bool, + + /// Whether to load example DAGs or not; defaults to false. The examples are used in the + /// [getting started guide](DOCS_BASE_URL_PLACEHOLDER/airflow/getting_started/). + #[serde(default)] + pub load_examples: bool, + + /// This field controls which type of Service the Operator creates for this AirflowCluster: + /// + /// * cluster-internal: Use a ClusterIP service + /// + /// * external-unstable: Use a NodePort service + /// + /// * external-stable: Use a LoadBalancer service + /// + /// This is a temporary solution with the goal to keep yaml manifests forward compatible. + /// In the future, this setting will control which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) + /// will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change. + #[serde(default)] + pub listener_class: CurrentlySupportedListenerClasses, + + /// Name of the Vector aggregator [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery). + /// It must contain the key `ADDRESS` with the address of the Vector aggregator. + /// Follow the [logging tutorial](DOCS_BASE_URL_PLACEHOLDER/tutorials/logging-vector-aggregator) + /// to learn how to configure log aggregation with Vector. + #[serde(skip_serializing_if = "Option::is_none")] + pub vector_aggregator_config_map_name: Option, + + /// Additional volumes to define. Use together with `volumeMounts` to mount the volumes. + #[serde(default)] + #[schemars(schema_with = "raw_object_list_schema")] + pub volumes: Vec, + + /// Additional volumes to mount. Use together with `volumes` to define volumes. + #[serde(default)] + #[schemars(schema_with = "raw_object_list_schema")] + pub volume_mounts: Vec, + } +} - // no doc string - See ClusterOperation struct +#[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AirflowClusterStatus { #[serde(default)] - pub cluster_operation: ClusterOperation, + pub conditions: Vec, +} - /// The `webserver` role provides the main UI for user interaction. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub webservers: Option>, +impl HasStatusCondition for v1alpha1::AirflowCluster { + fn conditions(&self) -> Vec { + match &self.status { + Some(status) => status.conditions.clone(), + None => vec![], + } + } +} - /// The `scheduler` is responsible for triggering jobs and persisting their metadata to the backend database. - /// Jobs are scheduled on the workers/executors. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub schedulers: Option>, +impl v1alpha1::AirflowCluster { + /// the worker role will not be returned if airflow provisions pods as needed (i.e. when + /// the kubernetes executor is specified) + pub fn get_role(&self, role: &AirflowRole) -> Option<&Role> { + match role { + AirflowRole::Webserver => self.spec.webservers.as_ref(), + AirflowRole::Scheduler => self.spec.schedulers.as_ref(), + AirflowRole::Worker => { + if let AirflowExecutor::CeleryExecutor { config } = &self.spec.executor { + Some(config) + } else { + None + } + } + } + } - #[serde(flatten)] - pub executor: AirflowExecutor, -} + pub fn role_config(&self, role: &AirflowRole) -> Option<&GenericRoleConfig> { + self.get_role(role).map(|r| &r.role_config) + } -#[derive(Clone, Deserialize, Debug, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct AirflowClusterConfig { - #[serde(default)] - pub authentication: Vec, + pub fn volumes(&self) -> &Vec { + &self.spec.cluster_config.volumes + } - /// The name of the Secret object containing the admin user credentials and database connection details. - /// Read the - /// [getting started guide first steps](DOCS_BASE_URL_PLACEHOLDER/airflow/getting_started/first_steps) - /// to find out more. - pub credentials_secret: String, + pub fn volume_mounts(&self) -> Vec { + let mut mounts = self.spec.cluster_config.volume_mounts.clone(); + if self.git_sync().is_some() { + mounts.push(VolumeMount { + name: GIT_SYNC_CONTENT.into(), + mount_path: GIT_SYNC_DIR.into(), + ..VolumeMount::default() + }); + } + mounts + } - /// The `gitSync` settings allow configuring DAGs to mount via `git-sync`. - /// Learn more in the - /// [mounting DAGs documentation](DOCS_BASE_URL_PLACEHOLDER/airflow/usage-guide/mounting-dags#_via_git_sync). - #[serde(default)] - pub dags_git_sync: Vec, + pub fn git_sync(&self) -> Option<&GitSync> { + let dags_git_sync = &self.spec.cluster_config.dags_git_sync; + // dags_git_sync is a list but only the first element is considered + // (this avoids a later breaking change when all list elements are processed) + if dags_git_sync.len() > 1 { + tracing::warn!( + "{:?} git-sync elements: only first will be considered...", + dags_git_sync.len() + ); + } + dags_git_sync.first() + } - /// for internal use only - not for production use. - #[serde(default)] - pub expose_config: bool, + /// The name of the role-level load-balanced Kubernetes `Service` + pub fn node_role_service_name(&self) -> Option { + self.metadata.name.clone() + } - /// Whether to load example DAGs or not; defaults to false. The examples are used in the - /// [getting started guide](DOCS_BASE_URL_PLACEHOLDER/airflow/getting_started/). - #[serde(default)] - pub load_examples: bool, + /// Retrieve and merge resource configs for role and role groups + pub fn merged_config( + &self, + role: &AirflowRole, + rolegroup_ref: &RoleGroupRef, + ) -> Result { + // Initialize the result with all default values as baseline + let conf_defaults = AirflowConfig::default_config(&self.name_any(), role); - /// This field controls which type of Service the Operator creates for this AirflowCluster: - /// - /// * cluster-internal: Use a ClusterIP service - /// - /// * external-unstable: Use a NodePort service - /// - /// * external-stable: Use a LoadBalancer service - /// - /// This is a temporary solution with the goal to keep yaml manifests forward compatible. - /// In the future, this setting will control which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) - /// will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change. - #[serde(default)] - pub listener_class: CurrentlySupportedListenerClasses, + let role = match role { + AirflowRole::Webserver => { + self.spec + .webservers + .as_ref() + .context(UnknownAirflowRoleSnafu { + role: role.to_string(), + roles: AirflowRole::roles(), + })? + } + AirflowRole::Worker => { + if let AirflowExecutor::CeleryExecutor { config } = &self.spec.executor { + config + } else { + return Err(Error::NoRoleForExecutorFailure); + } + } + AirflowRole::Scheduler => { + self.spec + .schedulers + .as_ref() + .context(UnknownAirflowRoleSnafu { + role: role.to_string(), + roles: AirflowRole::roles(), + })? + } + }; - /// Name of the Vector aggregator [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery). - /// It must contain the key `ADDRESS` with the address of the Vector aggregator. - /// Follow the [logging tutorial](DOCS_BASE_URL_PLACEHOLDER/tutorials/logging-vector-aggregator) - /// to learn how to configure log aggregation with Vector. - #[serde(skip_serializing_if = "Option::is_none")] - pub vector_aggregator_config_map_name: Option, + // Retrieve role resource config + let mut conf_role = role.config.config.to_owned(); - /// Additional volumes to define. Use together with `volumeMounts` to mount the volumes. - #[serde(default)] - #[schemars(schema_with = "raw_object_list_schema")] - pub volumes: Vec, + // Retrieve rolegroup specific resource config + let mut conf_rolegroup = role + .role_groups + .get(&rolegroup_ref.role_group) + .map(|rg| rg.config.config.clone()) + .unwrap_or_default(); - /// Additional volumes to mount. Use together with `volumes` to define volumes. - #[serde(default)] - #[schemars(schema_with = "raw_object_list_schema")] - pub volume_mounts: Vec, + // Merge more specific configs into default config + // Hierarchy is: + // 1. RoleGroup + // 2. Role + // 3. Default + conf_role.merge(&conf_defaults); + conf_rolegroup.merge(&conf_role); + + tracing::debug!("Merged config: {:?}", conf_rolegroup); + fragment::validate(conf_rolegroup).context(FragmentValidationFailureSnafu) + } + + /// Retrieve and merge resource configs for the executor template + pub fn merged_executor_config( + &self, + config: &ExecutorConfigFragment, + ) -> Result { + // use the worker defaults for executor pods + let resources = default_resources(&AirflowRole::Worker); + let logging = product_logging::spec::default_logging(); + let affinity = get_executor_affinity(&self.name_any()); + let graceful_shutdown_timeout = Some(DEFAULT_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT); + + let executor_defaults = ExecutorConfigFragment { + resources, + logging, + affinity, + graceful_shutdown_timeout, + }; + + let mut conf_executor = config.to_owned(); + conf_executor.merge(&executor_defaults); + + tracing::debug!("Merged executor config: {:?}", conf_executor); + fragment::validate(conf_executor).context(FragmentValidationFailureSnafu) + } } // TODO: Temporary solution until listener-operator is finished @@ -456,146 +606,6 @@ pub enum AirflowExecutor { }, } -impl AirflowCluster { - /// the worker role will not be returned if airflow provisions pods as needed (i.e. when - /// the kubernetes executor is specified) - pub fn get_role(&self, role: &AirflowRole) -> Option<&Role> { - match role { - AirflowRole::Webserver => self.spec.webservers.as_ref(), - AirflowRole::Scheduler => self.spec.schedulers.as_ref(), - AirflowRole::Worker => { - if let AirflowExecutor::CeleryExecutor { config } = &self.spec.executor { - Some(config) - } else { - None - } - } - } - } - - pub fn role_config(&self, role: &AirflowRole) -> Option<&GenericRoleConfig> { - self.get_role(role).map(|r| &r.role_config) - } - - pub fn volumes(&self) -> &Vec { - &self.spec.cluster_config.volumes - } - - pub fn volume_mounts(&self) -> Vec { - let mut mounts = self.spec.cluster_config.volume_mounts.clone(); - if self.git_sync().is_some() { - mounts.push(VolumeMount { - name: GIT_CONTENT.into(), - mount_path: GIT_SYNC_DIR.into(), - ..VolumeMount::default() - }); - } - mounts - } - - pub fn git_sync(&self) -> Option<&GitSync> { - let dags_git_sync = &self.spec.cluster_config.dags_git_sync; - // dags_git_sync is a list but only the first element is considered - // (this avoids a later breaking change when all list elements are processed) - if dags_git_sync.len() > 1 { - tracing::warn!( - "{:?} git-sync elements: only first will be considered...", - dags_git_sync.len() - ); - } - dags_git_sync.first() - } - - /// The name of the role-level load-balanced Kubernetes `Service` - pub fn node_role_service_name(&self) -> Option { - self.metadata.name.clone() - } - - /// Retrieve and merge resource configs for role and role groups - pub fn merged_config( - &self, - role: &AirflowRole, - rolegroup_ref: &RoleGroupRef, - ) -> Result { - // Initialize the result with all default values as baseline - let conf_defaults = AirflowConfig::default_config(&self.name_any(), role); - - let role = match role { - AirflowRole::Webserver => { - self.spec - .webservers - .as_ref() - .context(UnknownAirflowRoleSnafu { - role: role.to_string(), - roles: AirflowRole::roles(), - })? - } - AirflowRole::Worker => { - if let AirflowExecutor::CeleryExecutor { config } = &self.spec.executor { - config - } else { - return Err(Error::NoRoleForExecutorFailure); - } - } - AirflowRole::Scheduler => { - self.spec - .schedulers - .as_ref() - .context(UnknownAirflowRoleSnafu { - role: role.to_string(), - roles: AirflowRole::roles(), - })? - } - }; - - // Retrieve role resource config - let mut conf_role = role.config.config.to_owned(); - - // Retrieve rolegroup specific resource config - let mut conf_rolegroup = role - .role_groups - .get(&rolegroup_ref.role_group) - .map(|rg| rg.config.config.clone()) - .unwrap_or_default(); - - // Merge more specific configs into default config - // Hierarchy is: - // 1. RoleGroup - // 2. Role - // 3. Default - conf_role.merge(&conf_defaults); - conf_rolegroup.merge(&conf_role); - - tracing::debug!("Merged config: {:?}", conf_rolegroup); - fragment::validate(conf_rolegroup).context(FragmentValidationFailureSnafu) - } - - /// Retrieve and merge resource configs for the executor template - pub fn merged_executor_config( - &self, - config: &ExecutorConfigFragment, - ) -> Result { - // use the worker defaults for executor pods - let resources = default_resources(&AirflowRole::Worker); - let logging = product_logging::spec::default_logging(); - let affinity = get_executor_affinity(&self.name_any()); - let graceful_shutdown_timeout = Some(DEFAULT_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT); - - let executor_defaults = ExecutorConfigFragment { - resources, - logging, - affinity, - graceful_shutdown_timeout, - }; - - let mut conf_executor = config.to_owned(); - conf_executor.merge(&executor_defaults); - - tracing::debug!("Merged executor config: {:?}", conf_executor); - fragment::validate(conf_executor).context(FragmentValidationFailureSnafu) - } -} - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, Debug, Default, JsonSchema, PartialEq, Fragment)] #[fragment_attrs( @@ -649,7 +659,7 @@ pub enum Container { ), serde(rename_all = "camelCase") )] -pub struct AirflowConfig { +pub struct ExecutorConfig { #[fragment_attrs(serde(default))] pub resources: Resources, @@ -678,7 +688,7 @@ pub struct AirflowConfig { ), serde(rename_all = "camelCase") )] -pub struct ExecutorConfig { +pub struct AirflowConfig { #[fragment_attrs(serde(default))] pub resources: Resources, @@ -712,6 +722,40 @@ impl AirflowConfig { } } +impl Configuration for AirflowConfigFragment { + type Configurable = v1alpha1::AirflowCluster; + + fn compute_env( + &self, + cluster: &Self::Configurable, + _role_name: &str, + ) -> Result>, product_config_utils::Error> { + let mut env: BTreeMap> = BTreeMap::new(); + env.insert( + AirflowConfig::CREDENTIALS_SECRET_PROPERTY.to_string(), + Some(cluster.spec.cluster_config.credentials_secret.clone()), + ); + Ok(env) + } + + fn compute_cli( + &self, + _cluster: &Self::Configurable, + _role_name: &str, + ) -> Result>, product_config_utils::Error> { + Ok(BTreeMap::new()) + } + + fn compute_files( + &self, + _cluster: &Self::Configurable, + _role_name: &str, + _file: &str, + ) -> Result>, product_config_utils::Error> { + Ok(BTreeMap::new()) + } +} + fn default_resources(role: &AirflowRole) -> ResourcesFragment { let (cpu, memory) = match role { AirflowRole::Worker => ( @@ -753,56 +797,6 @@ fn default_resources(role: &AirflowRole) -> ResourcesFragment Result>, product_config_utils::Error> { - let mut env: BTreeMap> = BTreeMap::new(); - env.insert( - AirflowConfig::CREDENTIALS_SECRET_PROPERTY.to_string(), - Some(cluster.spec.cluster_config.credentials_secret.clone()), - ); - Ok(env) - } - - fn compute_cli( - &self, - _cluster: &Self::Configurable, - _role_name: &str, - ) -> Result>, product_config_utils::Error> { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - _cluster: &Self::Configurable, - _role_name: &str, - _file: &str, - ) -> Result>, product_config_utils::Error> { - Ok(BTreeMap::new()) - } -} - -#[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct AirflowClusterStatus { - #[serde(default)] - pub conditions: Vec, -} - -impl HasStatusCondition for AirflowCluster { - fn conditions(&self) -> Vec { - match &self.status { - Some(status) => status.conditions.clone(), - None => vec![], - } - } -} - /// Creates recommended `ObjectLabels` to be used in deployed resources pub fn build_recommended_labels<'a, T>( owner: &'a T, @@ -826,7 +820,7 @@ pub fn build_recommended_labels<'a, T>( mod tests { use stackable_operator::commons::product_image_selection::ResolvedProductImage; - use crate::AirflowCluster; + use crate::v1alpha1::AirflowCluster; #[test] fn test_cluster_config() { diff --git a/rust/operator-binary/src/env_vars.rs b/rust/operator-binary/src/env_vars.rs index a19ebdfb..29089d47 100644 --- a/rust/operator-binary/src/env_vars.rs +++ b/rust/operator-binary/src/env_vars.rs @@ -1,51 +1,56 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use product_config::types::PropertyNameKind; -use stackable_airflow_crd::{ - authentication::{ - AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, - }, - git_sync::GitSync, - AirflowCluster, AirflowConfig, AirflowExecutor, AirflowRole, ExecutorConfig, GIT_LINK, - GIT_SYNC_DIR, LOG_CONFIG_DIR, STACKABLE_LOG_DIR, TEMPLATE_LOCATION, TEMPLATE_NAME, -}; use stackable_operator::{ commons::authentication::oidc, k8s_openapi::api::core::v1::EnvVar, kube::ResourceExt, product_logging::framework::create_vector_shutdown_file_command, }; -use crate::util::env_var_from_secret; +use crate::{ + crd::{ + authentication::{ + AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, + }, + git_sync::{GitSync, GIT_SYNC_DIR, GIT_SYNC_LINK}, + v1alpha1, AirflowConfig, AirflowExecutor, AirflowRole, ExecutorConfig, LOG_CONFIG_DIR, + STACKABLE_LOG_DIR, TEMPLATE_LOCATION, TEMPLATE_NAME, + }, + util::env_var_from_secret, +}; -const AIRFLOW__LOGGING__LOGGING_CONFIG_CLASS: &str = "AIRFLOW__LOGGING__LOGGING_CONFIG_CLASS"; -const AIRFLOW__METRICS__STATSD_ON: &str = "AIRFLOW__METRICS__STATSD_ON"; -const AIRFLOW__METRICS__STATSD_HOST: &str = "AIRFLOW__METRICS__STATSD_HOST"; -const AIRFLOW__METRICS__STATSD_PORT: &str = "AIRFLOW__METRICS__STATSD_PORT"; -const GITSYNC_USERNAME: &str = "GITSYNC_USERNAME"; -const GITSYNC_PASSWORD: &str = "GITSYNC_PASSWORD"; -const AIRFLOW__API__AUTH_BACKEND: &str = "AIRFLOW__API__AUTH_BACKEND"; -const AIRFLOW__WEBSERVER__SECRET_KEY: &str = "AIRFLOW__WEBSERVER__SECRET_KEY"; -const AIRFLOW__CORE__SQL_ALCHEMY_CONN: &str = "AIRFLOW__CORE__SQL_ALCHEMY_CONN"; -const AIRFLOW__CELERY__RESULT_BACKEND: &str = "AIRFLOW__CELERY__RESULT_BACKEND"; -const AIRFLOW__CELERY__BROKER_URL: &str = "AIRFLOW__CELERY__BROKER_URL"; -const AIRFLOW__CORE__DAGS_FOLDER: &str = "AIRFLOW__CORE__DAGS_FOLDER"; -const PYTHONPATH: &str = "PYTHONPATH"; -const AIRFLOW__CORE__LOAD_EXAMPLES: &str = "AIRFLOW__CORE__LOAD_EXAMPLES"; -const AIRFLOW__WEBSERVER__EXPOSE_CONFIG: &str = "AIRFLOW__WEBSERVER__EXPOSE_CONFIG"; -const AIRFLOW__CORE__EXECUTOR: &str = "AIRFLOW__CORE__EXECUTOR"; -const AIRFLOW__KUBERNETES_EXECUTOR__POD_TEMPLATE_FILE: &str = +const AIRFLOW_LOGGING_LOGGING_CONFIG_CLASS: &str = "AIRFLOW__LOGGING__LOGGING_CONFIG_CLASS"; +const AIRFLOW_METRICS_STATSD_ON: &str = "AIRFLOW__METRICS__STATSD_ON"; +const AIRFLOW_METRICS_STATSD_HOST: &str = "AIRFLOW__METRICS__STATSD_HOST"; +const AIRFLOW_METRICS_STATSD_PORT: &str = "AIRFLOW__METRICS__STATSD_PORT"; +const AIRFLOW_API_AUTH_BACKEND: &str = "AIRFLOW__API__AUTH_BACKEND"; +const AIRFLOW_WEBSERVER_SECRET_KEY: &str = "AIRFLOW__WEBSERVER__SECRET_KEY"; +const AIRFLOW_CORE_SQL_ALCHEMY_CONN: &str = "AIRFLOW__CORE__SQL_ALCHEMY_CONN"; +const AIRFLOW_CELERY_RESULT_BACKEND: &str = "AIRFLOW__CELERY__RESULT_BACKEND"; +const AIRFLOW_CELERY_BROKER_URL: &str = "AIRFLOW__CELERY__BROKER_URL"; +const AIRFLOW_CORE_DAGS_FOLDER: &str = "AIRFLOW__CORE__DAGS_FOLDER"; +const AIRFLOW_CORE_LOAD_EXAMPLES: &str = "AIRFLOW__CORE__LOAD_EXAMPLES"; +const AIRFLOW_WEBSERVER_EXPOSE_CONFIG: &str = "AIRFLOW__WEBSERVER__EXPOSE_CONFIG"; +const AIRFLOW_CORE_EXECUTOR: &str = "AIRFLOW__CORE__EXECUTOR"; +const AIRFLOW_KUBERNETES_EXECUTOR_POD_TEMPLATE_FILE: &str = "AIRFLOW__KUBERNETES_EXECUTOR__POD_TEMPLATE_FILE"; -const AIRFLOW__KUBERNETES_EXECUTOR__NAMESPACE: &str = "AIRFLOW__KUBERNETES_EXECUTOR__NAMESPACE"; -const ADMIN_USERNAME: &str = "ADMIN_USERNAME"; +const AIRFLOW_KUBERNETES_EXECUTOR_NAMESPACE: &str = "AIRFLOW__KUBERNETES_EXECUTOR__NAMESPACE"; + const ADMIN_FIRSTNAME: &str = "ADMIN_FIRSTNAME"; +const ADMIN_USERNAME: &str = "ADMIN_USERNAME"; const ADMIN_LASTNAME: &str = "ADMIN_LASTNAME"; -const ADMIN_EMAIL: &str = "ADMIN_EMAIL"; const ADMIN_PASSWORD: &str = "ADMIN_PASSWORD"; +const ADMIN_EMAIL: &str = "ADMIN_EMAIL"; + +const GITSYNC_USERNAME: &str = "GITSYNC_USERNAME"; +const GITSYNC_PASSWORD: &str = "GITSYNC_PASSWORD"; + +const PYTHONPATH: &str = "PYTHONPATH"; /// Return environment variables to be applied to the statefulsets for the scheduler, webserver (and worker, /// for clusters utilizing `celeryExecutor`: for clusters using `kubernetesExecutor` a different set will be /// used which is defined in [`build_airflow_template_envs`]). pub fn build_airflow_statefulset_envs( - airflow: &AirflowCluster, + airflow: &v1alpha1::AirflowCluster, airflow_role: &AirflowRole, rolegroup_config: &HashMap>, executor: &AirflowExecutor, @@ -63,19 +68,19 @@ pub fn build_airflow_statefulset_envs( if let Some(secret) = secret_prop { env.insert( - AIRFLOW__WEBSERVER__SECRET_KEY.into(), + AIRFLOW_WEBSERVER_SECRET_KEY.into(), // The secret key is used to run the webserver flask app and also used to authorize // requests to Celery workers when logs are retrieved. env_var_from_secret( - AIRFLOW__WEBSERVER__SECRET_KEY, + AIRFLOW_WEBSERVER_SECRET_KEY, secret, "connections.secretKey", ), ); env.insert( - AIRFLOW__CORE__SQL_ALCHEMY_CONN.into(), + AIRFLOW_CORE_SQL_ALCHEMY_CONN.into(), env_var_from_secret( - AIRFLOW__CORE__SQL_ALCHEMY_CONN, + AIRFLOW_CORE_SQL_ALCHEMY_CONN, secret, "connections.sqlalchemyDatabaseUri", ), @@ -85,17 +90,17 @@ pub fn build_airflow_statefulset_envs( // see https://github.com/stackabletech/airflow-operator/issues/424 for details if matches!(executor, AirflowExecutor::CeleryExecutor { .. }) { env.insert( - AIRFLOW__CELERY__RESULT_BACKEND.into(), + AIRFLOW_CELERY_RESULT_BACKEND.into(), env_var_from_secret( - AIRFLOW__CELERY__RESULT_BACKEND, + AIRFLOW_CELERY_RESULT_BACKEND, secret, "connections.celeryResultBackend", ), ); env.insert( - AIRFLOW__CELERY__BROKER_URL.into(), + AIRFLOW_CELERY_BROKER_URL.into(), env_var_from_secret( - AIRFLOW__CELERY__BROKER_URL, + AIRFLOW_CELERY_BROKER_URL, secret, "connections.celeryBrokerUrl", ), @@ -105,9 +110,9 @@ pub fn build_airflow_statefulset_envs( let dags_folder = get_dags_folder(airflow); env.insert( - AIRFLOW__CORE__DAGS_FOLDER.into(), + AIRFLOW_CORE_DAGS_FOLDER.into(), EnvVar { - name: AIRFLOW__CORE__DAGS_FOLDER.into(), + name: AIRFLOW_CORE_DAGS_FOLDER.into(), value: Some(dags_folder), ..Default::default() }, @@ -115,18 +120,18 @@ pub fn build_airflow_statefulset_envs( if airflow.spec.cluster_config.load_examples { env.insert( - AIRFLOW__CORE__LOAD_EXAMPLES.into(), + AIRFLOW_CORE_LOAD_EXAMPLES.into(), EnvVar { - name: AIRFLOW__CORE__LOAD_EXAMPLES.into(), + name: AIRFLOW_CORE_LOAD_EXAMPLES.into(), value: Some("True".into()), ..Default::default() }, ); } else { env.insert( - AIRFLOW__CORE__LOAD_EXAMPLES.into(), + AIRFLOW_CORE_LOAD_EXAMPLES.into(), EnvVar { - name: AIRFLOW__CORE__LOAD_EXAMPLES.into(), + name: AIRFLOW_CORE_LOAD_EXAMPLES.into(), value: Some("False".into()), ..Default::default() }, @@ -135,9 +140,9 @@ pub fn build_airflow_statefulset_envs( if airflow.spec.cluster_config.expose_config { env.insert( - AIRFLOW__WEBSERVER__EXPOSE_CONFIG.into(), + AIRFLOW_WEBSERVER_EXPOSE_CONFIG.into(), EnvVar { - name: AIRFLOW__WEBSERVER__EXPOSE_CONFIG.into(), + name: AIRFLOW_WEBSERVER_EXPOSE_CONFIG.into(), value: Some("True".into()), ..Default::default() }, @@ -145,9 +150,9 @@ pub fn build_airflow_statefulset_envs( } env.insert( - AIRFLOW__CORE__EXECUTOR.into(), + AIRFLOW_CORE_EXECUTOR.into(), EnvVar { - name: AIRFLOW__CORE__EXECUTOR.into(), + name: AIRFLOW_CORE_EXECUTOR.into(), value: Some(executor.to_string()), ..Default::default() }, @@ -155,17 +160,17 @@ pub fn build_airflow_statefulset_envs( if let AirflowExecutor::KubernetesExecutor { .. } = executor { env.insert( - AIRFLOW__KUBERNETES_EXECUTOR__POD_TEMPLATE_FILE.into(), + AIRFLOW_KUBERNETES_EXECUTOR_POD_TEMPLATE_FILE.into(), EnvVar { - name: AIRFLOW__KUBERNETES_EXECUTOR__POD_TEMPLATE_FILE.into(), + name: AIRFLOW_KUBERNETES_EXECUTOR_POD_TEMPLATE_FILE.into(), value: Some(format!("{TEMPLATE_LOCATION}/{TEMPLATE_NAME}")), ..Default::default() }, ); env.insert( - AIRFLOW__KUBERNETES_EXECUTOR__NAMESPACE.into(), + AIRFLOW_KUBERNETES_EXECUTOR_NAMESPACE.into(), EnvVar { - name: AIRFLOW__KUBERNETES_EXECUTOR__NAMESPACE.into(), + name: AIRFLOW_KUBERNETES_EXECUTOR_NAMESPACE.into(), value: airflow.namespace(), ..Default::default() }, @@ -232,13 +237,13 @@ pub fn build_airflow_statefulset_envs( transform_map_to_vec(env) } -fn get_dags_folder(airflow: &AirflowCluster) -> String { +fn get_dags_folder(airflow: &v1alpha1::AirflowCluster) -> String { return if let Some(GitSync { git_folder: Some(dags_folder), .. }) = airflow.git_sync() { - format!("{GIT_SYNC_DIR}/{GIT_LINK}/{dags_folder}") + format!("{GIT_SYNC_DIR}/{GIT_SYNC_LINK}/{dags_folder}") } else { // if this has not been set for dag-provisioning via gitsync (above), set the default value // so that PYTHONPATH can refer to this. N.B. nested variables need to be resolved, so that @@ -250,7 +255,7 @@ fn get_dags_folder(airflow: &AirflowCluster) -> String { // This set of environment variables is a standard set that is not dependent on any // conditional logic and should be applied to the statefulset or the executor template config map. -fn static_envs(airflow: &AirflowCluster) -> BTreeMap { +fn static_envs(airflow: &v1alpha1::AirflowCluster) -> BTreeMap { let mut env: BTreeMap = BTreeMap::new(); let dags_folder = get_dags_folder(airflow); @@ -267,48 +272,48 @@ fn static_envs(airflow: &AirflowCluster) -> BTreeMap { }, ); env.insert( - AIRFLOW__LOGGING__LOGGING_CONFIG_CLASS.into(), + AIRFLOW_LOGGING_LOGGING_CONFIG_CLASS.into(), EnvVar { - name: AIRFLOW__LOGGING__LOGGING_CONFIG_CLASS.into(), + name: AIRFLOW_LOGGING_LOGGING_CONFIG_CLASS.into(), value: Some("log_config.LOGGING_CONFIG".into()), ..Default::default() }, ); env.insert( - AIRFLOW__METRICS__STATSD_ON.into(), + AIRFLOW_METRICS_STATSD_ON.into(), EnvVar { - name: AIRFLOW__METRICS__STATSD_ON.into(), + name: AIRFLOW_METRICS_STATSD_ON.into(), value: Some("True".into()), ..Default::default() }, ); env.insert( - AIRFLOW__METRICS__STATSD_HOST.into(), + AIRFLOW_METRICS_STATSD_HOST.into(), EnvVar { - name: AIRFLOW__METRICS__STATSD_HOST.into(), + name: AIRFLOW_METRICS_STATSD_HOST.into(), value: Some("0.0.0.0".into()), ..Default::default() }, ); env.insert( - AIRFLOW__METRICS__STATSD_PORT.into(), + AIRFLOW_METRICS_STATSD_PORT.into(), EnvVar { - name: AIRFLOW__METRICS__STATSD_PORT.into(), + name: AIRFLOW_METRICS_STATSD_PORT.into(), value: Some("9125".into()), ..Default::default() }, ); env.insert( - AIRFLOW__API__AUTH_BACKEND.into(), + AIRFLOW_API_AUTH_BACKEND.into(), // Authentication for the API is handled separately to the Web Authentication. // Basic authentication is used by the integration tests. // The default is to deny all requests to the API. EnvVar { - name: AIRFLOW__API__AUTH_BACKEND.into(), + name: AIRFLOW_API_AUTH_BACKEND.into(), value: Some("airflow.api.auth.backend.basic_auth".into()), ..Default::default() }, @@ -357,7 +362,7 @@ fn add_gitsync_credentials( /// Return environment variables to be applied to the configuration map used in conjunction with /// the `kubernetesExecutor` worker. pub fn build_airflow_template_envs( - airflow: &AirflowCluster, + airflow: &v1alpha1::AirflowCluster, env_overrides: &HashMap, config: &ExecutorConfig, ) -> Vec { @@ -365,27 +370,27 @@ pub fn build_airflow_template_envs( let secret = airflow.spec.cluster_config.credentials_secret.as_str(); env.insert( - AIRFLOW__CORE__SQL_ALCHEMY_CONN.into(), + AIRFLOW_CORE_SQL_ALCHEMY_CONN.into(), env_var_from_secret( - AIRFLOW__CORE__SQL_ALCHEMY_CONN, + AIRFLOW_CORE_SQL_ALCHEMY_CONN, secret, "connections.sqlalchemyDatabaseUri", ), ); env.insert( - AIRFLOW__CORE__EXECUTOR.into(), + AIRFLOW_CORE_EXECUTOR.into(), EnvVar { - name: AIRFLOW__CORE__EXECUTOR.into(), + name: AIRFLOW_CORE_EXECUTOR.into(), value: Some("LocalExecutor".to_string()), ..Default::default() }, ); env.insert( - AIRFLOW__KUBERNETES_EXECUTOR__NAMESPACE.into(), + AIRFLOW_KUBERNETES_EXECUTOR_NAMESPACE.into(), EnvVar { - name: AIRFLOW__KUBERNETES_EXECUTOR__NAMESPACE.into(), + name: AIRFLOW_KUBERNETES_EXECUTOR_NAMESPACE.into(), value: airflow.namespace(), ..Default::default() }, @@ -395,9 +400,9 @@ pub fn build_airflow_template_envs( // to the pods started by airflow. let dags_folder = get_dags_folder(airflow); env.insert( - AIRFLOW__CORE__DAGS_FOLDER.into(), + AIRFLOW_CORE_DAGS_FOLDER.into(), EnvVar { - name: AIRFLOW__CORE__DAGS_FOLDER.into(), + name: AIRFLOW_CORE_DAGS_FOLDER.into(), value: Some(dags_folder), ..Default::default() }, diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 6ff45399..869cfca5 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -1,16 +1,7 @@ -mod airflow_controller; -mod config; -mod controller_commons; -mod env_vars; -mod operations; -mod product_logging; -mod util; - use std::sync::Arc; use clap::{crate_description, crate_version, Parser}; use futures::StreamExt; -use stackable_airflow_crd::{AirflowCluster, APP_NAME, OPERATOR_NAME}; use stackable_operator::{ cli::{Command, ProductOperatorRun}, commons::authentication::AuthenticationClass, @@ -25,10 +16,23 @@ use stackable_operator::{ ResourceExt, }, logging::controller::report_controller_reconciled, - CustomResourceExt, + shared::yaml::SerializeOptions, + YamlSchema, +}; + +use crate::{ + airflow_controller::AIRFLOW_FULL_CONTROLLER_NAME, + crd::{v1alpha1, AirflowCluster, APP_NAME, OPERATOR_NAME}, }; -use crate::airflow_controller::AIRFLOW_FULL_CONTROLLER_NAME; +mod airflow_controller; +mod config; +mod controller_commons; +mod crd; +mod env_vars; +mod operations; +mod product_logging; +mod util; mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); @@ -47,7 +51,8 @@ async fn main() -> anyhow::Result<()> { match opts.cmd { Command::Crd => { - AirflowCluster::print_yaml_schema(built_info::PKG_VERSION)?; + AirflowCluster::merged_crd(AirflowCluster::V1Alpha1)? + .print_yaml_schema(built_info::PKG_VERSION, SerializeOptions::default())?; } Command::Run(ProductOperatorRun { product_config, @@ -88,7 +93,7 @@ async fn main() -> anyhow::Result<()> { )); let airflow_controller = Controller::new( - watch_namespace.get_api::>(&client), + watch_namespace.get_api::>(&client), watcher::Config::default(), ); @@ -110,9 +115,11 @@ async fn main() -> anyhow::Result<()> { airflow_store_1 .state() .into_iter() - .filter(move |airflow: &Arc>| { - references_authentication_class(airflow, &authentication_class) - }) + .filter( + move |airflow: &Arc>| { + references_authentication_class(airflow, &authentication_class) + }, + ) .map(|airflow| ObjectRef::from_obj(&*airflow)) }, ) @@ -149,7 +156,7 @@ async fn main() -> anyhow::Result<()> { } fn references_authentication_class( - airflow: &DeserializeGuard, + airflow: &DeserializeGuard, authentication_class: &DeserializeGuard, ) -> bool { let Ok(airflow) = &airflow.0 else { diff --git a/rust/operator-binary/src/operations/graceful_shutdown.rs b/rust/operator-binary/src/operations/graceful_shutdown.rs index 6d4c0dc5..dde8e074 100644 --- a/rust/operator-binary/src/operations/graceful_shutdown.rs +++ b/rust/operator-binary/src/operations/graceful_shutdown.rs @@ -1,7 +1,8 @@ use snafu::{ResultExt, Snafu}; -use stackable_airflow_crd::{AirflowConfig, ExecutorConfig}; use stackable_operator::builder::pod::PodBuilder; +use crate::crd::{AirflowConfig, ExecutorConfig}; + #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("Failed to set terminationGracePeriod"))] diff --git a/rust/operator-binary/src/operations/pdb.rs b/rust/operator-binary/src/operations/pdb.rs index fc65a062..216806ef 100644 --- a/rust/operator-binary/src/operations/pdb.rs +++ b/rust/operator-binary/src/operations/pdb.rs @@ -1,11 +1,13 @@ use snafu::{ResultExt, Snafu}; -use stackable_airflow_crd::{AirflowCluster, AirflowRole, APP_NAME, OPERATOR_NAME}; use stackable_operator::{ builder::pdb::PodDisruptionBudgetBuilder, client::Client, cluster_resources::ClusterResources, commons::pdb::PdbConfig, kube::ResourceExt, }; -use crate::airflow_controller::AIRFLOW_CONTROLLER_NAME; +use crate::{ + airflow_controller::AIRFLOW_CONTROLLER_NAME, + crd::{v1alpha1, AirflowExecutor, AirflowRole, APP_NAME, OPERATOR_NAME}, +}; #[derive(Snafu, Debug)] pub enum Error { @@ -23,7 +25,7 @@ pub enum Error { pub async fn add_pdbs( pdb: &PdbConfig, - airflow: &AirflowCluster, + airflow: &v1alpha1::AirflowCluster, role: &AirflowRole, client: &Client, cluster_resources: &mut ClusterResources, @@ -36,10 +38,8 @@ pub async fn add_pdbs( AirflowRole::Scheduler => max_unavailable_schedulers(), AirflowRole::Webserver => max_unavailable_webservers(), AirflowRole::Worker => match airflow.spec.executor { - stackable_airflow_crd::AirflowExecutor::CeleryExecutor { .. } => { - max_unavailable_workers() - } - stackable_airflow_crd::AirflowExecutor::KubernetesExecutor { .. } => { + AirflowExecutor::CeleryExecutor { .. } => max_unavailable_workers(), + AirflowExecutor::KubernetesExecutor { .. } => { // In case Airflow creates the Pods, we don't want to influence that. return Ok(()); } diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs index faedae45..3d8fe9cb 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/product_logging.rs @@ -1,7 +1,6 @@ use std::fmt::{Display, Write}; use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_airflow_crd::STACKABLE_LOG_DIR; use stackable_operator::{ builder::configmap::ConfigMapBuilder, client::Client, @@ -16,6 +15,8 @@ use stackable_operator::{ role_utils::RoleGroupRef, }; +use crate::crd::STACKABLE_LOG_DIR; + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("object has no namespace"))]