From 388c1032821d10ef46207f0df4e61878b83a0886 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Tue, 22 Oct 2024 11:06:28 +0200 Subject: [PATCH 01/25] First parts of airflow OIDC --- Cargo.lock | 490 ++++++++++++++++------------- Cargo.toml | 2 +- rust/crd/src/authentication.rs | 282 ++++++++++++----- rust/crd/src/lib.rs | 9 +- rust/operator-binary/src/config.rs | 69 ++++ 5 files changed, 540 insertions(+), 312 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6d9ce619..2e216d29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,18 +4,18 @@ version = 3 [[package]] name = "addr2line" -version = "0.22.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "ahash" @@ -96,7 +96,7 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -106,14 +106,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "37bf3594c4c988a53154954629820791dde498571819ae4ca50ca811e060cc95" [[package]] name = "async-broadcast" @@ -129,9 +129,9 @@ dependencies = [ [[package]] name = "async-stream" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" dependencies = [ "async-stream-impl", "futures-core", @@ -140,31 +140,31 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.82", ] [[package]] name = "async-trait" -version = "0.1.81" +version = "0.1.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.82", ] [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "backoff" @@ -179,17 +179,17 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", + "windows-targets", ] [[package]] @@ -242,9 +242,9 @@ dependencies = [ [[package]] name = "built" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "236e6289eda5a812bc6b53c3b024039382a2895fbbeef2d748b2931546d392c4" +checksum = "c360505aed52b7ec96a3636c3f039d99103c37d1d9b4f7a8c743d3ea9ffcd03b" dependencies = [ "chrono", "git2", @@ -264,15 +264,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "cc" -version = "1.1.13" +version = "1.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72db2f7947ecee9b03b510377e8bb9077afa27176fdbff55c51027e976fdcc48" +checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" dependencies = [ "jobserver", "libc", @@ -300,9 +300,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.17" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" dependencies = [ "clap_builder", "clap_derive", @@ -310,9 +310,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.17" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" dependencies = [ "anstream", "anstyle", @@ -322,14 +322,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.13" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.82", ] [[package]] @@ -391,9 +391,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -444,7 +444,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.77", + "syn 2.0.82", ] [[package]] @@ -455,18 +455,18 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.77", + "syn 2.0.82", ] [[package]] name = "delegate" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5060bb0febb73fa907273f8a7ed17ab4bf831d585eac835b28ec24a1e2460956" +checksum = "bc2323e10c92e1cf4d86e11538512e6dc03ceb586842970b6332af3d4046a046" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.82", ] [[package]] @@ -583,8 +583,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" dependencies = [ "bit-set", - "regex-automata 0.4.7", - "regex-syntax 0.8.4", + "regex-automata 0.4.8", + "regex-syntax 0.8.5", ] [[package]] @@ -619,9 +619,9 @@ checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -634,9 +634,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -644,15 +644,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -661,32 +661,32 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.82", ] [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-timer" @@ -696,9 +696,9 @@ checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures 0.1.31", "futures-channel", @@ -736,9 +736,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.29.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "git2" @@ -769,6 +769,12 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "hashbrown" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" + [[package]] name = "headers" version = "0.4.0" @@ -811,7 +817,7 @@ version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -850,9 +856,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "httpdate" @@ -862,9 +868,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" dependencies = [ "bytes", "futures-channel", @@ -893,7 +899,7 @@ dependencies = [ "hyper-rustls", "hyper-util", "pin-project-lite", - "rustls-native-certs", + "rustls-native-certs 0.7.3", "tokio", "tokio-rustls", "tower-service", @@ -901,9 +907,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.2" +version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", "http", @@ -911,7 +917,7 @@ dependencies = [ "hyper-util", "log", "rustls", - "rustls-native-certs", + "rustls-native-certs 0.8.0", "rustls-pki-types", "tokio", "tokio-rustls", @@ -933,9 +939,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.7" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" +checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" dependencies = [ "bytes", "futures-channel", @@ -946,16 +952,15 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower", "tower-service", "tracing", ] [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -992,12 +997,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.4.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.15.0", ] [[package]] @@ -1049,9 +1054,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.70" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" dependencies = [ "wasm-bindgen", ] @@ -1131,7 +1136,7 @@ dependencies = [ "bytes", "chrono", "either", - "futures 0.3.30", + "futures 0.3.31", "home", "http", "http-body", @@ -1187,7 +1192,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.77", + "syn 2.0.82", ] [[package]] @@ -1202,8 +1207,8 @@ dependencies = [ "async-trait", "backoff", "derivative", - "futures 0.3.30", - "hashbrown", + "futures 0.3.31", + "hashbrown 0.14.5", "json-patch", "jsonptr", "k8s-openapi", @@ -1226,9 +1231,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.158" +version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" [[package]] name = "libgit2-sys" @@ -1244,9 +1249,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.19" +version = "1.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc53a7799a7496ebc9fd29f31f7df80e83c9bda5299768af5f9e59eeea74647" +checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" dependencies = [ "cc", "libc", @@ -1293,11 +1298,11 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.7.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ - "adler", + "adler2", ] [[package]] @@ -1309,7 +1314,7 @@ dependencies = [ "hermit-abi", "libc", "wasi", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -1349,18 +1354,18 @@ dependencies = [ [[package]] name = "object" -version = "0.36.3" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "openssl-probe" @@ -1418,7 +1423,7 @@ dependencies = [ "lazy_static", "once_cell", "opentelemetry", - "ordered-float 4.2.2", + "ordered-float 4.4.0", "percent-encoding", "rand", "thiserror", @@ -1437,9 +1442,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "4.2.2" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a91171844676f8c7990ce64959210cd2eaef32c2612c50f9fae9f8aaa6065a6" +checksum = "83e7ccb95e240b7c9506a3d544f10d935e142cc90b0a1d56954fb44d89ad6b97" dependencies = [ "num-traits", ] @@ -1452,9 +1457,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "parking" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" @@ -1497,9 +1502,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.11" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd53dff83f26735fdc1ca837098ccf133605d794cdae66acfc2bfac3ec809d95" +checksum = "879952a81a83930934cbf1786752d6dedc3b1f29e8f8fb2ad1d0a36f377cf442" dependencies = [ "memchr", "thiserror", @@ -1508,9 +1513,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.11" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a548d2beca6773b1c244554d36fcf8548a8a58e74156968211567250e48e49a" +checksum = "d214365f632b123a47fd913301e14c946c61d1c183ee245fa76eb752e59a02dd" dependencies = [ "pest", "pest_generator", @@ -1518,22 +1523,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.11" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c93a82e8d145725dcbaf44e5ea887c8a869efdcc28706df2d08c69e17077183" +checksum = "eb55586734301717aea2ac313f50b2eb8f60d2fc3dc01d190eefa2e625f60c4e" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.82", ] [[package]] name = "pest_meta" -version = "2.7.11" +version = "2.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a941429fea7e08bedec25e4f6785b6ffaacc6b755da98df5ef3e7dcf4a124c4f" +checksum = "b75da2a70cf4d9cb76833c990ac9cd3923c9a8905a8929789ce347c84564d03d" dependencies = [ "once_cell", "pest", @@ -1542,22 +1547,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.82", ] [[package]] @@ -1574,9 +1579,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "powerfmt" @@ -1595,18 +1600,18 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" +checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" dependencies = [ "toml_edit", ] [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "7c3a7fc5db1e57d5a779a352c8cdb57b29aa4c40cc69c3a68a7fedc815fbf2f9" dependencies = [ "unicode-ident", ] @@ -1623,7 +1628,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu 0.8.4", + "snafu 0.8.5", "xml-rs", ] @@ -1668,23 +1673,23 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.3" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" dependencies = [ "bitflags 2.6.0", ] [[package]] name = "regex" -version = "1.10.6" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.7", - "regex-syntax 0.8.4", + "regex-automata 0.4.8", + "regex-syntax 0.8.5", ] [[package]] @@ -1698,13 +1703,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -1715,9 +1720,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "relative-path" @@ -1737,7 +1742,7 @@ dependencies = [ "libc", "spin", "untrusted", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -1746,7 +1751,7 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b423f0e62bdd61734b67cd21ff50871dfaeb9cc74f869dcd6af974fbcb19936" dependencies = [ - "futures 0.3.30", + "futures 0.3.31", "futures-timer", "rstest_macros", "rustc_version", @@ -1766,7 +1771,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.77", + "syn 2.0.82", "unicode-ident", ] @@ -1778,18 +1783,18 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] [[package]] name = "rustls" -version = "0.23.12" +version = "0.23.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +checksum = "5fbb44d7acc4e873d613422379f69f237a1b141928c02f6bc6ccfddddc2d7993" dependencies = [ "log", "once_cell", @@ -1802,9 +1807,22 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.2" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04182dffc9091a404e0fc069ea5cd60e5b866c3adf881eff99a32d048242dffa" +checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" dependencies = [ "openssl-probe", "rustls-pemfile", @@ -1815,25 +1833,24 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.3" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64 0.22.1", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" [[package]] name = "rustls-webpki" -version = "0.102.6" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ "ring", "rustls-pki-types", @@ -1842,9 +1859,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" [[package]] name = "ryu" @@ -1854,11 +1871,11 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -1883,7 +1900,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.77", + "syn 2.0.82", ] [[package]] @@ -1917,9 +1934,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" +checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" dependencies = [ "core-foundation-sys", "libc", @@ -1958,7 +1975,7 @@ checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.82", ] [[package]] @@ -1969,14 +1986,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.82", ] [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", "memchr", @@ -2070,11 +2087,11 @@ dependencies = [ [[package]] name = "snafu" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b835cb902660db3415a672d862905e791e54d306c6e8189168c7f3d9ae1c79d" +checksum = "223891c85e2a29c3fe8fb900c1fae5e69c2e42415e3177752e8718475efa5019" dependencies = [ - "snafu-derive 0.8.4", + "snafu-derive 0.8.5", ] [[package]] @@ -2090,14 +2107,14 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d1e02fca405f6280643174a50c942219f0bbf4dbf7d480f1dd864d6f211ae5" +checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.82", ] [[package]] @@ -2107,7 +2124,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -2125,7 +2142,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu 0.8.4", + "snafu 0.8.5", "stackable-operator", "strum", "tracing", @@ -2139,11 +2156,11 @@ dependencies = [ "built", "clap", "fnv", - "futures 0.3.30", + "futures 0.3.31", "product-config", "serde", "serde_yaml", - "snafu 0.8.4", + "snafu 0.8.5", "stackable-airflow-crd", "stackable-operator", "strum", @@ -2153,8 +2170,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.76.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.76.0#a7e70f174fb043a1766e0a80de95834cb4f7513d" +version = "0.79.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.79.0#a2ac5f525edfd9a2fab884ba753e79a325fbb752" dependencies = [ "chrono", "clap", @@ -2163,7 +2180,8 @@ dependencies = [ "derivative", "dockerfile-parser", "either", - "futures 0.3.30", + "futures 0.3.31", + "indexmap", "json-patch", "k8s-openapi", "kube", @@ -2176,8 +2194,9 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu 0.8.4", + "snafu 0.8.5", "stackable-operator-derive", + "stackable-shared", "strum", "tokio", "tracing", @@ -2190,12 +2209,24 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.76.0#a7e70f174fb043a1766e0a80de95834cb4f7513d" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.79.0#a2ac5f525edfd9a2fab884ba753e79a325fbb752" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.82", +] + +[[package]] +name = "stackable-shared" +version = "0.0.1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.79.0#a2ac5f525edfd9a2fab884ba753e79a325fbb752" +dependencies = [ + "kube", + "semver", + "serde", + "serde_yaml", + "snafu 0.8.5", ] [[package]] @@ -2223,7 +2254,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.77", + "syn 2.0.82", ] [[package]] @@ -2245,9 +2276,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.77" +version = "2.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +checksum = "83540f837a8afc019423a8edb95b52a8effe46957ee402287f4292fae35be021" dependencies = [ "proc-macro2", "quote", @@ -2256,22 +2287,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.82", ] [[package]] @@ -2367,7 +2398,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -2378,7 +2409,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.82", ] [[package]] @@ -2394,9 +2425,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" dependencies = [ "futures-core", "pin-project-lite", @@ -2405,9 +2436,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", @@ -2425,9 +2456,9 @@ checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" [[package]] name = "toml_edit" -version = "0.21.1" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ "indexmap", "toml_datetime", @@ -2514,7 +2545,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.82", ] [[package]] @@ -2601,36 +2632,36 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "ucd-trie" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] [[package]] name = "unicode-xid" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229730647fbc343e3a80e463c1db7f78f3855d3f3739bee0dda773c9a037c90a" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "unsafe-libyaml" @@ -2697,9 +2728,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", "once_cell", @@ -2708,24 +2739,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.82", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2733,22 +2764,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.82", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "web-time" @@ -2800,6 +2831,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -2866,18 +2906,18 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.5.40" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" dependencies = [ "memchr", ] [[package]] name = "xml-rs" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "539a77ee7c0de333dcc6da69b177380a0b81e0dacfa4f7344c465a36871ee601" +checksum = "af4e2e2f7cba5a093896c1e150fbfe177d1883e7448200efb81d40b9d339ef26" [[package]] name = "zerocopy" @@ -2897,7 +2937,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.82", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9b1beacf..e5f902db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.8" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.76.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.79.0" } strum = { version = "0.26", features = ["derive"] } tokio = { version = "1.40", features = ["full"] } tracing = "0.1" diff --git a/rust/crd/src/authentication.rs b/rust/crd/src/authentication.rs index eee0cd06..2f7d3d7f 100644 --- a/rust/crd/src/authentication.rs +++ b/rust/crd/src/authentication.rs @@ -1,21 +1,36 @@ +use std::{future::Future, mem}; + use serde::{Deserialize, Serialize}; -use snafu::{ResultExt, Snafu}; -use stackable_operator::commons::authentication::AuthenticationClassProvider; +use snafu::{ensure, ResultExt, Snafu}; use stackable_operator::{ client::Client, - commons::authentication::AuthenticationClass, + commons::authentication::{ + ldap, + oidc::{self, IdentityProviderHint}, + AuthenticationClass, AuthenticationClassProvider, ClientAuthenticationDetails, + }, kube::runtime::reflector::ObjectRef, schemars::{self, JsonSchema}, }; +use std::collections::BTreeSet; +use tracing::info; -const SUPPORTED_AUTHENTICATION_CLASS_PROVIDERS: [&str; 1] = ["LDAP"]; +const SUPPORTED_AUTHENTICATION_CLASS_PROVIDERS: [&str; 2] = ["LDAP", "OIDC"]; +const SUPPORTED_OIDC_PROVIDERS: &[oidc::IdentityProviderHint] = + &[oidc::IdentityProviderHint::Keycloak]; +// The assumed OIDC provider if no hint is given in the AuthClass +pub const DEFAULT_OIDC_PROVIDER: oidc::IdentityProviderHint = oidc::IdentityProviderHint::Keycloak; #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("Failed to retrieve AuthenticationClass {authentication_class}"))] - AuthenticationClassRetrieval { + #[snafu(display( + "The AuthenticationClass {auth_class_name:?} is referenced several times which is not allowed." + ))] + DuplicateAuthenticationClassReferencesNotAllowed { auth_class_name: String }, + + #[snafu(display("Failed to retrieve AuthenticationClass"))] + AuthenticationClassRetrievalFailed { source: stackable_operator::client::Error, - authentication_class: ObjectRef, }, // TODO: Adapt message if multiple authentication classes are supported simultaneously #[snafu(display("Only one authentication class is currently supported at a time"))] @@ -24,50 +39,56 @@ pub enum Error { "Failed to use authentication provider [{provider}] for authentication class [{authentication_class}] - supported providers: {SUPPORTED_AUTHENTICATION_CLASS_PROVIDERS:?}", ))] AuthenticationProviderNotSupported { - authentication_class: ObjectRef, + auth_class_name: String, provider: String, }, + #[snafu(display("Only one authentication type at a time is supported by Airflow, see https://github.com/dpgaspar/Flask-AppBuilder/issues/1924."))] + MultipleAuthenticationTypesNotSupported, + #[snafu(display("Only one LDAP provider at a time is supported by Airflow."))] + MultipleLdapProvidersNotSupported, + #[snafu(display("The OIDC provider {oidc_provider:?} is not yet supported (AuthenticationClass {auth_class_name:?})."))] + OidcProviderNotSupported { + auth_class_name: String, + oidc_provider: String, + }, + #[snafu(display( + "TLS verification cannot be disabled in Superset (AuthenticationClass {auth_class_name:?})." + ))] + TlsVerificationCannotBeDisabled { auth_class_name: String }, + #[snafu(display("Invalid OIDC configuration"))] + OidcConfigurationInvalid { + source: stackable_operator::commons::authentication::Error, + }, + #[snafu(display( + "{configured:?} is not a supported principalClaim in Airflow for the Keycloak OIDC provider. Please use {supported:?} in the AuthenticationClass {auth_class_name:?}" + ))] + OidcPrincipalClaimNotSupported { + configured: String, + supported: String, + auth_class_name: String, + }, } type Result = std::result::Result; - -/// Resolved counter part for `AirflowAuthenticationConfig`. -pub struct AirflowAuthenticationConfigResolved { - pub authentication_class: Option, - pub user_registration: bool, - pub user_registration_role: String, - pub sync_roles_at: FlaskRolesSyncMoment, -} - #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] -pub struct AirflowAuthentication { - /// The Airflow [authentication](DOCS_BASE_URL_PLACEHOLDER/airflow/usage-guide/security.html) settings. - /// Currently the underlying Flask App Builder only supports one authentication mechanism - /// at a time. This means the operator will error out if multiple references to an - /// AuthenticationClass are provided. - #[serde(default)] - authentication: Vec, -} +pub struct AirflowClientAuthenticationDetails { + #[serde(flatten)] + pub common: ClientAuthenticationDetails<()>, -#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct AirflowAuthenticationConfig { - /// Name of the [AuthenticationClass](DOCS_BASE_URL_PLACEHOLDER/concepts/authentication.html#authenticationclass) used to authenticate the users. - /// At the moment only LDAP is supported. - /// If not specified the default authentication (AUTH_DB) will be used. - pub authentication_class: Option, /// Allow users who are not already in the FAB DB. /// Gets mapped to `AUTH_USER_REGISTRATION` #[serde(default = "default_user_registration")] pub user_registration: bool, + /// This role will be given in addition to any AUTH_ROLES_MAPPING. /// Gets mapped to `AUTH_USER_REGISTRATION_ROLE` #[serde(default = "default_user_registration_role")] pub user_registration_role: String, + /// If we should replace ALL the user's roles each login, or only on registration. /// Gets mapped to `AUTH_ROLES_SYNC_AT_LOGIN` - #[serde(default = "default_sync_roles_at")] + #[serde(default)] pub sync_roles_at: FlaskRolesSyncMoment, } @@ -84,69 +105,164 @@ pub fn default_sync_roles_at() -> FlaskRolesSyncMoment { FlaskRolesSyncMoment::Registration } -#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize, Default)] pub enum FlaskRolesSyncMoment { + #[default] Registration, Login, } -impl AirflowAuthentication { - pub fn authentication_class_names(&self) -> Vec<&str> { - let mut auth_classes = vec![]; - for config in &self.authentication { - if let Some(auth_config) = &config.authentication_class { - auth_classes.push(auth_config.as_str()); +/// Resolved and validated counter part for `AirflowClientAuthenticationDetails`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AirflowClientAuthenticationDetailsResolved { + pub authentication_classes_resolved: Vec, + pub user_registration: bool, + pub user_registration_role: String, + pub sync_roles_at: FlaskRolesSyncMoment, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum AirflowAuthenticationClassResolved { + Ldap { + provider: ldap::AuthenticationProvider, + }, + Oidc { + provider: oidc::AuthenticationProvider, + oidc: oidc::ClientAuthenticationOptions<()>, + }, +} + +impl AirflowClientAuthenticationDetailsResolved { + pub async fn from( + auth_details: &[AirflowClientAuthenticationDetails], + client: &Client, + ) -> Result { + let resolve_auth_class = |auth_details: ClientAuthenticationDetails| async move { + auth_details.resolve_class(client).await + }; + AirflowClientAuthenticationDetailsResolved::resolve(auth_details, resolve_auth_class).await + } + pub async fn resolve( + auth_details: &[AirflowClientAuthenticationDetails], + resolve_auth_class: impl Fn(ClientAuthenticationDetails) -> R, + ) -> Result + where + R: Future>, + { + let mut resolved_auth_classes: Vec = Vec::new(); + let mut user_registration = None; + let mut user_registration_role = None; + let mut sync_roles_at = None; + + let mut auth_class_names = BTreeSet::new(); + + for entry in auth_details { + let auth_class_name = entry.common.authentication_class_name(); + + let is_new_auth_class = auth_class_names.insert(auth_class_name); + ensure!( + is_new_auth_class, + DuplicateAuthenticationClassReferencesNotAllowedSnafu { auth_class_name } + ); + + let auth_class = resolve_auth_class(entry.common.clone()) + .await + .context(AuthenticationClassRetrievalFailedSnafu)?; + + match &auth_class.spec.provider { + AuthenticationClassProvider::Ldap(provider) => { + let resolved_auth_class = AirflowAuthenticationClassResolved::Ldap { + provider: provider.to_owned(), + }; + if let Some(other) = resolved_auth_classes.first() { + ensure!( + mem::discriminant(other) == mem::discriminant(&resolved_auth_class), + MultipleAuthenticationTypesNotSupportedSnafu + ); + } + + ensure!( + resolved_auth_classes.is_empty(), + MultipleLdapProvidersNotSupportedSnafu + ); + + resolved_auth_classes.push(resolved_auth_class); + } + AuthenticationClassProvider::Oidc(provider) => { + let resolved_auth_class = + AirflowClientAuthenticationDetailsResolved::from_oidc( + auth_class_name, + provider, + entry, + )?; + + if let Some(other) = resolved_auth_classes.first() { + ensure!( + mem::discriminant(other) == mem::discriminant(&resolved_auth_class), + MultipleAuthenticationTypesNotSupportedSnafu + ); + } + resolved_auth_classes.push(resolved_auth_class); + } + + _ => { + return Err(Error::AuthenticationProviderNotSupported { + auth_class_name: auth_class_name.to_owned(), + provider: auth_class.spec.provider.to_string(), + }); + } } } - auth_classes + Ok(()) } - /// Retrieve all provided `AuthenticationClass` references. - pub async fn resolve( - &self, - client: &Client, - ) -> Result> { - let mut resolved = vec![]; + fn from_oidc( + auth_class_name: &str, + provider: &oidc::AuthenticationProvider, + auth_details: &AirflowClientAuthenticationDetails, + ) -> Result { + let oidc_provider = match &provider.provider_hint { + None => { + info!("No OIDC provider hint given in AuthClass {auth_class_name}, assuming {default_oidc_provider_name}", + default_oidc_provider_name = serde_json::to_string(&DEFAULT_OIDC_PROVIDER).unwrap()); + DEFAULT_OIDC_PROVIDER + } + Some(oidc_provider) => oidc_provider.to_owned(), + }; - // TODO: adapt if multiple authentication classes are supported by airflow. - // This is currently not possible due to the Flask App Builder not supporting it. - if self.authentication.len() > 1 { - return Err(Error::MultipleAuthenticationClassesProvided); - } + ensure!( + SUPPORTED_OIDC_PROVIDERS.contains(&oidc_provider), + OidcProviderNotSupportedSnafu { + auth_class_name, + oidc_provider: serde_json::to_string(&oidc_provider).unwrap(), + } + ); - for config in &self.authentication { - let auth_class = if let Some(auth_class) = &config.authentication_class { - let resolved = AuthenticationClass::resolve(client, auth_class) - .await - .context(AuthenticationClassRetrievalSnafu { - authentication_class: ObjectRef::::new(auth_class), - })?; - - // Checking for supported AuthenticationClass here is a little out of place, but is does not - // make sense to iterate further after finding an unsupported AuthenticationClass. - Some(match resolved.spec.provider { - AuthenticationClassProvider::Ldap(_) => resolved, - AuthenticationClassProvider::Tls(_) - | AuthenticationClassProvider::Oidc(_) - | AuthenticationClassProvider::Static(_) => { - return Err(Error::AuthenticationProviderNotSupported { - authentication_class: ObjectRef::from_obj(&resolved), - provider: resolved.spec.provider.to_string(), - }) + match oidc_provider { + IdentityProviderHint::Keycloak => { + ensure!( + &provider.principal_claim == "preferred_username", + OidcPrincipalClaimNotSupportedSnafu { + configured: provider.principal_claim.clone(), + supported: "preferred_username".to_owned(), + auth_class_name, } - }) - } else { - None - }; - - resolved.push(AirflowAuthenticationConfigResolved { - authentication_class: auth_class, - user_registration: config.user_registration, - user_registration_role: config.user_registration_role.clone(), - sync_roles_at: config.sync_roles_at.clone(), - }) + ); + } } - Ok(resolved) + ensure!( + !provider.tls.uses_tls() || provider.tls.uses_tls_verification(), + TlsVerificationCannotBeDisabledSnafu { auth_class_name } + ); + + Ok(AirflowAuthenticationClassResolved::Oidc { + provider: provider.to_owned(), + oidc: auth_details + .common + .oidc_or_error(auth_class_name) + .context(OidcConfigurationInvalidSnafu)? + .clone(), + }) } } diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index ada34642..3e9dd6b0 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -38,7 +38,7 @@ use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; use crate::{ affinity::{get_affinity, get_executor_affinity}, - authentication::AirflowAuthentication, + authentication::AirflowClientAuthenticationDetails, }; pub mod affinity; @@ -86,10 +86,12 @@ pub enum Error { NoRoleForExecutorFailure, } +// TODO: Doku! airflow #[derive(Display, EnumIter, EnumString)] #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] pub enum AirflowConfigOptions { AuthType, + OauthProviders, AuthLdapSearch, AuthLdapSearchFilter, AuthLdapServer, @@ -109,11 +111,12 @@ pub enum AirflowConfigOptions { AuthLdapTlsCacertfile, AuthLdapAllowSelfSigned, } - +// TODO: Doku! airflow impl FlaskAppConfigOptions for AirflowConfigOptions { fn python_type(&self) -> PythonType { match self { AirflowConfigOptions::AuthType => PythonType::Expression, + AirflowConfigOptions::OauthProviders => PythonType::Expression, AirflowConfigOptions::AuthUserRegistration => PythonType::BoolLiteral, AirflowConfigOptions::AuthUserRegistrationRole => PythonType::StringLiteral, AirflowConfigOptions::AuthRolesSyncAtLogin => PythonType::BoolLiteral, @@ -187,7 +190,7 @@ pub struct AirflowClusterSpec { #[serde(rename_all = "camelCase")] pub struct AirflowClusterConfig { #[serde(flatten)] - pub authentication: AirflowAuthentication, + pub authentication: Vec, /// The name of the Secret object containing the admin user credentials and database connection details. /// Read the diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index e48a4349..e61289a6 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -1,8 +1,10 @@ +use indoc::formatdoc; use snafu::{ResultExt, Snafu}; use stackable_airflow_crd::{ authentication::AirflowAuthenticationConfigResolved, authentication::FlaskRolesSyncMoment, AirflowConfigOptions, }; +use stackable_operator::commons::authentication::{ldap, oidc}; use stackable_operator::commons::authentication::{ ldap::AuthenticationProvider, AuthenticationClassProvider, }; @@ -162,6 +164,73 @@ fn append_ldap_config( Ok(()) } +fn append_oidc_config( + config: &mut BTreeMap, + providers: &[( + &oidc::AuthenticationProvider, + &oidc::ClientAuthenticationOptions<()>, + )], +) { + // Debatable: AUTH_OAUTH or AUTH_OID + // Additionally can be set via config... dunno + config.insert( + AirflowConfigOptions::AuthType.to_string(), + "AUTH_OID".into(), + ); + + let mut oauth_providers_config = Vec::new(); + + for (oidc, client_options) in providers { + let (env_client_id, env_client_secret) = + oidc::AuthenticationProvider::client_credentials_env_names( + &client_options.client_credentials_secret_ref, + ); + let mut scopes = oidc.scopes.clone(); + scopes.extend_from_slice(&client_options.extra_scopes); + + let oidc_provider = oidc + .provider_hint + .as_ref() + .unwrap_or(&DEFAULT_OIDC_PROVIDER); + + let oauth_providers_config_entry = match oidc_provider { + oidc::IdentityProviderHint::Keycloak => { + formatdoc!( + " + {{ 'name': 'keycloak', + 'icon': 'fa-key', + 'token_key': 'access_token', + 'remote_app': {{ + 'client_id': os.environ.get('{env_client_id}'), + 'client_secret': os.environ.get('{env_client_secret}'), + 'client_kwargs': {{ + 'scope': '{scopes}' + }}, + 'api_base_url': '{url}/protocol/', + 'server_metadata_url': '{url}/.well-known/openid-configuration', + }}, + }}", + url = oidc.endpoint_url().unwrap(), + scopes = scopes.join(" "), + ) + } + }; + + oauth_providers_config.push(oauth_providers_config_entry); + } + + config.insert( + AirflowConfigOptions::OauthProviders.to_string(), + formatdoc!( + "[ + {joined_oauth_providers_config} + ] + ", + joined_oauth_providers_config = oauth_providers_config.join(",\n") + ), + ); +} + #[cfg(test)] mod tests { use crate::config::add_airflow_config; From 82e514d3ba9e51ac263651a14765e37a9c4af85d Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Tue, 22 Oct 2024 12:45:57 +0200 Subject: [PATCH 02/25] Add integration test for OIDC --- .../templates/kuttl/oidc/00-patch-ns.yaml.j2 | 9 + tests/templates/kuttl/oidc/10-assert.yaml | 14 ++ .../kuttl/oidc/10-install-postgresql.yaml | 12 ++ tests/templates/kuttl/oidc/20-assert.yaml.j2 | 10 ++ ...tor-aggregator-discovery-configmap.yaml.j2 | 9 + tests/templates/kuttl/oidc/30-assert.yaml | 22 +++ .../kuttl/oidc/30-install-keycloak.yaml | 26 +++ tests/templates/kuttl/oidc/40-assert.yaml | 28 +++ .../kuttl/oidc/40-install-airflow.yaml | 8 + tests/templates/kuttl/oidc/50-assert.yaml | 14 ++ .../oidc/50-install-test-container.yaml.j2 | 80 +++++++++ tests/templates/kuttl/oidc/60-assert.yaml | 8 + tests/templates/kuttl/oidc/60-login.yaml | 9 + .../helm-bitnami-postgresql-values.yaml.j2 | 31 ++++ .../kuttl/oidc/install-airflow.yaml.j2 | 75 ++++++++ .../kuttl/oidc/install-keycloak.yaml.j2 | 170 ++++++++++++++++++ tests/templates/kuttl/oidc/login.py | 104 +++++++++++ tests/test-definition.yaml | 4 + 18 files changed, 633 insertions(+) create mode 100644 tests/templates/kuttl/oidc/00-patch-ns.yaml.j2 create mode 100644 tests/templates/kuttl/oidc/10-assert.yaml create mode 100644 tests/templates/kuttl/oidc/10-install-postgresql.yaml create mode 100644 tests/templates/kuttl/oidc/20-assert.yaml.j2 create mode 100644 tests/templates/kuttl/oidc/20-install-vector-aggregator-discovery-configmap.yaml.j2 create mode 100644 tests/templates/kuttl/oidc/30-assert.yaml create mode 100644 tests/templates/kuttl/oidc/30-install-keycloak.yaml create mode 100644 tests/templates/kuttl/oidc/40-assert.yaml create mode 100644 tests/templates/kuttl/oidc/40-install-airflow.yaml create mode 100644 tests/templates/kuttl/oidc/50-assert.yaml create mode 100644 tests/templates/kuttl/oidc/50-install-test-container.yaml.j2 create mode 100644 tests/templates/kuttl/oidc/60-assert.yaml create mode 100644 tests/templates/kuttl/oidc/60-login.yaml create mode 100644 tests/templates/kuttl/oidc/helm-bitnami-postgresql-values.yaml.j2 create mode 100644 tests/templates/kuttl/oidc/install-airflow.yaml.j2 create mode 100644 tests/templates/kuttl/oidc/install-keycloak.yaml.j2 create mode 100644 tests/templates/kuttl/oidc/login.py diff --git a/tests/templates/kuttl/oidc/00-patch-ns.yaml.j2 b/tests/templates/kuttl/oidc/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/oidc/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/oidc/10-assert.yaml b/tests/templates/kuttl/oidc/10-assert.yaml new file mode 100644 index 00000000..319e927a --- /dev/null +++ b/tests/templates/kuttl/oidc/10-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-airflow-postgresql +timeout: 480 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-postgresql +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/oidc/10-install-postgresql.yaml b/tests/templates/kuttl/oidc/10-install-postgresql.yaml new file mode 100644 index 00000000..ab7b4004 --- /dev/null +++ b/tests/templates/kuttl/oidc/10-install-postgresql.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install airflow-postgresql + --namespace $NAMESPACE + --version 12.5.6 + --values helm-bitnami-postgresql-values.yaml + --repo https://charts.bitnami.com/bitnami postgresql + --wait + timeout: 600 diff --git a/tests/templates/kuttl/oidc/20-assert.yaml.j2 b/tests/templates/kuttl/oidc/20-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/oidc/20-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/oidc/20-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/oidc/20-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/oidc/20-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/oidc/30-assert.yaml b/tests/templates/kuttl/oidc/30-assert.yaml new file mode 100644 index 00000000..93965698 --- /dev/null +++ b/tests/templates/kuttl/oidc/30-assert.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-keycloak +timeout: 480 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: keycloak1 +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: keycloak2 +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/oidc/30-install-keycloak.yaml b/tests/templates/kuttl/oidc/30-install-keycloak.yaml new file mode 100644 index 00000000..f1af62f4 --- /dev/null +++ b/tests/templates/kuttl/oidc/30-install-keycloak.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + INSTANCE_NAME=keycloak1 \ + REALM=test1 \ + USERNAME=jane.doe \ + FIRST_NAME=Jane \ + LAST_NAME=Doe \ + EMAIL=jane.doe@stackable.tech \ + PASSWORD=T8mn72D9 \ + CLIENT_ID=airflow1 \ + CLIENT_SECRET=R1bxHUD569vHeQdw \ + envsubst < install-keycloak.yaml | kubectl apply -n $NAMESPACE -f - + + INSTANCE_NAME=keycloak2 \ + REALM=test2 \ + USERNAME=richard.roe \ + FIRST_NAME=Richard \ + LAST_NAME=Roe \ + EMAIL=richard.roe@stackable.tech \ + PASSWORD=NvfpU518 \ + CLIENT_ID=airflow2 \ + CLIENT_SECRET=scWzh0D4v0GN8NrN \ + envsubst < install-keycloak.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/oidc/40-assert.yaml b/tests/templates/kuttl/oidc/40-assert.yaml new file mode 100644 index 00000000..ad3c8974 --- /dev/null +++ b/tests/templates/kuttl/oidc/40-assert.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: install-airflow +timeout: 1200 +commands: + - script: > + kubectl --namespace $NAMESPACE + wait --for=condition=available=true + airflowclusters.airflow.stackable.tech/airflow + --timeout 301s +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-webserver-default +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-scheduler-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/oidc/40-install-airflow.yaml b/tests/templates/kuttl/oidc/40-install-airflow.yaml new file mode 100644 index 00000000..9ef8e0ab --- /dev/null +++ b/tests/templates/kuttl/oidc/40-install-airflow.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 300 +commands: + - script: > + envsubst '$NAMESPACE' < install-airflow.yaml | + kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/oidc/50-assert.yaml b/tests/templates/kuttl/oidc/50-assert.yaml new file mode 100644 index 00000000..58987778 --- /dev/null +++ b/tests/templates/kuttl/oidc/50-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: install-test-container +timeout: 300 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: python +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/oidc/50-install-test-container.yaml.j2 b/tests/templates/kuttl/oidc/50-install-test-container.yaml.j2 new file mode 100644 index 00000000..d1199711 --- /dev/null +++ b/tests/templates/kuttl/oidc/50-install-test-container.yaml.j2 @@ -0,0 +1,80 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: python +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: python +{% if test_scenario['values']['openshift'] == 'true' %} +rules: +- apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] +{% endif %} +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: python +subjects: + - kind: ServiceAccount + name: python +roleRef: + kind: Role + name: python + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +metadata: + name: install-test-container +timeout: 300 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: python + labels: + app: python +spec: + replicas: 1 + selector: + matchLabels: + app: python + template: + metadata: + labels: + app: python + spec: + serviceAccountName: python + securityContext: + fsGroup: 1000 + containers: + - name: python + image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev + stdin: true + tty: true + resources: + requests: + memory: "128Mi" + cpu: "512m" + limits: + memory: "128Mi" + cpu: "1" + volumeMounts: + - name: tls + mountPath: /stackable/tls + env: + - name: REQUESTS_CA_BUNDLE + value: /stackable/tls/ca.crt + volumes: + - name: tls + csi: + driver: secrets.stackable.tech + volumeAttributes: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: pod diff --git a/tests/templates/kuttl/oidc/60-assert.yaml b/tests/templates/kuttl/oidc/60-assert.yaml new file mode 100644 index 00000000..fee66ceb --- /dev/null +++ b/tests/templates/kuttl/oidc/60-assert.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: login +timeout: 300 +commands: + - script: kubectl exec -n $NAMESPACE python-0 -- python /stackable/login.py diff --git a/tests/templates/kuttl/oidc/60-login.yaml b/tests/templates/kuttl/oidc/60-login.yaml new file mode 100644 index 00000000..0745bc4b --- /dev/null +++ b/tests/templates/kuttl/oidc/60-login.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +metadata: + name: login +commands: + - script: > + envsubst '$NAMESPACE' < login.py | + kubectl exec -n $NAMESPACE -i python-0 -- tee /stackable/login.py > /dev/null diff --git a/tests/templates/kuttl/oidc/helm-bitnami-postgresql-values.yaml.j2 b/tests/templates/kuttl/oidc/helm-bitnami-postgresql-values.yaml.j2 new file mode 100644 index 00000000..951804d6 --- /dev/null +++ b/tests/templates/kuttl/oidc/helm-bitnami-postgresql-values.yaml.j2 @@ -0,0 +1,31 @@ +--- +volumePermissions: + enabled: false + securityContext: + runAsUser: auto + +primary: + podSecurityContext: +{% if test_scenario['values']['openshift'] == 'true' %} + enabled: false +{% else %} + enabled: true +{% endif %} + containerSecurityContext: + enabled: false + resources: + requests: + memory: "128Mi" + cpu: "512m" + limits: + memory: "128Mi" + cpu: "1" + +shmVolume: + chmod: + enabled: false + +auth: + username: airflow + password: airflow + database: airflow diff --git a/tests/templates/kuttl/oidc/install-airflow.yaml.j2 b/tests/templates/kuttl/oidc/install-airflow.yaml.j2 new file mode 100644 index 00000000..9be91877 --- /dev/null +++ b/tests/templates/kuttl/oidc/install-airflow.yaml.j2 @@ -0,0 +1,75 @@ +# $NAMESPACE will be replaced with the namespace of the test case. +--- +apiVersion: v1 +kind: Secret +metadata: + name: airflow-credentials +type: Opaque +stringData: + adminUser.username: airflow + adminUser.firstname: Airflow + adminUser.lastname: Admin + adminUser.email: airflow@airflow.com + adminUser.password: airflow + connections.secretKey: thisISaSECRET_1234 + connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow +--- +apiVersion: v1 +kind: Secret +metadata: + name: airflow-keycloak1-client +stringData: + clientId: airflow1 + clientSecret: R1bxHUD569vHeQdw +--- +apiVersion: v1 +kind: Secret +metadata: + name: airflow-keycloak2-client +stringData: + clientId: airflow2 + clientSecret: scWzh0D4v0GN8NrN +--- +apiVersion: airflow.stackable.tech/v1alpha1 +kind: AirflowCluster +metadata: + name: airflow +spec: + image: +{% if test_scenario['values']['airflow'].find(",") > 0 %} + custom: "{{ test_scenario['values']['airflow'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['airflow'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['airflow'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + authentication: + - authenticationClass: keycloak1-$NAMESPACE + oidc: + clientCredentialsSecret: airflow-keycloak1-client + - authenticationClass: keycloak2-$NAMESPACE + oidc: + clientCredentialsSecret: airflow-keycloak2-client + credentialsSecret: airflow-credentials +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + webservers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 + kubernetesExecutors: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + schedulers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/oidc/install-keycloak.yaml.j2 b/tests/templates/kuttl/oidc/install-keycloak.yaml.j2 new file mode 100644 index 00000000..7197df2f --- /dev/null +++ b/tests/templates/kuttl/oidc/install-keycloak.yaml.j2 @@ -0,0 +1,170 @@ +# The environment variables must be replaced. +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: $INSTANCE_NAME-realms +data: + test-realm.json: | + { + "realm": "$REALM", + "enabled": true, + "users": [ + { + "enabled": true, + "username": "$USERNAME", + "firstName" : "$FIRST_NAME", + "lastName" : "$LAST_NAME", + "email" : "$EMAIL", + "credentials": [ + { + "type": "password", + "value": "$PASSWORD" + } + ], + "realmRoles": [ + "user" + ] + } + ], + "roles": { + "realm": [ + { + "name": "user", + "description": "User privileges" + } + ] + }, + "clients": [ + { + "clientId": "$CLIENT_ID", + "enabled": true, + "clientAuthenticatorType": "client-secret", + "secret": "$CLIENT_SECRET", + "redirectUris": [ + "*" + ], + "webOrigins": [ + "*" + ], + "standardFlowEnabled": true, + "protocol": "openid-connect" + } + ] + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: $INSTANCE_NAME + labels: + app: $INSTANCE_NAME +spec: + replicas: 1 + selector: + matchLabels: + app: $INSTANCE_NAME + template: + metadata: + labels: + app: $INSTANCE_NAME + spec: + serviceAccountName: keycloak + containers: + - name: keycloak + image: quay.io/keycloak/keycloak:23.0.4 + args: + - start-dev + - --import-realm + - --https-certificate-file=/tls/tls.crt + - --https-certificate-key-file=/tls/tls.key + env: + - name: KEYCLOAK_ADMIN + value: admin + - name: KEYCLOAK_ADMIN_PASSWORD + value: admin + ports: + - name: https + containerPort: 8443 + volumeMounts: + - name: realms + mountPath: /opt/keycloak/data/import + - name: tls + mountPath: /tls + readinessProbe: + httpGet: + scheme: HTTPS + path: /realms/$REALM + port: 8443 + volumes: + - name: realms + configMap: + name: $INSTANCE_NAME-realms + - name: tls + csi: + driver: secrets.stackable.tech + volumeAttributes: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: service=$INSTANCE_NAME +--- +apiVersion: v1 +kind: Service +metadata: + name: $INSTANCE_NAME +spec: + selector: + app: $INSTANCE_NAME + ports: + - protocol: TCP + port: 8443 +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: $INSTANCE_NAME-$NAMESPACE +spec: + provider: + oidc: + hostname: $INSTANCE_NAME.$NAMESPACE.svc.cluster.local + port: 8443 + rootPath: /realms/$REALM + scopes: + - email + - openid + - profile + principalClaim: preferred_username + providerHint: Keycloak + tls: + verification: + server: + caCert: + secretClass: tls +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: keycloak +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: keycloak +{% if test_scenario['values']['openshift'] == 'true' %} +rules: +- apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] +{% endif %} +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: keycloak +subjects: + - kind: ServiceAccount + name: keycloak +roleRef: + kind: Role + name: keycloak + apiGroup: rbac.authorization.k8s.io diff --git a/tests/templates/kuttl/oidc/login.py b/tests/templates/kuttl/oidc/login.py new file mode 100644 index 00000000..f52a2d56 --- /dev/null +++ b/tests/templates/kuttl/oidc/login.py @@ -0,0 +1,104 @@ +# $NAMESPACE will be replaced with the namespace of the test case. + +import json +import logging +import requests +import sys +from bs4 import BeautifulSoup + +logging.basicConfig( + level='DEBUG', + format="%(asctime)s %(levelname)s: %(message)s", + stream=sys.stdout) + +session = requests.Session() + +# Click on "Sign In with keycloak" in Airflow +login_page = session.get("http://airflow-webserver:8080/login/keycloak?next=") + +assert login_page.ok, "Redirection from Airflow to Keycloak failed" +assert login_page.url.startswith("https://keycloak1.$NAMESPACE.svc.cluster.local:8443/realms/test1/protocol/openid-connect/auth?response_type=code&client_id=airflow1"), \ + "Redirection to the Keycloak login page expected" + +# Enter username and password into the Keycloak login page and click on "Sign In" +login_page_html = BeautifulSoup(login_page.text, 'html.parser') +authenticate_url = login_page_html.form['action'] +welcome_page = session.post(authenticate_url, data={ + 'username': "jane.doe", + 'password': "T8mn72D9" +}) + +assert welcome_page.ok, "Login failed" +assert welcome_page.url == "http://airflow-webserver:8080/airflow/welcome/", \ + "Redirection to the Airflow welcome page expected" + +# Open the user information page in Airflow +userinfo_page = session.get("http://airflow-webserver:8080/users/userinfo/") + +assert userinfo_page.ok, "Retrieving user information failed" +assert userinfo_page.url == "http://airflow-webserver:8080/airflow/welcome/", \ + "Redirection to the Airflow welcome page expected" + +# Expect the user data provided by Keycloak in Airflow +userinfo_page_html = BeautifulSoup(userinfo_page.text, 'html.parser') +raw_data = userinfo_page_html.find(id='app')['data-bootstrap'] +data = json.loads(raw_data) +user_data = data['user'] + +assert user_data['firstName'] == "Jane", \ + "The first name of the user in Airflow should match the one provided by Keycloak" +assert user_data['lastName'] == "Doe", \ + "The last name of the user in Airflow should match the one provided by Keycloak" +assert user_data['email'] == "jane.doe@stackable.tech", \ + "The email of the user in Airflow should match the one provided by Keycloak" + +# TODO Use different OIDC providers (currently only Keycloak is +# supported) +# +# It would be beneficial if the second OAuth provider keycloak2 could +# also be tested. This would ensure that the Airflow configuration is +# correct. The problem is that the Flask-AppBuilder (and hence Airflow) +# do not support multiple OAuth providers with the same name. But +# keycloak1 and keycloak2 use the same name, namely "keycloak": +# +# OAUTH_PROVIDERS = [ +# { 'name': 'keycloak', +# 'icon': 'fa-key', +# 'token_key': 'access_token', +# 'remote_app': { +# 'client_id': os.environ.get('OIDC_728D9B504A6E9A10_CLIENT_ID'), +# 'client_secret': os.environ.get('OIDC_728D9B504A6E9A10_CLIENT_SECRET'), +# 'client_kwargs': { +# 'scope': 'email openid profile' +# }, +# 'api_base_url': 'https://keycloak1.kuttl.svc.cluster.local:8443/realms/test1/protocol/', +# 'server_metadata_url': 'https://keycloak1.kuttl.svc.cluster.local:8443/realms/test1/.well-known/openid-configuration', +# }, +# }, +# { 'name': 'keycloak', +# 'icon': 'fa-key', +# 'token_key': 'access_token', +# 'remote_app': { +# 'client_id': os.environ.get('OIDC_607BA683B09BC0B8_CLIENT_ID'), +# 'client_secret': os.environ.get('OIDC_607BA683B09BC0B8_CLIENT_SECRET'), +# 'client_kwargs': { +# 'scope': 'email openid profile' +# }, +# 'api_base_url': 'https://keycloak2.kuttl.svc.cluster.local:8443/realms/test2/protocol/', +# 'server_metadata_url': 'https://keycloak2.kuttl.svc.cluster.local:8443/realms/test2/.well-known/openid-configuration', +# }, +# } +# ] +# +# This name is set in the operator and cannot be changed. The reason is +# that the name is also used in Flask-AppBuilder to determine how the +# user information must be interpreted. +# +# Airflow actually shows two "Sign In with keycloak" buttons in this +# test but the second one cannot be clicked. +# +# It is nevertheless useful to have two Keycloak instances in this test +# because it ensures that several authentication entries can be +# specified, no volumes or volume mounts are added twice, and that the +# configuration is correct to the extent that Airflow does not complain +# about it. diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 769128bb..7be72360 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -51,6 +51,10 @@ tests: - openshift - ldap-authentication - executor + - name: oidc + dimensions: + - airflow + - openshift - name: resources dimensions: - airflow-latest From d2c07fa432d27a57a7d45e8c589bfd101e3780d4 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Tue, 22 Oct 2024 15:18:37 +0200 Subject: [PATCH 03/25] Document OIDC support --- .../airflow/pages/usage-guide/security.adoc | 115 +++++++++++++++++- 1 file changed, 109 insertions(+), 6 deletions(-) diff --git a/docs/modules/airflow/pages/usage-guide/security.adoc b/docs/modules/airflow/pages/usage-guide/security.adoc index 3c8fea46..13b9afd2 100644 --- a/docs/modules/airflow/pages/usage-guide/security.adoc +++ b/docs/modules/airflow/pages/usage-guide/security.adoc @@ -1,14 +1,21 @@ = Security -:description: Secure Apache Airflow by configuring user authentication and authorization, either with built-in methods or LDAP. +:description: Secure Apache Airflow by configuring user authentication and authorization. :airflow-access-control-docs: https://airflow.apache.org/docs/apache-airflow/stable/security/access-control.html +:keycloak: https://www.keycloak.org/ Secure Apache Airflow by configuring user authentication and authorization. -Airflow provides built-in user and role management, but can also connect to a LDAP server to manage users centrally instead. +Airflow provides built-in user and role management, but can also connect to an LDAP server or an OIDC provider to manage users centrally instead. == Authentication -Users need to authenticate themselves before using Airflow, and there are two ways to configure users: -The built-in user management or LDAP. +Users need to authenticate themselves before using Airflow, and there are several ways in which this can be set up. + +[IMPORTANT] +.Multiple authentication methods +==== +Only one authentication method is supported at a time, and in case of LDAP, only one authentication class is allowed. +This means, it is not possible to configure both LDAP and OIDC authentication methods at the same time, but *it is* possible to configure multiple OIDC classes *or* one LDAP authentication class. +==== === Built-in user management @@ -19,7 +26,7 @@ image::airflow_security.png[Airflow Security menu] === LDAP -Airflow supports xref:concepts:authentication.adoc[user authentication] via LDAP. +Airflow supports xref:concepts:authentication.adoc[user authentication] againts a single LDAP server. Set up an AuthenticationClass for the LDAP server and reference it in the Airflow Stacklet resource as shown: [source,yaml] @@ -30,7 +37,7 @@ metadata: name: airflow-with-ldap spec: image: - productVersion: 2.9.3 + productVersion: 2.10.2 clusterConfig: authentication: - authenticationClass: ldap # <1> @@ -48,6 +55,79 @@ The users and roles can be viewed as before in the Webserver UI, but the blue "+ image::airflow_security_ldap.png[Airflow Security menu] +=== [[oidc]]OpenID Connect + +An OpenID Connect provider can be used for authentication. +Unfortunately, there is no generic support for OpenID Connect built into Airflow. +This means that only specific OpenID Connect providers can be configured. + +IMPORTANT: Airflow deployments on the Stackable Data Platform only support {keycloak}[Keycloak]. + +[source,yaml] +---- +apiVersion: airflow.stackable.tech/v1alpha1 +kind: AirflowCluster +metadata: + name: airflow-with-oidc +spec: + image: + productVersion: 2.10.2 + clusterConfig: + authentication: + - authenticationClass: keycloak # <1> + oidc: + clientCredentialsSecret: airflow-keycloak-client # <2> + userRegistrationRole: User # <3> +---- + +<1> The reference to an AuthenticationClass called `keycloak` +<2> The reference to the Secret containing the Airflow client credentials +<3> The default role to which all users are assigned + +Users that log in with OpenID Connect are assigned to a default {airflow-access-control-docs}[role] which is specified with the `userRegistrationRole` property. + +The Secret containing the Airflow client credentials: + +[source,yaml] +---- +apiVersion: v1 +kind: Secret +metadata: + name: airflow-keycloak-client +stringData: + clientId: airflow # <1> + clientSecret: airflow_client_secret # <2> +---- + +<1> The client ID of Airflow as defined in Keycloak +<2> The client secret as defined in Keycloak + +A minimum client configuration in Keycloak for this example looks like this: + +[source,json] +---- +{ + "clientId": "airflow", + "enabled": true, + "clientAuthenticatorType": "client-secret", # <1> + "secret": "airflow_client_secret", + "redirectUris": [ + "*" + ], + "webOrigins": [ + "*" + ], + "standardFlowEnabled": true, # <2> + "protocol": "openid-connect" # <3> +} +---- + +<1> Sets the OIDC type to confidential access type. +<2> Enables the OAuth2 "Authorization Code Flow". +<3> Enables OpenID Connect and OAuth2 support. + +Further information for specifying an AuthenticationClass for an OIDC provider can be found at the xref:concepts:authentication.adoc#_oidc[concepts page]. + == Authorization The Airflow Webserver delegates the {airflow-access-control-docs}[handling of user access control] to https://flask-appbuilder.readthedocs.io/en/latest/security.html[Flask AppBuilder]. @@ -74,3 +154,26 @@ spec: <1> The reference to an AuthenticationClass called `ldap` <2> All users are assigned to the `Admin` role + +=== OpenID Connect + +The mechanism for assigning roles to users described in the LDAP section also applies to OpenID Connect. +Airflow supports assigning {airflow-access-control-docs}[Roles] to users based on their OpenID Connect scopes, though this is not yet supported by the Stackable operator. +All the users logging in via OpenID Connect get assigned to the same role which you can configure via the attribute `authentication[*].userRegistrationRole` on the `AirflowCluster` object: + +[source,yaml] +---- +apiVersion: airflow.stackable.tech/v1alpha1 +kind: AirflowCluster +metadata: + name: airflow-with-oidc +spec: + clusterConfig: + authentication: + - authenticationClass: keycloak + oidc: + clientCredentialsSecret: airflow-keycloak-client + userRegistrationRole: Admin # <1> +---- + +<1> All users are assigned to the `Admin` role From 6bf5d045525fdaea2d9712de3639d789794afcb1 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 23 Oct 2024 12:50:41 +0200 Subject: [PATCH 04/25] Update wip --- Cargo.lock | 23 +++++--- Cargo.toml | 1 + rust/crd/src/authentication.rs | 51 +++++++++++++++- rust/operator-binary/Cargo.toml | 1 + rust/operator-binary/src/config.rs | 93 ++++++++++++++++++++---------- 5 files changed, 126 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2e216d29..eccb3da7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -264,9 +264,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" [[package]] name = "cc" @@ -1005,6 +1005,12 @@ dependencies = [ "hashbrown 0.15.0", ] +[[package]] +name = "indoc" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" + [[package]] name = "instant" version = "0.1.13" @@ -1950,9 +1956,9 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.211" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "1ac55e59090389fb9f0dd9e0f3c09615afed1d19094284d0b200441f13550793" dependencies = [ "serde_derive", ] @@ -1969,9 +1975,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.211" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "54be4f245ce16bc58d57ef2716271d0d4519e0f6defa147f6e081005bcb278ff" dependencies = [ "proc-macro2", "quote", @@ -2157,6 +2163,7 @@ dependencies = [ "clap", "fnv", "futures 0.3.31", + "indoc", "product-config", "serde", "serde_yaml", @@ -2385,9 +2392,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.40.0" +version = "1.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" dependencies = [ "backtrace", "bytes", diff --git a/Cargo.toml b/Cargo.toml index e5f902db..b87adb3f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ built = { version = "0.7", features = ["chrono", "git2"] } clap = "4.5" fnv = "1.0" futures = { version = "0.3", features = ["compat"] } +indoc = "2.0" product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" } rstest = "0.22" semver = "1.0" diff --git a/rust/crd/src/authentication.rs b/rust/crd/src/authentication.rs index 2f7d3d7f..ff98dfb1 100644 --- a/rust/crd/src/authentication.rs +++ b/rust/crd/src/authentication.rs @@ -9,7 +9,6 @@ use stackable_operator::{ oidc::{self, IdentityProviderHint}, AuthenticationClass, AuthenticationClassProvider, ClientAuthenticationDetails, }, - kube::runtime::reflector::ObjectRef, schemars::{self, JsonSchema}, }; use std::collections::BTreeSet; @@ -36,7 +35,7 @@ pub enum Error { #[snafu(display("Only one authentication class is currently supported at a time"))] MultipleAuthenticationClassesProvided, #[snafu(display( - "Failed to use authentication provider [{provider}] for authentication class [{authentication_class}] - supported providers: {SUPPORTED_AUTHENTICATION_CLASS_PROVIDERS:?}", + "Failed to use authentication provider [{provider}] for authentication class [{auth_class_name}] - supported providers: {SUPPORTED_AUTHENTICATION_CLASS_PROVIDERS:?}", ))] AuthenticationProviderNotSupported { auth_class_name: String, @@ -55,6 +54,18 @@ pub enum Error { "TLS verification cannot be disabled in Superset (AuthenticationClass {auth_class_name:?})." ))] TlsVerificationCannotBeDisabled { auth_class_name: String }, + #[snafu(display( + "The userRegistrationRole settings must not differ between the authentication entries.", + ))] + DifferentUserRegistrationRoleSettingsNotAllowed, + #[snafu(display( + "The userRegistration settings must not differ between the authentication entries.", + ))] + DifferentUserRegistrationSettingsNotAllowed, + #[snafu(display( + "The syncRolesAt settings must not differ between the authentication entries.", + ))] + DifferentSyncRolesAtSettingsNotAllowed, #[snafu(display("Invalid OIDC configuration"))] OidcConfigurationInvalid { source: stackable_operator::commons::authentication::Error, @@ -212,8 +223,42 @@ impl AirflowClientAuthenticationDetailsResolved { }); } } + + match user_registration { + Some(user_registration) => { + ensure!( + user_registration == entry.user_registration, + DifferentUserRegistrationSettingsNotAllowedSnafu + ); + } + None => user_registration = Some(entry.user_registration), + } + match &user_registration_role { + Some(user_registration_role) => { + ensure!( + user_registration_role == &entry.user_registration_role, + DifferentUserRegistrationRoleSettingsNotAllowedSnafu + ); + } + None => user_registration_role = Some(entry.user_registration_role.to_owned()), + } + match &sync_roles_at { + Some(sync_roles_at) => { + ensure!( + sync_roles_at == &entry.sync_roles_at, + DifferentSyncRolesAtSettingsNotAllowedSnafu + ); + } + None => sync_roles_at = Some(entry.sync_roles_at.to_owned()), + } } - Ok(()) + Ok(AirflowClientAuthenticationDetailsResolved { + authentication_classes_resolved: resolved_auth_classes, + user_registration: user_registration.unwrap_or_else(default_user_registration), + user_registration_role: user_registration_role + .unwrap_or_else(default_user_registration_role), + sync_roles_at: sync_roles_at.unwrap_or_else(FlaskRolesSyncMoment::default), + }) } fn from_oidc( diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index fdd08d96..9ffd50f7 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -23,6 +23,7 @@ product-config.workspace = true strum.workspace = true tokio.workspace = true tracing.workspace = true +indoc.workspace = true [build-dependencies] diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index e61289a6..86da040f 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -1,13 +1,13 @@ use indoc::formatdoc; use snafu::{ResultExt, Snafu}; use stackable_airflow_crd::{ - authentication::AirflowAuthenticationConfigResolved, authentication::FlaskRolesSyncMoment, + authentication::{ + AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, + FlaskRolesSyncMoment, DEFAULT_OIDC_PROVIDER, + }, AirflowConfigOptions, }; -use stackable_operator::commons::authentication::{ldap, oidc}; -use stackable_operator::commons::authentication::{ - ldap::AuthenticationProvider, AuthenticationClassProvider, -}; +use stackable_operator::commons::authentication::{ldap::AuthenticationProvider, oidc}; use stackable_operator::commons::tls_verification::TlsVerification; use std::collections::BTreeMap; @@ -30,7 +30,7 @@ pub enum Error { pub fn add_airflow_config( config: &mut BTreeMap, - authentication_config: &Vec, + authentication_config: &AirflowClientAuthenticationDetailsResolved, ) -> Result<()> { if !config.contains_key(&*AirflowConfigOptions::AuthType.to_string()) { config.insert( @@ -47,31 +47,53 @@ pub fn add_airflow_config( fn append_authentication_config( config: &mut BTreeMap, - authentication_config: &Vec, -) -> Result<()> { - // TODO: we make sure in crd/src/authentication.rs that currently there is only one - // AuthenticationClass provided. If the FlaskAppBuilder ever supports this we have - // to adapt the config here accordingly - for auth_config in authentication_config { - if let Some(auth_class) = &auth_config.authentication_class { - if let AuthenticationClassProvider::Ldap(ldap) = &auth_class.spec.provider { - append_ldap_config(config, ldap)?; + auth_config: &AirflowClientAuthenticationDetailsResolved, +) -> Result<(), Error> { + let ldap_providers = auth_config + .authentication_classes_resolved + .iter() + .filter_map(|auth_class| { + if let AirflowAuthenticationClassResolved::Ldap { provider } = auth_class { + Some(provider) + } else { + None } - } + }) + .collect::>(); - config.insert( - AirflowConfigOptions::AuthUserRegistration.to_string(), - auth_config.user_registration.to_string(), - ); - config.insert( - AirflowConfigOptions::AuthUserRegistrationRole.to_string(), - auth_config.user_registration_role.to_string(), - ); - config.insert( - AirflowConfigOptions::AuthRolesSyncAtLogin.to_string(), - (auth_config.sync_roles_at == FlaskRolesSyncMoment::Login).to_string(), - ); + let oidc_providers = auth_config + .authentication_classes_resolved + .iter() + .filter_map(|auth_class| { + if let AirflowAuthenticationClassResolved::Oidc { provider, oidc } = auth_class { + Some((provider, oidc)) + } else { + None + } + }) + .collect::>(); + + if let Some(ldap_provider) = ldap_providers.first() { + append_ldap_config(config, ldap_provider)?; + } + + if !oidc_providers.is_empty() { + append_oidc_config(config, &oidc_providers); } + + config.insert( + AirflowConfigOptions::AuthUserRegistration.to_string(), + auth_config.user_registration.to_string(), + ); + config.insert( + AirflowConfigOptions::AuthUserRegistrationRole.to_string(), + auth_config.user_registration_role.to_string(), + ); + config.insert( + AirflowConfigOptions::AuthRolesSyncAtLogin.to_string(), + (auth_config.sync_roles_at == FlaskRolesSyncMoment::Login).to_string(), + ); + Ok(()) } @@ -235,14 +257,21 @@ fn append_oidc_config( mod tests { use crate::config::add_airflow_config; use stackable_airflow_crd::authentication::{ - default_sync_roles_at, default_user_registration, AirflowAuthenticationConfigResolved, + default_sync_roles_at, default_user_registration, + AirflowClientAuthenticationDetailsResolved, }; use stackable_airflow_crd::AirflowConfigOptions; use stackable_operator::commons::authentication::AuthenticationClass; use std::collections::BTreeMap; #[test] - fn test_no_ldap() { + // pub struct AirflowClientAuthenticationDetailsResolved { + // pub authentication_classes_resolved: Vec, + // pub user_registration: bool, + // pub user_registration_role: String, + // pub sync_roles_at: FlaskRolesSyncMoment, + // } + fn test_no_external_auth() { let mut result = BTreeMap::new(); add_airflow_config(&mut result, &vec![]).expect("Ok"); assert_eq!( @@ -278,8 +307,8 @@ mod tests { let authentication_class: AuthenticationClass = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); - let resolved_config = AirflowAuthenticationConfigResolved { - authentication_class: Some(authentication_class), + let resolved_config = AirflowClientAuthenticationDetailsResolved { + authentication_classes_resolved: authentication_class, user_registration: default_user_registration(), user_registration_role: "Admin".to_string(), sync_roles_at: default_sync_roles_at(), From 816e1cb38e197e63eadb78878d5273ffed40ac87 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 23 Oct 2024 15:36:23 +0200 Subject: [PATCH 05/25] Adapt module tests in operator-binary/src/config.rs --- rust/operator-binary/src/config.rs | 204 ++++++++++++++++++++++------- 1 file changed, 158 insertions(+), 46 deletions(-) diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index 86da040f..6a595cf9 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -256,72 +256,184 @@ fn append_oidc_config( #[cfg(test)] mod tests { use crate::config::add_airflow_config; + use indoc::indoc; use stackable_airflow_crd::authentication::{ - default_sync_roles_at, default_user_registration, - AirflowClientAuthenticationDetailsResolved, + default_sync_roles_at, default_user_registration, AirflowAuthenticationClassResolved, + AirflowClientAuthenticationDetailsResolved, FlaskRolesSyncMoment, }; - use stackable_airflow_crd::AirflowConfigOptions; - use stackable_operator::commons::authentication::AuthenticationClass; + use stackable_operator::commons::authentication::{ldap, oidc}; use std::collections::BTreeMap; #[test] - // pub struct AirflowClientAuthenticationDetailsResolved { - // pub authentication_classes_resolved: Vec, - // pub user_registration: bool, - // pub user_registration_role: String, - // pub sync_roles_at: FlaskRolesSyncMoment, - // } - fn test_no_external_auth() { + fn test_auth_db_config() { + let authentication_config = AirflowClientAuthenticationDetailsResolved { + authentication_classes_resolved: vec![], + user_registration: true, + user_registration_role: "User".to_string(), + sync_roles_at: FlaskRolesSyncMoment::Registration, + }; + let mut result = BTreeMap::new(); - add_airflow_config(&mut result, &vec![]).expect("Ok"); + add_airflow_config(&mut result, &authentication_config).expect("Ok"); + assert_eq!( - BTreeMap::from([("AUTH_TYPE".into(), "AUTH_DB".into())]), + BTreeMap::from([ + ("AUTH_ROLES_SYNC_AT_LOGIN".into(), "false".into()), + ("AUTH_TYPE".into(), "AUTH_DB".into()), + ("AUTH_USER_REGISTRATION".into(), "true".into()), + ("AUTH_USER_REGISTRATION_ROLE".into(), "User".into()) + ]), result ); } #[test] - fn test_ldap() { - let authentication_class = " - apiVersion: authentication.stackable.tech/v1alpha1 - kind: AuthenticationClass - metadata: - name: airflow-with-ldap-server-veri-tls-ldap - spec: - provider: - ldap: - hostname: openldap.default.svc.cluster.local - port: 636 - searchBase: ou=users,dc=example,dc=org - ldapFieldNames: - uid: uid - bindCredentials: - secretClass: airflow-with-ldap-server-veri-tls-ldap-bind - tls: - verification: - server: - caCert: - secretClass: openldap-tls - "; - let deserializer = serde_yaml::Deserializer::from_str(authentication_class); - let authentication_class: AuthenticationClass = + fn test_ldap_config() { + let ldap_provider_yaml = r#" + hostname: openldap.default.svc.cluster.local + port: 636 + searchBase: ou=users,dc=example,dc=org + ldapFieldNames: + uid: uid + bindCredentials: + secretClass: airflow-with-ldap-server-veri-tls-ldap-bind + tls: + verification: + server: + caCert: + secretClass: openldap-tls + "#; + let deserializer = serde_yaml::Deserializer::from_str(ldap_provider_yaml); + let ldap_provider: ldap::AuthenticationProvider = + serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); + + let authentication_config = AirflowClientAuthenticationDetailsResolved { + authentication_classes_resolved: vec![AirflowAuthenticationClassResolved::Ldap { + provider: ldap_provider, + }], + user_registration: true, + user_registration_role: "Admin".to_string(), + sync_roles_at: FlaskRolesSyncMoment::Registration, + }; + + let mut result = BTreeMap::new(); + add_airflow_config(&mut result, &authentication_config).expect("Ok"); + + assert_eq!(BTreeMap::from([ + ("AUTH_LDAP_ALLOW_SELF_SIGNED".into(), "false".into()), + ("AUTH_LDAP_BIND_PASSWORD".into(), "open('/stackable/secrets/airflow-with-ldap-server-veri-tls-ldap-bind/password').read()".into()), + ("AUTH_LDAP_BIND_USER".into(), "open('/stackable/secrets/airflow-with-ldap-server-veri-tls-ldap-bind/user').read()".into()), + ("AUTH_LDAP_FIRSTNAME_FIELD".into(), "givenName".into()), + ("AUTH_LDAP_GROUP_FIELD".into(), "memberof".into()), + ("AUTH_LDAP_LASTNAME_FIELD".into(), "sn".into()), + ("AUTH_LDAP_SEARCH".into(), "ou=users,dc=example,dc=org".into()), + ("AUTH_LDAP_SEARCH_FILTER".into(), "".into()), + ("AUTH_LDAP_SERVER".into(), "ldaps://openldap.default.svc.cluster.local:636".into()), + ("AUTH_LDAP_TLS_CACERTFILE".into(), "/stackable/secrets/openldap-tls/ca.crt".into()), + ("AUTH_LDAP_TLS_DEMAND".into(), "true".into()), + ("AUTH_LDAP_UID_FIELD".into(), "uid".into()), + ("AUTH_ROLES_SYNC_AT_LOGIN".into(), "false".into()), + ("AUTH_TYPE".into(), "AUTH_LDAP".into()), + ("AUTH_USER_REGISTRATION".into(), "true".into()), + ("AUTH_USER_REGISTRATION_ROLE".into(), "Admin".into()) + ]), result); + } + + #[test] + fn test_oidc_config() { + let oidc_provider_yaml1 = r#" + hostname: my.keycloak1.server + port: 12345 + rootPath: my-root-path + tls: + verification: + server: + caCert: + secretClass: keycloak-ca-cert + principalClaim: preferred_username + scopes: + - openid + - email + - profile + provider_hint: Keycloak + "#; + let deserializer = serde_yaml::Deserializer::from_str(oidc_provider_yaml1); + let oidc_provider1: oidc::AuthenticationProvider = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); - let resolved_config = AirflowClientAuthenticationDetailsResolved { - authentication_classes_resolved: authentication_class, + let oidc_provider_yaml2 = r#" + hostname: my.keycloak2.server + principalClaim: preferred_username + scopes: + - openid + provider_hint: Keycloak + "#; + let deserializer = serde_yaml::Deserializer::from_str(oidc_provider_yaml2); + let oidc_provider2: oidc::AuthenticationProvider = + serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); + + let authentication_config = AirflowClientAuthenticationDetailsResolved { + authentication_classes_resolved: vec![ + AirflowAuthenticationClassResolved::Oidc { + provider: oidc_provider1, + oidc: oidc::ClientAuthenticationOptions { + client_credentials_secret_ref: "test-client-secret1".to_string(), + extra_scopes: vec!["roles".to_string()], + product_specific_fields: (), + }, + }, + AirflowAuthenticationClassResolved::Oidc { + provider: oidc_provider2, + oidc: oidc::ClientAuthenticationOptions { + client_credentials_secret_ref: "test-client-secret2".to_string(), + extra_scopes: vec![], + product_specific_fields: (), + }, + }, + ], user_registration: default_user_registration(), user_registration_role: "Admin".to_string(), sync_roles_at: default_sync_roles_at(), }; let mut result = BTreeMap::new(); - add_airflow_config(&mut result, &vec![resolved_config]).expect("Ok"); + add_airflow_config(&mut result, &authentication_config).expect("Ok"); - assert_eq!( - "AUTH_LDAP", - result - .get(&AirflowConfigOptions::AuthType.to_string()) - .unwrap() - ); + assert_eq!(BTreeMap::from([ + ("AUTH_ROLES_SYNC_AT_LOGIN".into(), "false".into()), + ("AUTH_TYPE".into(), "AUTH_OAUTH".into()), + ("AUTH_USER_REGISTRATION".into(), "true".into()), + ("AUTH_USER_REGISTRATION_ROLE".into(), "Admin".into()), + ("OAUTH_PROVIDERS".into(), indoc!(" + [ + { 'name': 'keycloak', + 'icon': 'fa-key', + 'token_key': 'access_token', + 'remote_app': { + 'client_id': os.environ.get('OIDC_A96BCC4FA49835D2_CLIENT_ID'), + 'client_secret': os.environ.get('OIDC_A96BCC4FA49835D2_CLIENT_SECRET'), + 'client_kwargs': { + 'scope': 'openid email profile roles' + }, + 'api_base_url': 'https://my.keycloak1.server:12345/my-root-path/protocol/', + 'server_metadata_url': 'https://my.keycloak1.server:12345/my-root-path/.well-known/openid-configuration', + }, + }, + { 'name': 'keycloak', + 'icon': 'fa-key', + 'token_key': 'access_token', + 'remote_app': { + 'client_id': os.environ.get('OIDC_3A305E38C3B561F3_CLIENT_ID'), + 'client_secret': os.environ.get('OIDC_3A305E38C3B561F3_CLIENT_SECRET'), + 'client_kwargs': { + 'scope': 'openid' + }, + 'api_base_url': 'http://my.keycloak2.server//protocol/', + 'server_metadata_url': 'http://my.keycloak2.server//.well-known/openid-configuration', + }, + } + ] + ").into()) + ]), result); } } From c79ce8a9bfe89a3605e539ded78bc8576a6f4a6f Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 24 Oct 2024 11:33:29 +0200 Subject: [PATCH 06/25] Adding unit tests --- Cargo.lock | 2 + rust/crd/Cargo.toml | 2 + rust/crd/src/authentication.rs | 633 ++++++++++++++++++++++++++++++++- 3 files changed, 636 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index eccb3da7..df9fd9b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2143,6 +2143,7 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" name = "stackable-airflow-crd" version = "0.0.0-dev" dependencies = [ + "indoc", "product-config", "rstest", "serde", @@ -2151,6 +2152,7 @@ dependencies = [ "snafu 0.8.5", "stackable-operator", "strum", + "tokio", "tracing", ] diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml index 4548c22e..c3c7f31d 100644 --- a/rust/crd/Cargo.toml +++ b/rust/crd/Cargo.toml @@ -16,6 +16,8 @@ stackable-operator.workspace = true product-config.workspace = true strum.workspace = true tracing.workspace = true +indoc.workspace = true +tokio.workspace = true [dev-dependencies] serde_yaml.workspace = true diff --git a/rust/crd/src/authentication.rs b/rust/crd/src/authentication.rs index ff98dfb1..48cb6b42 100644 --- a/rust/crd/src/authentication.rs +++ b/rust/crd/src/authentication.rs @@ -51,7 +51,7 @@ pub enum Error { oidc_provider: String, }, #[snafu(display( - "TLS verification cannot be disabled in Superset (AuthenticationClass {auth_class_name:?})." + "TLS verification cannot be disabled in Airflow (AuthenticationClass {auth_class_name:?})." ))] TlsVerificationCannotBeDisabled { auth_class_name: String }, #[snafu(display( @@ -311,3 +311,634 @@ impl AirflowClientAuthenticationDetailsResolved { }) } } + +#[cfg(test)] +mod tests { + use std::pin::Pin; + + use indoc::indoc; + use stackable_operator::commons::networking::HostName; + use stackable_operator::commons::tls_verification::{ + CaCert, Tls, TlsClientDetails, TlsServerVerification, TlsVerification, + }; + use stackable_operator::{commons::authentication::oidc, kube}; + + use super::*; + + #[tokio::test] + async fn resolve_without_authentication_details() { + let auth_details_resolved = test_resolve_and_expect_success("[]", "").await; + + assert_eq!( + AirflowClientAuthenticationDetailsResolved { + authentication_classes_resolved: Vec::default(), + user_registration: default_user_registration(), + user_registration_role: default_user_registration_role(), + sync_roles_at: FlaskRolesSyncMoment::default() + }, + auth_details_resolved + ); + } + + #[tokio::test] + async fn resolve_ldap_with_all_authentication_details() { + // Avoid using defaults here + let auth_details_resolved = test_resolve_and_expect_success( + indoc! {" + - authenticationClass: ldap + userRegistration: false + userRegistrationRole: Gamma + syncRolesAt: Login + "}, + indoc! {" + --- + apiVersion: authentication.stackable.tech/v1alpha1 + kind: AuthenticationClass + metadata: + name: ldap + spec: + provider: + ldap: + hostname: my.ldap.server + "}, + ) + .await; + + assert_eq!( + AirflowClientAuthenticationDetailsResolved { + authentication_classes_resolved: vec![AirflowAuthenticationClassResolved::Ldap { + provider: serde_yaml::from_str("hostname: my.ldap.server").unwrap() + }], + user_registration: false, + user_registration_role: "Gamma".into(), + sync_roles_at: FlaskRolesSyncMoment::Login + }, + auth_details_resolved + ); + } + + #[tokio::test] + async fn resolve_oidc_with_all_authentication_details() { + // Avoid using defaults here + let auth_details_resolved = test_resolve_and_expect_success( + indoc! {" + - authenticationClass: oidc1 + oidc: + clientCredentialsSecret: airflow-oidc-client1 + extraScopes: + - groups + userRegistration: false + userRegistrationRole: Gamma + syncRolesAt: Login + - authenticationClass: oidc2 + oidc: + clientCredentialsSecret: airflow-oidc-client2 + userRegistration: false + userRegistrationRole: Gamma + syncRolesAt: Login + "}, + indoc! {" + --- + apiVersion: authentication.stackable.tech/v1alpha1 + kind: AuthenticationClass + metadata: + name: oidc1 + spec: + provider: + oidc: + hostname: first.oidc.server + port: 443 + rootPath: /realms/main + principalClaim: preferred_username + scopes: + - openid + - email + - profile + providerHint: Keycloak + tls: + verification: + server: + caCert: + secretClass: tls + --- + apiVersion: authentication.stackable.tech/v1alpha1 + kind: AuthenticationClass + metadata: + name: oidc2 + spec: + provider: + oidc: + hostname: second.oidc.server + rootPath: /realms/test + principalClaim: preferred_username + scopes: + - openid + - email + - profile + "}, + ) + .await; + + assert_eq!( + AirflowClientAuthenticationDetailsResolved { + authentication_classes_resolved: vec![ + AirflowAuthenticationClassResolved::Oidc { + provider: oidc::AuthenticationProvider::new( + HostName::try_from("first.oidc.server".to_string()).unwrap(), + Some(443), + "/realms/main".into(), + TlsClientDetails { + tls: Some(Tls { + verification: TlsVerification::Server(TlsServerVerification { + ca_cert: CaCert::SecretClass("tls".into()) + }) + }) + }, + "preferred_username".into(), + vec!["openid".into(), "email".into(), "profile".into()], + Some(IdentityProviderHint::Keycloak) + ), + oidc: oidc::ClientAuthenticationOptions { + client_credentials_secret_ref: "airflow-oidc-client1".into(), + extra_scopes: vec!["groups".into()], + product_specific_fields: () + } + }, + AirflowAuthenticationClassResolved::Oidc { + provider: oidc::AuthenticationProvider::new( + HostName::try_from("second.oidc.server".to_string()).unwrap(), + None, + "/realms/test".into(), + TlsClientDetails { tls: None }, + "preferred_username".into(), + vec!["openid".into(), "email".into(), "profile".into()], + None + ), + oidc: oidc::ClientAuthenticationOptions { + client_credentials_secret_ref: "airflow-oidc-client2".into(), + extra_scopes: Vec::new(), + product_specific_fields: () + } + } + ], + user_registration: false, + user_registration_role: "Gamma".into(), + sync_roles_at: FlaskRolesSyncMoment::Login + }, + auth_details_resolved + ); + } + + #[tokio::test] + async fn reject_duplicate_authentication_class_references() { + let error_message = test_resolve_and_expect_error( + indoc! {" + - authenticationClass: oidc + oidc: + clientCredentialsSecret: airflow-oidc-client1 + - authenticationClass: oidc + oidc: + clientCredentialsSecret: airflow-oidc-client2 + "}, + indoc! {" + --- + apiVersion: authentication.stackable.tech/v1alpha1 + kind: AuthenticationClass + metadata: + name: oidc + spec: + provider: + oidc: + hostname: my.oidc.server + principalClaim: preferred_username + scopes: [] + "}, + ) + .await; + + assert_eq!( + r#"The AuthenticationClass "oidc" is referenced several times which is not allowed."#, + error_message + ); + } + #[tokio::test] + async fn reject_different_authentication_types() { + let error_message = test_resolve_and_expect_error( + indoc! {" + - authenticationClass: oidc + oidc: + clientCredentialsSecret: airflow-oidc-client + - authenticationClass: ldap + "}, + indoc! {" + --- + apiVersion: authentication.stackable.tech/v1alpha1 + kind: AuthenticationClass + metadata: + name: oidc + spec: + provider: + oidc: + hostname: my.oidc.server + principalClaim: preferred_username + scopes: [] + --- + apiVersion: authentication.stackable.tech/v1alpha1 + kind: AuthenticationClass + metadata: + name: ldap + spec: + provider: + ldap: + hostname: my.ldap.server + "}, + ) + .await; + + assert_eq!( + "Only one authentication type at a time is supported by Airflow, see https://github.com/dpgaspar/Flask-AppBuilder/issues/1924.", + error_message + ); + } + + #[tokio::test] + async fn reject_multiple_ldap_providers() { + let error_message = test_resolve_and_expect_error( + indoc! {" + - authenticationClass: ldap1 + - authenticationClass: ldap2 + "}, + indoc! {" + --- + apiVersion: authentication.stackable.tech/v1alpha1 + kind: AuthenticationClass + metadata: + name: ldap1 + spec: + provider: + ldap: + hostname: first.ldap.server + --- + apiVersion: authentication.stackable.tech/v1alpha1 + kind: AuthenticationClass + metadata: + name: ldap2 + spec: + provider: + ldap: + hostname: second.ldap.server + "}, + ) + .await; + + assert_eq!( + "Only one LDAP provider at a time is supported by Airflow.", + error_message + ); + } + + #[tokio::test] + async fn reject_different_user_registration_settings() { + let error_message = test_resolve_and_expect_error( + indoc! {" + - authenticationClass: oidc1 + oidc: + clientCredentialsSecret: superset-oidc-client1 + - authenticationClass: oidc2 + oidc: + clientCredentialsSecret: superset-oidc-client2 + userRegistration: false + "}, + indoc! {" + --- + apiVersion: authentication.stackable.tech/v1alpha1 + kind: AuthenticationClass + metadata: + name: oidc1 + spec: + provider: + oidc: + hostname: first.oidc.server + principalClaim: preferred_username + scopes: [] + --- + apiVersion: authentication.stackable.tech/v1alpha1 + kind: AuthenticationClass + metadata: + name: oidc2 + spec: + provider: + oidc: + hostname: second.oidc.server + principalClaim: preferred_username + scopes: [] + "}, + ) + .await; + + assert_eq!( + "The userRegistration settings must not differ between the authentication entries.", + error_message + ); + } + + #[tokio::test] + async fn reject_different_user_registration_role_settings() { + let error_message = test_resolve_and_expect_error( + indoc! {" + - authenticationClass: oidc1 + oidc: + clientCredentialsSecret: airflow-oidc-client1 + - authenticationClass: oidc2 + oidc: + clientCredentialsSecret: airflow-oidc-client2 + userRegistrationRole: Gamma + "}, + indoc! {" + --- + apiVersion: authentication.stackable.tech/v1alpha1 + kind: AuthenticationClass + metadata: + name: oidc1 + spec: + provider: + oidc: + hostname: first.oidc.server + principalClaim: preferred_username + scopes: [] + --- + apiVersion: authentication.stackable.tech/v1alpha1 + kind: AuthenticationClass + metadata: + name: oidc2 + spec: + provider: + oidc: + hostname: second.oidc.server + principalClaim: preferred_username + scopes: [] + "}, + ) + .await; + + assert_eq!( + "The userRegistrationRole settings must not differ between the authentication entries.", + error_message + ); + } + + #[tokio::test] + async fn reject_different_sync_roles_at_settings() { + let error_message = test_resolve_and_expect_error( + indoc! {" + - authenticationClass: oidc1 + oidc: + clientCredentialsSecret: airflow-oidc-client1 + - authenticationClass: oidc2 + oidc: + clientCredentialsSecret: airflow-oidc-client2 + syncRolesAt: Login + "}, + indoc! {" + --- + apiVersion: authentication.stackable.tech/v1alpha1 + kind: AuthenticationClass + metadata: + name: oidc1 + spec: + provider: + oidc: + hostname: first.oidc.server + principalClaim: preferred_username + scopes: [] + --- + apiVersion: authentication.stackable.tech/v1alpha1 + kind: AuthenticationClass + metadata: + name: oidc2 + spec: + provider: + oidc: + hostname: second.oidc.server + principalClaim: preferred_username + scopes: [] + "}, + ) + .await; + + assert_eq!( + "The syncRolesAt settings must not differ between the authentication entries.", + error_message + ); + } + + #[tokio::test] + async fn reject_if_oidc_details_are_missing() { + let error_message = test_resolve_and_expect_error( + indoc! {" + - authenticationClass: oidc + "}, + indoc! {" + --- + apiVersion: authentication.stackable.tech/v1alpha1 + kind: AuthenticationClass + metadata: + name: oidc + spec: + provider: + oidc: + hostname: my.oidc.server + principalClaim: preferred_username + scopes: [] + "}, + ) + .await; + + assert_eq!( + indoc! { r#" + Invalid OIDC configuration + Caused by this error: + 1: OIDC authentication details not specified. The AuthenticationClass "oidc" uses an OIDC provider, you need to specify OIDC authentication details (such as client credentials) as well"# + }, + error_message + ); + } + + #[tokio::test] + async fn reject_wrong_principal_claim() { + let error_message = test_resolve_and_expect_error( + indoc! {" + - authenticationClass: oidc + oidc: + clientCredentialsSecret: airflow-oidc-client + "}, + indoc! {" + --- + apiVersion: authentication.stackable.tech/v1alpha1 + kind: AuthenticationClass + metadata: + name: oidc + spec: + provider: + oidc: + hostname: my.oidc.server + principalClaim: sub + scopes: [] + "}, + ) + .await; + + assert_eq!( + r#""sub" is not a supported principalClaim in Airflow for the Keycloak OIDC provider. Please use "preferred_username" in the AuthenticationClass "oidc""#, + error_message + ); + } + + #[tokio::test] + async fn reject_disabled_tls_verification() { + let error_message = test_resolve_and_expect_error( + indoc! {" + - authenticationClass: oidc + oidc: + clientCredentialsSecret: airflow-oidc-client + "}, + indoc! {" + --- + apiVersion: authentication.stackable.tech/v1alpha1 + kind: AuthenticationClass + metadata: + name: oidc + spec: + provider: + oidc: + hostname: my.oidc.server + principalClaim: preferred_username + scopes: [] + tls: + verification: + none: {} + "}, + ) + .await; + + assert_eq!( + r#"TLS verification cannot be disabled in Airflow (AuthenticationClass "oidc")."#, + error_message + ); + } + + /// Call `AirflowClientAuthenticationDetailsResolved::resolve` with + /// the given lists of `AirflowClientAuthenticationDetails` and + /// `AuthenticationClass`es and return the + /// `AirflowClientAuthenticationDetailsResolved`. + /// + /// The parameters are meant to be valid and resolvable. Just fail + /// if there is an error. + async fn test_resolve_and_expect_success( + auth_details_yaml: &str, + auth_classes_yaml: &str, + ) -> AirflowClientAuthenticationDetailsResolved { + test_resolve(auth_details_yaml, auth_classes_yaml) + .await + .expect("The AirflowClientAuthenticationDetails should be resolvable.") + } + + /// Call `AirflowClientAuthenticationDetailsResolved::resolve` with + /// the given lists of `AirflowClientAuthenticationDetails` and + /// `AuthenticationClass`es and return the error message. + /// + /// The parameters are meant to be invalid or not resolvable. Just + /// fail if there is no error. + async fn test_resolve_and_expect_error( + auth_details_yaml: &str, + auth_classes_yaml: &str, + ) -> String { + let error = test_resolve(auth_details_yaml, auth_classes_yaml) + .await + .expect_err( + "The AirflowClientAuthenticationDetails are invalid and should not be resolvable.", + ); + snafu::Report::from_error(error) + .to_string() + .trim_end() + .to_owned() + } + + /// Call `AirflowClientAuthenticationDetailsResolved::resolve` with + /// the given lists of `AirflowClientAuthenticationDetails` and + /// `AuthenticationClass`es and return the result. + async fn test_resolve( + auth_details_yaml: &str, + auth_classes_yaml: &str, + ) -> Result { + let auth_details = deserialize_airflow_client_authentication_details(auth_details_yaml); + + let auth_classes = deserialize_auth_classes(auth_classes_yaml); + + let resolve_auth_class = create_auth_class_resolver(auth_classes); + + AirflowClientAuthenticationDetailsResolved::resolve(&auth_details, resolve_auth_class).await + } + + /// Deserialize the given list of + /// `AirflowClientAuthenticationDetails`. + /// + /// Fail if the given string cannot be deserialized. + fn deserialize_airflow_client_authentication_details( + input: &str, + ) -> Vec { + serde_yaml::from_str(input) + .expect("The definition of the authentication configuration should be valid.") + } + + /// Deserialize the given `AuthenticationClass` YAML documents. + /// + /// Fail if the given string cannot be deserialized. + fn deserialize_auth_classes(input: &str) -> Vec { + if input.is_empty() { + Vec::new() + } else { + let deserializer = serde_yaml::Deserializer::from_str(input); + deserializer + .map(|d| { + serde_yaml::with::singleton_map_recursive::deserialize(d) + .expect("The definition of the AuthenticationClass should be valid.") + }) + .collect() + } + } + /// Returns a function which resolves `AuthenticationClass` names to + /// the given list of `AuthenticationClass`es. + /// + /// Use this function in the tests to replace + /// `stackable_operator::commons::authentication::ClientAuthenticationDetails` + /// which requires a Kubernetes client. + fn create_auth_class_resolver( + auth_classes: Vec, + ) -> impl Fn( + ClientAuthenticationDetails, + ) -> Pin< + Box>>, + > { + move |auth_details: ClientAuthenticationDetails| { + let auth_classes = auth_classes.clone(); + Box::pin(async move { + auth_classes + .iter() + .find(|auth_class| { + auth_class.metadata.name.as_ref() + == Some(auth_details.authentication_class_name()) + }) + .cloned() + .ok_or_else(|| stackable_operator::client::Error::ListResources { + source: kube::Error::Api(kube::error::ErrorResponse { + code: 404, + message: "AuthenticationClass not found".into(), + reason: "NotFound".into(), + status: "Failure".into(), + }), + }) + }) + } + } +} From ec14d14d5a145c3560011658e429a5f390b77d08 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 24 Oct 2024 16:26:07 +0200 Subject: [PATCH 07/25] WIP before merge --- .../operator-binary/src/airflow_controller.rs | 164 ++++++++++++++---- rust/operator-binary/src/util.rs | 6 + 2 files changed, 132 insertions(+), 38 deletions(-) diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 81d74c03..ccbb1ad7 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -5,14 +5,16 @@ use product_config::{ ProductConfigManager, }; use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_airflow_crd::git_sync::GitSync; use stackable_airflow_crd::{ - authentication::AirflowAuthenticationConfigResolved, build_recommended_labels, AirflowCluster, - AirflowClusterStatus, AirflowConfig, AirflowConfigFragment, AirflowConfigOptions, - AirflowExecutor, AirflowRole, Container, ExecutorConfig, ExecutorConfigFragment, - AIRFLOW_CONFIG_FILENAME, AIRFLOW_UID, APP_NAME, CONFIG_PATH, GIT_CONTENT, GIT_ROOT, - GIT_SYNC_NAME, LOG_CONFIG_DIR, OPERATOR_NAME, STACKABLE_LOG_DIR, TEMPLATE_CONFIGMAP_NAME, - TEMPLATE_LOCATION, TEMPLATE_NAME, TEMPLATE_VOLUME_NAME, + authentication::AirflowAuthenticationClassResolved, git_sync::GitSync, +}; +use stackable_airflow_crd::{ + authentication::AirflowClientAuthenticationDetailsResolved, build_recommended_labels, + AirflowCluster, AirflowClusterStatus, AirflowConfig, AirflowConfigFragment, + AirflowConfigOptions, AirflowExecutor, AirflowRole, Container, ExecutorConfig, + ExecutorConfigFragment, AIRFLOW_CONFIG_FILENAME, AIRFLOW_UID, APP_NAME, CONFIG_PATH, + GIT_CONTENT, GIT_ROOT, GIT_SYNC_NAME, LOG_CONFIG_DIR, OPERATOR_NAME, STACKABLE_LOG_DIR, + TEMPLATE_CONFIGMAP_NAME, TEMPLATE_LOCATION, TEMPLATE_NAME, TEMPLATE_VOLUME_NAME, }; use stackable_operator::k8s_openapi::api::core::v1::{EnvVar, PodTemplateSpec, VolumeMount}; use stackable_operator::kube::api::ObjectMeta; @@ -27,7 +29,7 @@ use stackable_operator::{ }, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, commons::{ - authentication::{ldap, AuthenticationClass, AuthenticationClassProvider}, + authentication::{ldap, oidc, AuthenticationClass}, product_image_selection::ResolvedProductImage, rbac::build_rbac_resources, }, @@ -67,7 +69,7 @@ use stackable_operator::{ utils::COMMON_BASH_TRAP_FUNCTIONS, }; use std::{ - collections::{BTreeMap, HashMap}, + collections::{BTreeMap, BTreeSet, HashMap}, io::Write, str::FromStr, sync::Arc, @@ -88,6 +90,7 @@ use crate::{ pdb::add_pdbs, }, product_logging::{extend_config_map_with_log_config, resolve_vector_aggregator_address}, + util::add_cert_to_python_certifi_command, }; pub const AIRFLOW_CONTROLLER_NAME: &str = "airflowcluster"; @@ -289,6 +292,16 @@ pub enum Error { "failed to write to String (Vec to be precise) containing Airflow config" ))] WriteToConfigFileString { source: std::io::Error }, + + #[snafu(display("failed to add TLS Volumes and VolumeMounts"))] + AddTlsVolumesAndVolumeMounts { + source: stackable_operator::commons::tls_verification::TlsClientDetailsError, + }, + + #[snafu(display("failed to add LDAP Volumes and VolumeMounts"))] + AddLdapVolumesAndVolumeMounts { + source: stackable_operator::commons::authentication::ldap::Error, + }, } type Result = std::result::Result; @@ -311,13 +324,12 @@ pub async fn reconcile_airflow(airflow: Arc, ctx: Arc) -> R let cluster_operation_cond_builder = ClusterOperationsConditionBuilder::new(&airflow.spec.cluster_operation); - let authentication_config = airflow - .spec - .cluster_config - .authentication - .resolve(client) - .await - .context(InvalidAuthenticationConfigSnafu)?; + let authentication_config = AirflowClientAuthenticationDetailsResolved::from( + &airflow.spec.cluster_config.authentication, + client, + ) + .await + .context(InvalidAuthenticationConfigSnafu)?; let mut roles = HashMap::new(); @@ -450,7 +462,7 @@ pub async fn reconcile_airflow(airflow: Arc, ctx: Arc) -> R &airflow_role, &rolegroup, rolegroup_config, - authentication_config.as_ref(), + &authentication_config, &rbac_sa.name_unchecked(), &merged_airflow_config, airflow_executor, @@ -470,7 +482,7 @@ pub async fn reconcile_airflow(airflow: Arc, ctx: Arc) -> R &resolved_product_image, &rolegroup, rolegroup_config, - authentication_config.as_ref(), + &authentication_config, &merged_airflow_config.logging, vector_aggregator_address.as_deref(), &Container::Airflow, @@ -519,7 +531,7 @@ async fn build_executor_template( airflow: &Arc, common_config: &CommonConfiguration, resolved_product_image: &ResolvedProductImage, - authentication_config: &Vec, + authentication_config: &AirflowClientAuthenticationDetailsResolved, vector_aggregator_address: &Option, cluster_resources: &mut ClusterResources, client: &stackable_operator::client::Client, @@ -538,7 +550,7 @@ async fn build_executor_template( resolved_product_image, &rolegroup, &HashMap::new(), - authentication_config, + &authentication_config, &merged_executor_config.logging, vector_aggregator_address.as_deref(), &Container::Base, @@ -635,7 +647,7 @@ fn build_rolegroup_config_map( resolved_product_image: &ResolvedProductImage, rolegroup: &RoleGroupRef, rolegroup_config: &HashMap>, - authentication_config: &Vec, + authentication_config: &AirflowClientAuthenticationDetailsResolved, logging: &Logging, vector_aggregator_address: Option<&str>, container: &Container, @@ -797,7 +809,7 @@ fn build_server_rolegroup_statefulset( airflow_role: &AirflowRole, rolegroup_ref: &RoleGroupRef, rolegroup_config: &HashMap>, - authentication_config: &Vec, + authentication_config: &AirflowClientAuthenticationDetailsResolved, sa_name: &str, merged_airflow_config: &AirflowConfig, executor: &AirflowExecutor, @@ -1059,7 +1071,7 @@ fn build_logging_container( fn build_executor_template_config_map( airflow: &AirflowCluster, resolved_product_image: &ResolvedProductImage, - authentication_config: &Vec, + authentication_config: &AirflowClientAuthenticationDetailsResolved, sa_name: &str, merged_executor_config: &ExecutorConfig, env_overrides: &HashMap, @@ -1104,7 +1116,7 @@ fn build_executor_template_config_map( &mut airflow_container, &mut pb, )?; - + // TODO: Propagate error rather then ignore it with unwrap() airflow_container .image_from_product_image(resolved_product_image) .resources(merged_executor_config.resources.clone().into()) @@ -1114,8 +1126,11 @@ fn build_executor_template_config_map( merged_executor_config, )) .add_volume_mounts(airflow.volume_mounts()) + .unwrap() .add_volume_mount(CONFIG_VOLUME_NAME, CONFIG_PATH) + .unwrap() .add_volume_mount(LOG_CONFIG_VOLUME_NAME, LOG_CONFIG_DIR) + .unwrap() .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR); pb.add_container(airflow_container.build()); @@ -1210,7 +1225,9 @@ fn build_gitsync_container( ]) .args(vec![gitsync.get_args(one_time).join("\n")]) .add_volume_mount(GIT_CONTENT, GIT_ROOT) + .unwrap() .add_volume_mounts(volume_mounts) + .unwrap() .resources( ResourceRequirementsBuilder::new() .with_cpu_request("100m") @@ -1226,27 +1243,98 @@ fn build_gitsync_container( pub fn error_policy(_obj: Arc, _error: &Error, _ctx: Arc) -> Action { Action::requeue(*Duration::from_secs(5)) } +// I want to add secret volumes right here fn add_authentication_volumes_and_volume_mounts( - authentication_config: &Vec, + authentication_config: &AirflowClientAuthenticationDetailsResolved, cb: &mut ContainerBuilder, pb: &mut PodBuilder, ) -> Result<()> { - // TODO: Currently there can be only one AuthenticationClass due to FlaskAppBuilder restrictions. - // Needs adaptation once FAB and airflow support multiple auth methods. - // The checks for max one AuthenticationClass and the provider are done in crd/src/authentication.rs - for config in authentication_config { - if let Some(auth_class) = &config.authentication_class { - match &auth_class.spec.provider { - AuthenticationClassProvider::Ldap(ldap) => { - ldap.add_volumes_and_mounts(pb, vec![cb]) - .context(VolumeAndMountsSnafu)?; - } - AuthenticationClassProvider::Tls(_) - | AuthenticationClassProvider::Oidc(_) - | AuthenticationClassProvider::Static(_) => {} + // Different authentication entries can reference the same secret + // class or TLS certificate. It must be ensured that the volumes + // and volume mounts are only added once in such a case. + + let mut ldap_authentication_providers = BTreeSet::new(); + let mut tls_client_credentials = BTreeSet::new(); + + for auth_class_resolved in &authentication_config.authentication_classes_resolved { + match auth_class_resolved { + AirflowAuthenticationClassResolved::Ldap { provider } => { + ldap_authentication_providers.insert(provider); + } + AirflowAuthenticationClassResolved::Oidc { provider, oidc } => { + tls_client_credentials.insert(&provider.tls); } } } + + for provider in ldap_authentication_providers { + provider + .add_volumes_and_mounts(pb, vec![cb]) + .context(AddLdapVolumesAndVolumeMountsSnafu)?; + } + + for tls in tls_client_credentials { + tls.add_volumes_and_mounts(pb, vec![cb]) + .context(AddTlsVolumesAndVolumeMountsSnafu)?; + } Ok(()) } + +fn authentication_env_vars( + auth_config: &AirflowClientAuthenticationDetailsResolved, +) -> Vec { + // Different OIDC authentication entries can reference the same + // client secret. It must be ensured that the env variables are only + // added once in such a case. + + let mut oidc_client_credentials_secrets = BTreeSet::new(); + + for auth_class_resolved in &auth_config.authentication_classes_resolved { + match auth_class_resolved { + AirflowAuthenticationClassResolved::Ldap { .. } => {} + AirflowAuthenticationClassResolved::Oidc { oidc, .. } => { + oidc_client_credentials_secrets + .insert(oidc.client_credentials_secret_ref.to_owned()); + } + } + } + + oidc_client_credentials_secrets + .iter() + .cloned() + .flat_map(oidc::AuthenticationProvider::client_credentials_env_var_mounts) + .collect() +} + +fn authentication_start_commands( + auth_config: &AirflowClientAuthenticationDetailsResolved, +) -> String { + let mut commands = Vec::new(); + + let mut tls_client_credentials = BTreeSet::new(); + + for auth_class_resolved in &auth_config.authentication_classes_resolved { + match auth_class_resolved { + AirflowAuthenticationClassResolved::Oidc { provider, .. } => { + tls_client_credentials.insert(&provider.tls); + + // WebPKI will be handled implicitly + } + AirflowAuthenticationClassResolved::Ldap { .. } => {} + } + } + + for tls in tls_client_credentials { + commands.push(tls.tls_ca_cert_mount_path().map(|tls_ca_cert_mount_path| { + add_cert_to_python_certifi_command(&tls_ca_cert_mount_path) + })); + } + + commands + .iter() + .flatten() + .cloned() + .collect::>() + .join("\n") +} diff --git a/rust/operator-binary/src/util.rs b/rust/operator-binary/src/util.rs index 38e70f21..85ef4032 100644 --- a/rust/operator-binary/src/util.rs +++ b/rust/operator-binary/src/util.rs @@ -14,3 +14,9 @@ pub fn env_var_from_secret(var_name: &str, secret: &str, secret_key: &str) -> En ..Default::default() } } + +/// Adds a CA file from `cert_file` to the root certificates of Python Certifi + +pub fn add_cert_to_python_certifi_command(cert_file: &str) -> String { + format!("cat {cert_file} >> \"$(python -c 'import certifi; print(certifi.where())')\"") +} From bffe07644315a7f8a290ed50234c7729b7181219 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 28 Oct 2024 10:32:21 +0100 Subject: [PATCH 08/25] WIP compiling operator --- Cargo.lock | 9 +++++++++ Cargo.toml | 1 + .../operator-binary/src/airflow_controller.rs | 19 ++++++++++++------- rust/operator-binary/src/main.rs | 10 +++++----- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f3bfa6e..247ba76a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1037,6 +1037,12 @@ dependencies = [ "hashbrown 0.15.0", ] +[[package]] +name = "indoc" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" + [[package]] name = "instant" version = "0.1.13" @@ -2168,6 +2174,7 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" name = "stackable-airflow-crd" version = "0.0.0-dev" dependencies = [ + "indoc", "product-config", "rstest", "serde", @@ -2176,6 +2183,7 @@ dependencies = [ "snafu 0.8.5", "stackable-operator", "strum", + "tokio", "tracing", ] @@ -2188,6 +2196,7 @@ dependencies = [ "clap", "fnv", "futures 0.3.31", + "indoc", "product-config", "serde", "serde_yaml", diff --git a/Cargo.toml b/Cargo.toml index be77133e..e76c5e35 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", strum = { version = "0.26", features = ["derive"] } tokio = { version = "1.40", features = ["full"] } tracing = "0.1" +indoc = "2.0" # [patch."https://github.com/stackabletech/operator-rs.git"] # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index cef4bf6a..4c4be227 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -1,6 +1,6 @@ //! Ensures that `Pod`s are configured and running for each [`AirflowCluster`] use std::{ - collections::{BTreeMap, HashMap}, + collections::{BTreeMap, BTreeSet, HashMap}, io::Write, str::FromStr, sync::Arc, @@ -77,12 +77,7 @@ use stackable_operator::{ time::Duration, utils::COMMON_BASH_TRAP_FUNCTIONS, }; -use std::{ - collections::{BTreeMap, BTreeSet, HashMap}, - io::Write, - str::FromStr, - sync::Arc, -}; + use strum::{EnumDiscriminants, IntoEnumIterator, IntoStaticStr}; use crate::{ @@ -312,6 +307,16 @@ pub enum Error { source: builder::pod::container::Error, }, + #[snafu(display("failed to add LDAP Volumes and VolumeMounts"))] + AddLdapVolumesAndVolumeMounts { + source: stackable_operator::commons::authentication::ldap::Error, + }, + + #[snafu(display("failed to add TLS Volumes and VolumeMounts"))] + AddTlsVolumesAndVolumeMounts { + source: stackable_operator::commons::tls_verification::TlsClientDetailsError, + }, + #[snafu(display("AirflowCluster object is invalid"))] InvalidAirflowCluster { source: error_boundary::InvalidObject, diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index a9d73738..83cbff3f 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -21,6 +21,7 @@ use stackable_operator::{ reflector::{Lookup, ObjectRef}, watcher, Controller, }, + ResourceExt, }, logging::controller::report_controller_reconciled, CustomResourceExt, @@ -135,13 +136,12 @@ fn references_authentication_class( let Ok(airflow) = &airflow.0 else { return false; }; - let Some(authn_class_name) = authentication_class.name() else { - return false; - }; + let authentication_class_name = authentication_class.name_any(); + airflow .spec .cluster_config .authentication - .authentication_class_names() - .contains(&&*authn_class_name) + .iter() + .any(|c| c.common.authentication_class_name() == &authentication_class_name) } From b1d2d2f7fd2a3c1355f916887def94a83f539b5c Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 28 Oct 2024 14:08:24 +0100 Subject: [PATCH 09/25] Fixng auth tests --- rust/crd/src/authentication.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/rust/crd/src/authentication.rs b/rust/crd/src/authentication.rs index 48cb6b42..d6743800 100644 --- a/rust/crd/src/authentication.rs +++ b/rust/crd/src/authentication.rs @@ -346,20 +346,20 @@ mod tests { let auth_details_resolved = test_resolve_and_expect_success( indoc! {" - authenticationClass: ldap - userRegistration: false - userRegistrationRole: Gamma - syncRolesAt: Login + userRegistration: false + userRegistrationRole: Gamma + syncRolesAt: Login "}, indoc! {" --- apiVersion: authentication.stackable.tech/v1alpha1 kind: AuthenticationClass metadata: - name: ldap + name: ldap spec: - provider: + provider: ldap: - hostname: my.ldap.server + hostname: my.ldap.server "}, ) .await; @@ -757,9 +757,9 @@ mod tests { assert_eq!( indoc! { r#" Invalid OIDC configuration + Caused by this error: - 1: OIDC authentication details not specified. The AuthenticationClass "oidc" uses an OIDC provider, you need to specify OIDC authentication details (such as client credentials) as well"# - }, + 1: authentication details for OIDC were not specified. The AuthenticationClass "oidc" uses an OIDC provider, you need to specify OIDC authentication details (such as client credentials) as well"# }, error_message ); } From 5a8e0e7f54618dde377e3c4938e8d351d80aa7a2 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Mon, 28 Oct 2024 16:56:42 +0100 Subject: [PATCH 10/25] Update the OIDC integration test --- .../kuttl/oidc/install-airflow.yaml.j2 | 4 +++ tests/templates/kuttl/oidc/login.py | 25 ++++++++----------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/tests/templates/kuttl/oidc/install-airflow.yaml.j2 b/tests/templates/kuttl/oidc/install-airflow.yaml.j2 index 9be91877..868b83ab 100644 --- a/tests/templates/kuttl/oidc/install-airflow.yaml.j2 +++ b/tests/templates/kuttl/oidc/install-airflow.yaml.j2 @@ -48,9 +48,11 @@ spec: - authenticationClass: keycloak1-$NAMESPACE oidc: clientCredentialsSecret: airflow-keycloak1-client + userRegistrationRole: Admin - authenticationClass: keycloak2-$NAMESPACE oidc: clientCredentialsSecret: airflow-keycloak2-client + userRegistrationRole: Admin credentialsSecret: airflow-credentials {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery @@ -59,6 +61,8 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + envOverrides: + REQUESTS_CA_BUNDLE: /stackable/secrets/tls/ca.crt roleGroups: default: replicas: 1 diff --git a/tests/templates/kuttl/oidc/login.py b/tests/templates/kuttl/oidc/login.py index f52a2d56..4fa243d9 100644 --- a/tests/templates/kuttl/oidc/login.py +++ b/tests/templates/kuttl/oidc/login.py @@ -1,6 +1,5 @@ # $NAMESPACE will be replaced with the namespace of the test case. -import json import logging import requests import sys @@ -23,33 +22,29 @@ # Enter username and password into the Keycloak login page and click on "Sign In" login_page_html = BeautifulSoup(login_page.text, 'html.parser') authenticate_url = login_page_html.form['action'] -welcome_page = session.post(authenticate_url, data={ - 'username': "jane.doe", - 'password': "T8mn72D9" -}) +welcome_page = session.post(authenticate_url, data={ 'username': "jane.doe", 'password': "T8mn72D9" }) assert welcome_page.ok, "Login failed" -assert welcome_page.url == "http://airflow-webserver:8080/airflow/welcome/", \ - "Redirection to the Airflow welcome page expected" +assert welcome_page.url == "http://airflow-webserver:8080/home", \ + "Redirection to the Airflow home page expected" # Open the user information page in Airflow userinfo_page = session.get("http://airflow-webserver:8080/users/userinfo/") assert userinfo_page.ok, "Retrieving user information failed" -assert userinfo_page.url == "http://airflow-webserver:8080/airflow/welcome/", \ - "Redirection to the Airflow welcome page expected" +assert userinfo_page.url == "http://airflow-webserver:8080/users/userinfo/", \ + "Redirection to the Airflow user info page expected" # Expect the user data provided by Keycloak in Airflow userinfo_page_html = BeautifulSoup(userinfo_page.text, 'html.parser') -raw_data = userinfo_page_html.find(id='app')['data-bootstrap'] -data = json.loads(raw_data) -user_data = data['user'] +table_rows = userinfo_page_html.find_all('tr') +user_data = {tr.find('th').text:tr.find('td').text for tr in table_rows} -assert user_data['firstName'] == "Jane", \ +assert user_data['First Name'] == "Jane", \ "The first name of the user in Airflow should match the one provided by Keycloak" -assert user_data['lastName'] == "Doe", \ +assert user_data['Last Name'] == "Doe", \ "The last name of the user in Airflow should match the one provided by Keycloak" -assert user_data['email'] == "jane.doe@stackable.tech", \ +assert user_data['Email'] == "jane.doe@stackable.tech", \ "The email of the user in Airflow should match the one provided by Keycloak" # TODO Use different OIDC providers (currently only Keycloak is From 18412e13f8e811d7d82bf8a2343266aa63264eb3 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 28 Oct 2024 18:08:12 +0100 Subject: [PATCH 11/25] Env vars oidc --- rust/crd/src/lib.rs | 2 +- .../operator-binary/src/airflow_controller.rs | 29 +----- rust/operator-binary/src/config.rs | 4 +- rust/operator-binary/src/env_vars.rs | 89 +++++++++++++------ 4 files changed, 68 insertions(+), 56 deletions(-) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 3e9dd6b0..40951267 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -189,7 +189,7 @@ pub struct AirflowClusterSpec { #[derive(Clone, Deserialize, Debug, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct AirflowClusterConfig { - #[serde(flatten)] + #[serde(default)] pub authentication: Vec, /// The name of the Secret object containing the admin user credentials and database connection details. diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 4c4be227..729d13da 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -902,6 +902,7 @@ fn build_server_rolegroup_statefulset( airflow_role, rolegroup_config, executor, + authentication_config, )); let volume_mounts = airflow.volume_mounts(); @@ -1152,12 +1153,12 @@ fn build_executor_template_config_map( let mut airflow_container = ContainerBuilder::new(&Container::Base.to_string()).context(InvalidContainerNameSnafu)?; + // Works too, had been changed add_authentication_volumes_and_volume_mounts( authentication_config, &mut airflow_container, &mut pb, )?; - // TODO: Propagate error rather then ignore it with unwrap() airflow_container .image_from_product_image(resolved_product_image) .resources(merged_executor_config.resources.clone().into()) @@ -1335,32 +1336,6 @@ fn add_authentication_volumes_and_volume_mounts( Ok(()) } -fn authentication_env_vars( - auth_config: &AirflowClientAuthenticationDetailsResolved, -) -> Vec { - // Different OIDC authentication entries can reference the same - // client secret. It must be ensured that the env variables are only - // added once in such a case. - - let mut oidc_client_credentials_secrets = BTreeSet::new(); - - for auth_class_resolved in &auth_config.authentication_classes_resolved { - match auth_class_resolved { - AirflowAuthenticationClassResolved::Ldap { .. } => {} - AirflowAuthenticationClassResolved::Oidc { oidc, .. } => { - oidc_client_credentials_secrets - .insert(oidc.client_credentials_secret_ref.to_owned()); - } - } - } - - oidc_client_credentials_secrets - .iter() - .cloned() - .flat_map(oidc::AuthenticationProvider::client_credentials_env_var_mounts) - .collect() -} - fn authentication_start_commands( auth_config: &AirflowClientAuthenticationDetailsResolved, ) -> String { diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index 6a595cf9..2d7feff7 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -194,10 +194,10 @@ fn append_oidc_config( )], ) { // Debatable: AUTH_OAUTH or AUTH_OID - // Additionally can be set via config... dunno + // Additionally can be set via config config.insert( AirflowConfigOptions::AuthType.to_string(), - "AUTH_OID".into(), + "AUTH_OAUTH".into(), ); let mut oauth_providers_config = Vec::new(); diff --git a/rust/operator-binary/src/env_vars.rs b/rust/operator-binary/src/env_vars.rs index 4d4a58dc..e1e77a5b 100644 --- a/rust/operator-binary/src/env_vars.rs +++ b/rust/operator-binary/src/env_vars.rs @@ -1,15 +1,19 @@ use crate::util::env_var_from_secret; use product_config::types::PropertyNameKind; +use stackable_airflow_crd::authentication::{ + AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, +}; use stackable_airflow_crd::git_sync::GitSync; use stackable_airflow_crd::{ AirflowCluster, AirflowConfig, AirflowExecutor, AirflowRole, ExecutorConfig, LOG_CONFIG_DIR, STACKABLE_LOG_DIR, }; use stackable_airflow_crd::{GIT_LINK, GIT_SYNC_DIR, TEMPLATE_LOCATION, TEMPLATE_NAME}; +use stackable_operator::commons::authentication::oidc; use stackable_operator::k8s_openapi::api::core::v1::EnvVar; use stackable_operator::kube::ResourceExt; use stackable_operator::product_logging::framework::create_vector_shutdown_file_command; -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; const AIRFLOW__LOGGING__LOGGING_CONFIG_CLASS: &str = "AIRFLOW__LOGGING__LOGGING_CONFIG_CLASS"; const AIRFLOW__METRICS__STATSD_ON: &str = "AIRFLOW__METRICS__STATSD_ON"; @@ -44,6 +48,7 @@ pub fn build_airflow_statefulset_envs( airflow_role: &AirflowRole, rolegroup_config: &HashMap>, executor: &AirflowExecutor, + auth_config: &AirflowClientAuthenticationDetailsResolved, ) -> Vec { let mut env: BTreeMap = BTreeMap::new(); @@ -166,32 +171,38 @@ pub fn build_airflow_statefulset_envs( ); } - // Database initialization is limited to the scheduler. - // See https://github.com/stackabletech/airflow-operator/issues/259 - if airflow_role == &AirflowRole::Scheduler { - let secret = &airflow.spec.cluster_config.credentials_secret; - env.insert( - ADMIN_USERNAME.into(), - env_var_from_secret(ADMIN_USERNAME, secret, "adminUser.username"), - ); - env.insert( - ADMIN_FIRSTNAME.into(), - env_var_from_secret(ADMIN_FIRSTNAME, secret, "adminUser.firstname"), - ); - env.insert( - ADMIN_LASTNAME.into(), - env_var_from_secret(ADMIN_LASTNAME, secret, "adminUser.lastname"), - ); - env.insert( - ADMIN_EMAIL.into(), - env_var_from_secret(ADMIN_EMAIL, secret, "adminUser.email"), - ); - env.insert( - ADMIN_PASSWORD.into(), - env_var_from_secret(ADMIN_PASSWORD, secret, "adminUser.password"), - ); + match airflow_role { + // Database initialization is limited to the scheduler. + // See https://github.com/stackabletech/airflow-operator/issues/259 + AirflowRole::Scheduler => { + let secret = &airflow.spec.cluster_config.credentials_secret; + env.insert( + ADMIN_USERNAME.into(), + env_var_from_secret(ADMIN_USERNAME, secret, "adminUser.username"), + ); + env.insert( + ADMIN_FIRSTNAME.into(), + env_var_from_secret(ADMIN_FIRSTNAME, secret, "adminUser.firstname"), + ); + env.insert( + ADMIN_LASTNAME.into(), + env_var_from_secret(ADMIN_LASTNAME, secret, "adminUser.lastname"), + ); + env.insert( + ADMIN_EMAIL.into(), + env_var_from_secret(ADMIN_EMAIL, secret, "adminUser.email"), + ); + env.insert( + ADMIN_PASSWORD.into(), + env_var_from_secret(ADMIN_PASSWORD, secret, "adminUser.password"), + ); + } + AirflowRole::Webserver => { + let auth_vars = authentication_env_vars(auth_config); + env.extend(auth_vars.into_iter().map(|var| (var.name.to_owned(), var))); + } + _ => {} } - // apply overrides last of all with a fixed ordering if let Some(env_vars) = env_vars { for (k, v) in env_vars.iter().collect::>() { @@ -454,3 +465,29 @@ fn gitsync_vars_map(k: &String, env: &mut BTreeMap, v: &String) fn transform_map_to_vec(env_map: BTreeMap) -> Vec { env_map.into_values().collect::>() } + +fn authentication_env_vars( + auth_config: &AirflowClientAuthenticationDetailsResolved, +) -> Vec { + // Different OIDC authentication entries can reference the same + // client secret. It must be ensured that the env variables are only + // added once in such a case. + + let mut oidc_client_credentials_secrets = BTreeSet::new(); + + for auth_class_resolved in &auth_config.authentication_classes_resolved { + match auth_class_resolved { + AirflowAuthenticationClassResolved::Ldap { .. } => {} + AirflowAuthenticationClassResolved::Oidc { oidc, .. } => { + oidc_client_credentials_secrets + .insert(oidc.client_credentials_secret_ref.to_owned()); + } + } + } + + oidc_client_credentials_secrets + .iter() + .cloned() + .flat_map(oidc::AuthenticationProvider::client_credentials_env_var_mounts) + .collect() +} From 72bb1e96bc04a1a6fb274a57de0e962c1c6f90ff Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Tue, 29 Oct 2024 09:48:25 +0100 Subject: [PATCH 12/25] Cleaning up --- .../operator-binary/src/airflow_controller.rs | 38 ++----------------- rust/operator-binary/src/main.rs | 5 +-- rust/operator-binary/src/util.rs | 6 --- 3 files changed, 4 insertions(+), 45 deletions(-) diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 729d13da..78f10bc1 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -35,7 +35,7 @@ use stackable_operator::{ }, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, commons::{ - authentication::{ldap, oidc, AuthenticationClass}, + authentication::{ldap, AuthenticationClass}, product_image_selection::ResolvedProductImage, rbac::build_rbac_resources, }, @@ -93,7 +93,6 @@ use crate::{ pdb::add_pdbs, }, product_logging::{extend_config_map_with_log_config, resolve_vector_aggregator_address}, - util::add_cert_to_python_certifi_command, }; pub const AIRFLOW_CONTROLLER_NAME: &str = "airflowcluster"; @@ -571,6 +570,7 @@ async fn build_executor_template( role: "executor".into(), role_group: "kubernetes".into(), }; + let rg_configmap = build_rolegroup_config_map( airflow, resolved_product_image, @@ -1317,7 +1317,7 @@ fn add_authentication_volumes_and_volume_mounts( AirflowAuthenticationClassResolved::Ldap { provider } => { ldap_authentication_providers.insert(provider); } - AirflowAuthenticationClassResolved::Oidc { provider, oidc } => { + AirflowAuthenticationClassResolved::Oidc { provider, .. } => { tls_client_credentials.insert(&provider.tls); } } @@ -1335,35 +1335,3 @@ fn add_authentication_volumes_and_volume_mounts( } Ok(()) } - -fn authentication_start_commands( - auth_config: &AirflowClientAuthenticationDetailsResolved, -) -> String { - let mut commands = Vec::new(); - - let mut tls_client_credentials = BTreeSet::new(); - - for auth_class_resolved in &auth_config.authentication_classes_resolved { - match auth_class_resolved { - AirflowAuthenticationClassResolved::Oidc { provider, .. } => { - tls_client_credentials.insert(&provider.tls); - - // WebPKI will be handled implicitly - } - AirflowAuthenticationClassResolved::Ldap { .. } => {} - } - } - - for tls in tls_client_credentials { - commands.push(tls.tls_ca_cert_mount_path().map(|tls_ca_cert_mount_path| { - add_cert_to_python_certifi_command(&tls_ca_cert_mount_path) - })); - } - - commands - .iter() - .flatten() - .cloned() - .collect::>() - .join("\n") -} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 83cbff3f..03c70ece 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -17,10 +17,7 @@ use stackable_operator::{ k8s_openapi::api::{apps::v1::StatefulSet, core::v1::Service}, kube::{ core::DeserializeGuard, - runtime::{ - reflector::{Lookup, ObjectRef}, - watcher, Controller, - }, + runtime::{reflector::ObjectRef, watcher, Controller}, ResourceExt, }, logging::controller::report_controller_reconciled, diff --git a/rust/operator-binary/src/util.rs b/rust/operator-binary/src/util.rs index 85ef4032..38e70f21 100644 --- a/rust/operator-binary/src/util.rs +++ b/rust/operator-binary/src/util.rs @@ -14,9 +14,3 @@ pub fn env_var_from_secret(var_name: &str, secret: &str, secret_key: &str) -> En ..Default::default() } } - -/// Adds a CA file from `cert_file` to the root certificates of Python Certifi - -pub fn add_cert_to_python_certifi_command(cert_file: &str) -> String { - format!("cat {cert_file} >> \"$(python -c 'import certifi; print(certifi.where())')\"") -} From 08a1019e3fdcc7a4df6e39b33d1a0437ef8fc105 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Tue, 29 Oct 2024 09:55:47 +0100 Subject: [PATCH 13/25] Adding changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26c95c09..68f9cbac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Use the env var `KUBERNETES_CLUSTER_DOMAIN` or the operator Helm chart property `kubernetesClusterDomain` to set a non-default cluster domain ([#518]). - Support for `2.9.3` ([#494]). - Experimental Support for `2.10.2` ([#512]). +- Add support for OpenID Connect ([#524]) ### Changed @@ -30,6 +31,7 @@ [#494]: https://github.com/stackabletech/airflow-operator/pull/494 [#518]: https://github.com/stackabletech/airflow-operator/pull/518 [#520]: https://github.com/stackabletech/airflow-operator/pull/520 +[#524]: https://github.com/stackabletech/airflow-operator/pull/524 ## [24.7.0] - 2024-07-24 From b67e026e1c6e35d8f2e340f9a7407a8d1f2f808e Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Tue, 29 Oct 2024 10:09:48 +0100 Subject: [PATCH 14/25] Remove reference --- rust/operator-binary/src/airflow_controller.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 78f10bc1..97c96713 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -576,7 +576,7 @@ async fn build_executor_template( resolved_product_image, &rolegroup, &HashMap::new(), - &authentication_config, + authentication_config, &merged_executor_config.logging, vector_aggregator_address.as_deref(), &Container::Base, From c2abb386a1fde94989bcdb6d3d8a507cb11e685f Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Tue, 29 Oct 2024 10:13:40 +0100 Subject: [PATCH 15/25] Regenerate charts and Nix files --- Cargo.nix | 425 ++++++++++++++------ crate-hashes.json | 5 +- deploy/helm/airflow-operator/crds/crds.yaml | 22 +- 3 files changed, 322 insertions(+), 130 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index 5670eeed..afc575fd 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -359,9 +359,9 @@ rec { }; "anyhow" = rec { crateName = "anyhow"; - version = "1.0.89"; + version = "1.0.91"; edition = "2018"; - sha256 = "1xh1vg89n56h6nqikcmgbpmkixjds33492klrp9m96xrbmhgizc6"; + sha256 = "1j2hg3d0lk2pihfsknxrncmijhz07fspkzd5c0vxhzyp7s7i0hn0"; authors = [ "David Tolnay " ]; @@ -448,7 +448,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; features = [ "full" "visit-mut" ]; } ]; @@ -475,7 +475,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; usesDefaultFeatures = false; features = [ "full" "parsing" "printing" "proc-macro" "visit-mut" ]; } @@ -701,9 +701,9 @@ rec { }; "built" = rec { crateName = "built"; - version = "0.7.4"; + version = "0.7.5"; edition = "2021"; - sha256 = "1i4jsd31b4xj93bz5vmvby4s50lk0cjb1hskdfy15a55xn4n4vi3"; + sha256 = "0fyhzjgymls3qylggd6rs4vkq44rkl1kyv33lfbfrdsjxmd50q63"; authors = [ "Lukas Lueg " ]; @@ -760,9 +760,9 @@ rec { }; "bytes" = rec { crateName = "bytes"; - version = "1.7.2"; + version = "1.8.0"; edition = "2018"; - sha256 = "1wzs7l57iwqmrszdpr2mmqf1b1hgvpxafc30imxhnry0zfl9m3a2"; + sha256 = "1nnhpb7jlpj393qnjr1n9n6sgpl3w5ymrwl3pnjmrriam861bh4s"; authors = [ "Carl Lerche " "Sean McArthur " @@ -775,9 +775,9 @@ rec { }; "cc" = rec { crateName = "cc"; - version = "1.1.30"; + version = "1.1.31"; edition = "2018"; - sha256 = "0icr3vn2r5scpgylbplffd5mh7jcdivqh9dfgsxymnc13fk06s5i"; + sha256 = "0vscf59yxf665s4fv9yn3l39gfw99mgp6wnbc76cyv80ahmrdry2"; authors = [ "Alex Crichton " ]; @@ -982,7 +982,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; features = [ "full" ]; } ]; @@ -1286,7 +1286,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; features = [ "full" "extra-traits" ]; } ]; @@ -1316,7 +1316,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; } ]; @@ -1342,7 +1342,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; features = [ "full" "visit-mut" ]; } ]; @@ -1496,6 +1496,48 @@ rec { ]; }; + "educe" = rec { + crateName = "educe"; + version = "0.6.0"; + edition = "2021"; + sha256 = "05wlg6wzil59lxc3kbs476gwykj6m63dbfv83frxx35xw54w0yqx"; + procMacro = true; + authors = [ + "Magic Len " + ]; + dependencies = [ + { + name = "enum-ordinalize"; + packageId = "enum-ordinalize"; + usesDefaultFeatures = false; + features = [ "derive" ]; + } + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "syn"; + packageId = "syn 2.0.82"; + } + ]; + devDependencies = [ + { + name = "syn"; + packageId = "syn 2.0.82"; + features = [ "full" ]; + } + ]; + features = { + "default" = [ "Debug" "Clone" "Copy" "PartialEq" "Eq" "PartialOrd" "Ord" "Hash" "Default" "Deref" "DerefMut" "Into" ]; + "full" = [ "syn/full" ]; + }; + resolvedDefaultFeatures = [ "Clone" "Debug" "Hash" "PartialEq" ]; + }; "either" = rec { crateName = "either"; version = "1.13.0"; @@ -1549,6 +1591,51 @@ rec { ]; }; + "enum-ordinalize" = rec { + crateName = "enum-ordinalize"; + version = "4.3.0"; + edition = "2021"; + sha256 = "1max64z9giii61qcwl56rndd7pakaylkaij5zqbbbvjl9vxdr87y"; + libName = "enum_ordinalize"; + dependencies = [ + { + name = "enum-ordinalize-derive"; + packageId = "enum-ordinalize-derive"; + optional = true; + usesDefaultFeatures = false; + } + ]; + features = { + "default" = [ "derive" "traits" ]; + "derive" = [ "dep:enum-ordinalize-derive" ]; + "traits" = [ "enum-ordinalize-derive?/traits" ]; + }; + resolvedDefaultFeatures = [ "derive" ]; + }; + "enum-ordinalize-derive" = rec { + crateName = "enum-ordinalize-derive"; + version = "4.3.1"; + edition = "2021"; + sha256 = "1zy53fabazimwv5cl0366k834ybixzl84lxj9mfavbnlfn532a0d"; + procMacro = true; + libName = "enum_ordinalize_derive"; + dependencies = [ + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "syn"; + packageId = "syn 2.0.82"; + } + ]; + features = { + }; + }; "equivalent" = rec { crateName = "equivalent"; version = "1.0.1"; @@ -1887,7 +1974,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; features = [ "full" ]; } ]; @@ -2990,6 +3077,17 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; + "indoc" = rec { + crateName = "indoc"; + version = "2.0.5"; + edition = "2021"; + sha256 = "1dgjk49rkmx4kjy07k4b90qb5vl89smgb5rcw02n0q0x9ligaj5j"; + procMacro = true; + authors = [ + "David Tolnay " + ]; + + }; "instant" = rec { crateName = "instant"; version = "0.1.13"; @@ -3283,9 +3381,9 @@ rec { }; "kube" = rec { crateName = "kube"; - version = "0.95.0"; + version = "0.96.0"; edition = "2021"; - sha256 = "1y04cqn7ami9rs6csjsf63c4clm5zyvyx3bzbmyaf828hly0c8gs"; + sha256 = "07ws50li6nxja26b0w40k2dqir60k4s5fi2hsvjz6kmxy0yypzzg"; authors = [ "clux " "Natalie Klestrup Röijezon " @@ -3348,15 +3446,16 @@ rec { "socks5" = [ "kube-client/socks5" "client" ]; "unstable-client" = [ "kube-client/unstable-client" "client" ]; "unstable-runtime" = [ "kube-runtime/unstable-runtime" "runtime" ]; + "webpki-roots" = [ "kube-client/webpki-roots" "client" ]; "ws" = [ "kube-client/ws" "kube-core/ws" ]; }; resolvedDefaultFeatures = [ "client" "config" "derive" "jsonpatch" "kube-client" "kube-derive" "kube-runtime" "runtime" "rustls-tls" ]; }; "kube-client" = rec { crateName = "kube-client"; - version = "0.95.0"; + version = "0.96.0"; edition = "2021"; - sha256 = "1dwvvd37psgzcaqv87lppx2yrmdb8wggx9ki1s8132lxbigkbhii"; + sha256 = "1wg0blziqkfyfmmyn6l1fj6wp7qy156sr3g7birj93gzx3n73x4b"; libName = "kube_client"; authors = [ "clux " @@ -3474,7 +3573,6 @@ rec { { name = "secrecy"; packageId = "secrecy"; - features = [ "alloc" "serde" ]; } { name = "serde"; @@ -3590,15 +3688,16 @@ rec { "tower" = [ "dep:tower" ]; "tower-http" = [ "dep:tower-http" ]; "tracing" = [ "dep:tracing" ]; + "webpki-roots" = [ "hyper-rustls/webpki-roots" ]; "ws" = [ "client" "tokio-tungstenite" "rand" "kube-core/ws" "tokio/macros" ]; }; resolvedDefaultFeatures = [ "__non_core" "base64" "bytes" "chrono" "client" "config" "either" "futures" "home" "http-body" "http-body-util" "hyper" "hyper-rustls" "hyper-timeout" "hyper-util" "jsonpatch" "jsonpath-rust" "pem" "rustls" "rustls-pemfile" "rustls-tls" "serde_yaml" "tokio" "tokio-util" "tower" "tower-http" "tracing" ]; }; "kube-core" = rec { crateName = "kube-core"; - version = "0.95.0"; + version = "0.96.0"; edition = "2021"; - sha256 = "0rp6n4k4b1ai6ghxqci72wy67kxrgn6x9rs70ajl9dcx3kchn0zk"; + sha256 = "0xrxzqk7nbbymf7ycm02wshs6ynf3dlrnm2wvix1skdk1g9lc8zl"; libName = "kube_core"; authors = [ "clux " @@ -3673,9 +3772,9 @@ rec { }; "kube-derive" = rec { crateName = "kube-derive"; - version = "0.95.0"; + version = "0.96.0"; edition = "2021"; - sha256 = "09qcgysprgrdhl838y28xfvvyrak0x367s5879vhmmyxisbvx67s"; + sha256 = "1bc23sismxyyncsry902b2i2v0aifpxvgs3fdh9q412yrh24wdpr"; procMacro = true; libName = "kube_derive"; authors = [ @@ -3702,7 +3801,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; features = [ "extra-traits" ]; } ]; @@ -3710,9 +3809,9 @@ rec { }; "kube-runtime" = rec { crateName = "kube-runtime"; - version = "0.95.0"; + version = "0.96.0"; edition = "2021"; - sha256 = "053r458alv2gk2q7vb8nbycw4d1bclsvja7i10j95b21ls5cp5aq"; + sha256 = "0i5xr5i9xf44fwih1pvypr35sq30pcw979m9sbqnb3m9zzvg3yyk"; libName = "kube_runtime"; authors = [ "clux " @@ -3741,8 +3840,10 @@ rec { packageId = "backoff"; } { - name = "derivative"; - packageId = "derivative"; + name = "educe"; + packageId = "educe"; + usesDefaultFeatures = false; + features = [ "Clone" "Debug" "Hash" "PartialEq" ]; } { name = "futures"; @@ -3826,7 +3927,7 @@ rec { } ]; features = { - "unstable-runtime" = [ "unstable-runtime-subscribe" "unstable-runtime-predicates" "unstable-runtime-stream-control" "unstable-runtime-reconcile-on" ]; + "unstable-runtime" = [ "unstable-runtime-subscribe" "unstable-runtime-stream-control" "unstable-runtime-reconcile-on" ]; }; }; "lazy_static" = rec { @@ -3844,9 +3945,9 @@ rec { }; "libc" = rec { crateName = "libc"; - version = "0.2.159"; + version = "0.2.161"; edition = "2015"; - sha256 = "1i9xpia0hn1y8dws7all8rqng6h3lc8ymlgslnljcvm376jrf7an"; + sha256 = "1lc5s3zd0491x9zxrv2kvclai1my1spz950pkkyry4vwh318k54f"; authors = [ "The Rust Project Developers" ]; @@ -4448,7 +4549,7 @@ rec { } { name = "ordered-float"; - packageId = "ordered-float 4.3.0"; + packageId = "ordered-float 4.4.0"; } { name = "percent-encoding"; @@ -4536,11 +4637,11 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; - "ordered-float 4.3.0" = rec { + "ordered-float 4.4.0" = rec { crateName = "ordered-float"; - version = "4.3.0"; + version = "4.4.0"; edition = "2021"; - sha256 = "0dr5d8byvqkiclxjimp5kyh3m9qz4zvwifx6cg0d6w9glzqh3ma4"; + sha256 = "15vbmn4lvd2gjmb1s2hbr4n18plk1pql9md30sapq2r4bswwrrw3"; libName = "ordered_float"; authors = [ "Jonathan Reem " @@ -4558,6 +4659,7 @@ rec { "borsh" = [ "dep:borsh" ]; "bytemuck" = [ "dep:bytemuck" ]; "default" = [ "std" ]; + "derive-visitor" = [ "dep:derive-visitor" ]; "num-cmp" = [ "dep:num-cmp" ]; "proptest" = [ "dep:proptest" ]; "rand" = [ "dep:rand" ]; @@ -4799,7 +4901,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; } ]; features = { @@ -4872,7 +4974,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; usesDefaultFeatures = false; features = [ "parsing" "printing" "clone-impls" "proc-macro" "full" "visit-mut" ]; } @@ -4963,9 +5065,9 @@ rec { }; "proc-macro2" = rec { crateName = "proc-macro2"; - version = "1.0.87"; + version = "1.0.89"; edition = "2021"; - sha256 = "16mifsq1nqzk81qm82aszib44jsd23gpqic5z4kbmzpnvjhdmr5k"; + sha256 = "0vlq56v41dsj69pnk7lil7fxvbfid50jnzdn3xnr31g05mkb0fgi"; libName = "proc_macro2"; authors = [ "David Tolnay " @@ -5435,9 +5537,9 @@ rec { }; "rstest" = rec { crateName = "rstest"; - version = "0.22.0"; + version = "0.23.0"; edition = "2021"; - sha256 = "0dlrn6y4z5xgsvf6ky3lrjwsxpvi13sizlkwnqs1gmmxc873yhkv"; + sha256 = "0d90hr3i2yajzgpzvsh6p2yjzmcb3nm8884xdbb5sswvwmdmhb0a"; authors = [ "Michele d'Amico " ]; @@ -5473,9 +5575,9 @@ rec { }; "rstest_macros" = rec { crateName = "rstest_macros"; - version = "0.22.0"; + version = "0.23.0"; edition = "2021"; - sha256 = "0hiba8l3d20ajkifd3kz5rzzpxsy311ca4c4ll94pxqlglg73qf5"; + sha256 = "0nmdm7a4ysihnh0zz6w6gqrmw205zfp7xqkb2id3858vg20afpl2"; procMacro = true; authors = [ "Michele d'Amico " @@ -5512,7 +5614,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; features = [ "full" "parsing" "extra-traits" "visit" "visit-mut" ]; } { @@ -5562,9 +5664,9 @@ rec { }; "rustls" = rec { crateName = "rustls"; - version = "0.23.14"; + version = "0.23.15"; edition = "2021"; - sha256 = "1a0b2sdvq69vqrz08wvjmlqafzh7pfgzhn9j0n107f9wd529jpa1"; + sha256 = "14vr5pfdvzfcqrmjzh1834a1nyi3kzv7j8s22gb77s64mkbl9fsz"; dependencies = [ { name = "log"; @@ -5901,7 +6003,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; features = [ "extra-traits" ]; } ]; @@ -5921,31 +6023,23 @@ rec { }; "secrecy" = rec { crateName = "secrecy"; - version = "0.8.0"; - edition = "2018"; - sha256 = "07p9h2bpkkg61f1fzzdqqbf74kwv1gg095r1cdmjzzbcl17cblcv"; + version = "0.10.3"; + edition = "2021"; + sha256 = "0nmfsf9qm8921v2jliz08bj8zrryqar4gj3d6irqfc3kaj2az4g8"; authors = [ "Tony Arcieri " ]; dependencies = [ - { - name = "serde"; - packageId = "serde"; - optional = true; - } { name = "zeroize"; packageId = "zeroize"; usesDefaultFeatures = false; + features = [ "alloc" ]; } ]; features = { - "alloc" = [ "zeroize/alloc" ]; - "bytes" = [ "dep:bytes" ]; - "default" = [ "alloc" ]; "serde" = [ "dep:serde" ]; }; - resolvedDefaultFeatures = [ "alloc" "default" "serde" ]; }; "security-framework" = rec { crateName = "security-framework"; @@ -6041,9 +6135,9 @@ rec { }; "serde" = rec { crateName = "serde"; - version = "1.0.210"; + version = "1.0.213"; edition = "2018"; - sha256 = "0flc0z8wgax1k4j5bf2zyq48bgzyv425jkd5w0i6wbh7f8j5kqy8"; + sha256 = "1hcv1q7ziy27c2awc0lnhigjj6rli1863fr0szw6sip2ylzqk9ry"; authors = [ "Erick Tryzelaar " "David Tolnay " @@ -6096,9 +6190,9 @@ rec { }; "serde_derive" = rec { crateName = "serde_derive"; - version = "1.0.210"; + version = "1.0.213"; edition = "2015"; - sha256 = "07yzy4wafk79ps0hmbqmsqh5xjna4pm4q57wc847bb8gl3nh4f94"; + sha256 = "198g92m9c8whvwrnrbxppwdm3pvbq7ddd35agkl5h2y514hav1by"; procMacro = true; authors = [ "Erick Tryzelaar " @@ -6119,7 +6213,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; usesDefaultFeatures = false; features = [ "clone-impls" "derive" "parsing" "printing" "proc-macro" ]; } @@ -6151,7 +6245,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; usesDefaultFeatures = false; features = [ "clone-impls" "derive" "parsing" "printing" ]; } @@ -6160,9 +6254,9 @@ rec { }; "serde_json" = rec { crateName = "serde_json"; - version = "1.0.128"; + version = "1.0.132"; edition = "2021"; - sha256 = "1n43nia50ybpcfmh3gcw4lcc627qsg9nyakzwgkk9pm10xklbxbg"; + sha256 = "00yv8vyn1qiplziswm1vwam4a0xs1rfr162q75njc85kyjpvy9np"; authors = [ "Erick Tryzelaar " "David Tolnay " @@ -6523,7 +6617,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; features = [ "full" ]; } ]; @@ -6592,6 +6686,10 @@ rec { "Stackable GmbH " ]; dependencies = [ + { + name = "indoc"; + packageId = "indoc"; + } { name = "product-config"; packageId = "product-config"; @@ -6618,6 +6716,11 @@ rec { packageId = "strum"; features = [ "derive" ]; } + { + name = "tokio"; + packageId = "tokio"; + features = [ "full" ]; + } { name = "tracing"; packageId = "tracing"; @@ -6668,6 +6771,10 @@ rec { packageId = "futures 0.3.31"; features = [ "compat" ]; } + { + name = "indoc"; + packageId = "indoc"; + } { name = "product-config"; packageId = "product-config"; @@ -6725,13 +6832,13 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.76.0"; + version = "0.80.0"; edition = "2021"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "a7e70f174fb043a1766e0a80de95834cb4f7513d"; - sha256 = "1cyyyn6lizd0wdq79fc9fjnksnzx073ipydxmh7llciq5si5dnq6"; + rev = "6fbe32300b60f95e0baa2ab0ff2daf961b06531c"; + sha256 = "16jrq3wdwz63210jgmqbx3snrr15wxw6l1smqhzv7b7jpq8qvya3"; }; libName = "stackable_operator"; authors = [ @@ -6772,6 +6879,10 @@ rec { name = "futures"; packageId = "futures 0.3.31"; } + { + name = "indexmap"; + packageId = "indexmap"; + } { name = "json-patch"; packageId = "json-patch"; @@ -6836,6 +6947,10 @@ rec { name = "stackable-operator-derive"; packageId = "stackable-operator-derive"; } + { + name = "stackable-shared"; + packageId = "stackable-shared"; + } { name = "strum"; packageId = "strum"; @@ -6880,8 +6995,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech/operator-rs.git"; - rev = "a7e70f174fb043a1766e0a80de95834cb4f7513d"; - sha256 = "1cyyyn6lizd0wdq79fc9fjnksnzx073ipydxmh7llciq5si5dnq6"; + rev = "6fbe32300b60f95e0baa2ab0ff2daf961b06531c"; + sha256 = "16jrq3wdwz63210jgmqbx3snrr15wxw6l1smqhzv7b7jpq8qvya3"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -6903,7 +7018,48 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; + } + ]; + + }; + "stackable-shared" = rec { + crateName = "stackable-shared"; + version = "0.0.1"; + edition = "2021"; + workspace_member = null; + src = pkgs.fetchgit { + url = "https://github.com/stackabletech/operator-rs.git"; + rev = "6fbe32300b60f95e0baa2ab0ff2daf961b06531c"; + sha256 = "16jrq3wdwz63210jgmqbx3snrr15wxw6l1smqhzv7b7jpq8qvya3"; + }; + libName = "stackable_shared"; + authors = [ + "Stackable GmbH " + ]; + dependencies = [ + { + name = "kube"; + packageId = "kube"; + usesDefaultFeatures = false; + features = [ "client" "jsonpatch" "runtime" "derive" "rustls-tls" ]; + } + { + name = "semver"; + packageId = "semver"; + } + { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + { + name = "serde_yaml"; + packageId = "serde_yaml"; + } + { + name = "snafu"; + packageId = "snafu 0.8.5"; } ]; @@ -6976,7 +7132,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; features = [ "parsing" "extra-traits" ]; } ]; @@ -7029,11 +7185,11 @@ rec { }; resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "full" "parsing" "printing" "proc-macro" "quote" "visit" ]; }; - "syn 2.0.79" = rec { + "syn 2.0.82" = rec { crateName = "syn"; - version = "2.0.79"; + version = "2.0.82"; edition = "2021"; - sha256 = "147mk4sgigmvsb9l8qzj199ygf0fgb0bphwdsghn8205pz82q4w9"; + sha256 = "08g0bgizm4j2gwl05r3yjm3gxvx8a9dvkvd84fa03z4aga1hym43"; authors = [ "David Tolnay " ]; @@ -7062,11 +7218,24 @@ rec { }; resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "extra-traits" "full" "parsing" "printing" "proc-macro" "visit" "visit-mut" ]; }; + "sync_wrapper" = rec { + crateName = "sync_wrapper"; + version = "0.1.2"; + edition = "2018"; + sha256 = "0q01lyj0gr9a93n10nxsn8lwbzq97jqd6b768x17c8f7v7gccir0"; + authors = [ + "Actyx AG " + ]; + features = { + "futures" = [ "futures-core" ]; + "futures-core" = [ "dep:futures-core" ]; + }; + }; "thiserror" = rec { crateName = "thiserror"; - version = "1.0.64"; + version = "1.0.65"; edition = "2021"; - sha256 = "114s8lmssxl0c2480s671am88vzlasbaikxbvfv8pyqrq6mzh2nm"; + sha256 = "1mdkawq9l9p02zvq7y4py739rjk9wk2ha27mbsb3i6sdb7csn4ax"; authors = [ "David Tolnay " ]; @@ -7080,9 +7249,9 @@ rec { }; "thiserror-impl" = rec { crateName = "thiserror-impl"; - version = "1.0.64"; + version = "1.0.65"; edition = "2021"; - sha256 = "1hvzmjx9iamln854l74qyhs0jl2pg3hhqzpqm9p8gszmf9v4x408"; + sha256 = "00l6gyrx6qlm1d7if3dcfl2sl0mg8k21caknkpk7glnb481pfwdf"; procMacro = true; libName = "thiserror_impl"; authors = [ @@ -7099,7 +7268,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; } ]; @@ -7338,9 +7507,9 @@ rec { }; "tokio" = rec { crateName = "tokio"; - version = "1.40.0"; + version = "1.41.0"; edition = "2021"; - sha256 = "166rllhfkyqp0fs7sxn6crv74iizi4wzd3cvxkcpmlk52qip1c72"; + sha256 = "1fwb4nm630hmy9cyl2ar6wxqckgvsakwhg1rhjza4is3a09k8pql"; authors = [ "Tokio Contributors " ]; @@ -7461,7 +7630,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; features = [ "full" ]; } ]; @@ -7659,9 +7828,9 @@ rec { }; "tower" = rec { crateName = "tower"; - version = "0.4.13"; + version = "0.5.1"; edition = "2018"; - sha256 = "073wncyqav4sak1p755hf6vl66njgfc1z1g1di9rxx3cvvh9pymq"; + sha256 = "0kvbp97bhb4sk24vhihcz74ngn0i4ygxqikmxndgng3w926r6wr8"; authors = [ "Tower Maintainers " ]; @@ -7679,13 +7848,13 @@ rec { features = [ "alloc" ]; } { - name = "pin-project"; - packageId = "pin-project"; + name = "pin-project-lite"; + packageId = "pin-project-lite"; optional = true; } { - name = "pin-project-lite"; - packageId = "pin-project-lite"; + name = "sync_wrapper"; + packageId = "sync_wrapper"; optional = true; } { @@ -7726,12 +7895,17 @@ rec { packageId = "tokio"; features = [ "macros" "sync" "test-util" "rt-multi-thread" ]; } + { + name = "tracing"; + packageId = "tracing"; + usesDefaultFeatures = false; + features = [ "std" ]; + } ]; features = { "__common" = [ "futures-core" "pin-project-lite" ]; - "balance" = [ "discover" "load" "ready-cache" "make" "rand" "slab" ]; + "balance" = [ "discover" "load" "ready-cache" "make" "slab" "util" ]; "buffer" = [ "__common" "tokio/sync" "tokio/rt" "tokio-util" "tracing" ]; - "default" = [ "log" ]; "discover" = [ "__common" ]; "filter" = [ "__common" "futures-util" ]; "full" = [ "balance" "buffer" "discover" "filter" "hedge" "limit" "load" "load-shed" "make" "ready-cache" "reconnect" "retry" "spawn-ready" "steer" "timeout" "util" ]; @@ -7745,28 +7919,27 @@ rec { "load-shed" = [ "__common" ]; "log" = [ "tracing/log" ]; "make" = [ "futures-util" "pin-project-lite" "tokio/io-std" ]; - "pin-project" = [ "dep:pin-project" ]; "pin-project-lite" = [ "dep:pin-project-lite" ]; - "rand" = [ "dep:rand" ]; "ready-cache" = [ "futures-core" "futures-util" "indexmap" "tokio/sync" "tracing" "pin-project-lite" ]; "reconnect" = [ "make" "tokio/io-std" "tracing" ]; - "retry" = [ "__common" "tokio/time" ]; + "retry" = [ "__common" "tokio/time" "util" ]; "slab" = [ "dep:slab" ]; "spawn-ready" = [ "__common" "futures-util" "tokio/sync" "tokio/rt" "util" "tracing" ]; + "sync_wrapper" = [ "dep:sync_wrapper" ]; "timeout" = [ "pin-project-lite" "tokio/time" ]; "tokio" = [ "dep:tokio" ]; "tokio-stream" = [ "dep:tokio-stream" ]; "tokio-util" = [ "dep:tokio-util" ]; "tracing" = [ "dep:tracing" ]; - "util" = [ "__common" "futures-util" "pin-project" ]; + "util" = [ "__common" "futures-util" "pin-project-lite" "sync_wrapper" ]; }; - resolvedDefaultFeatures = [ "__common" "buffer" "default" "filter" "futures-core" "futures-util" "log" "pin-project" "pin-project-lite" "tokio" "tokio-util" "tracing" "util" ]; + resolvedDefaultFeatures = [ "__common" "buffer" "filter" "futures-core" "futures-util" "pin-project-lite" "sync_wrapper" "tokio" "tokio-util" "tracing" "util" ]; }; "tower-http" = rec { crateName = "tower-http"; - version = "0.5.2"; + version = "0.6.1"; edition = "2018"; - sha256 = "1xakj3x0anp55gjqibiwvzma5iz0w9pcjsr7qk97sx4qm4sd970y"; + sha256 = "15yb8rh970ll3yrd3lydshysi0x89bnxbqqrypqcbj5vnq51adw4"; libName = "tower_http"; authors = [ "Tower Maintainers " @@ -7774,7 +7947,7 @@ rec { dependencies = [ { name = "base64"; - packageId = "base64 0.21.7"; + packageId = "base64 0.22.1"; optional = true; } { @@ -7792,10 +7965,7 @@ rec { { name = "http-body"; packageId = "http-body"; - } - { - name = "http-body-util"; - packageId = "http-body-util"; + optional = true; } { name = "mime"; @@ -7827,39 +7997,44 @@ rec { name = "bytes"; packageId = "bytes"; } + { + name = "http-body"; + packageId = "http-body"; + } ]; features = { "async-compression" = [ "dep:async-compression" ]; "auth" = [ "base64" "validate-request" ]; "base64" = [ "dep:base64" ]; - "catch-panic" = [ "tracing" "futures-util/std" ]; - "compression-br" = [ "async-compression/brotli" "futures-core" "tokio-util" "tokio" ]; - "compression-deflate" = [ "async-compression/zlib" "futures-core" "tokio-util" "tokio" ]; + "catch-panic" = [ "tracing" "futures-util/std" "dep:http-body" "dep:http-body-util" ]; + "compression-br" = [ "async-compression/brotli" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; + "compression-deflate" = [ "async-compression/zlib" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; "compression-full" = [ "compression-br" "compression-deflate" "compression-gzip" "compression-zstd" ]; - "compression-gzip" = [ "async-compression/gzip" "futures-core" "tokio-util" "tokio" ]; - "compression-zstd" = [ "async-compression/zstd" "futures-core" "tokio-util" "tokio" ]; - "decompression-br" = [ "async-compression/brotli" "futures-core" "tokio-util" "tokio" ]; - "decompression-deflate" = [ "async-compression/zlib" "futures-core" "tokio-util" "tokio" ]; + "compression-gzip" = [ "async-compression/gzip" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; + "compression-zstd" = [ "async-compression/zstd" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; + "decompression-br" = [ "async-compression/brotli" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; + "decompression-deflate" = [ "async-compression/zlib" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; "decompression-full" = [ "decompression-br" "decompression-deflate" "decompression-gzip" "decompression-zstd" ]; - "decompression-gzip" = [ "async-compression/gzip" "futures-core" "tokio-util" "tokio" ]; - "decompression-zstd" = [ "async-compression/zstd" "futures-core" "tokio-util" "tokio" ]; - "follow-redirect" = [ "futures-util" "iri-string" "tower/util" ]; - "fs" = [ "futures-util" "tokio/fs" "tokio-util/io" "tokio/io-util" "dep:http-range-header" "mime_guess" "mime" "percent-encoding" "httpdate" "set-status" "futures-util/alloc" "tracing" ]; + "decompression-gzip" = [ "async-compression/gzip" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; + "decompression-zstd" = [ "async-compression/zstd" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; + "follow-redirect" = [ "futures-util" "dep:http-body" "iri-string" "tower/util" ]; + "fs" = [ "futures-util" "dep:http-body" "dep:http-body-util" "tokio/fs" "tokio-util/io" "tokio/io-util" "dep:http-range-header" "mime_guess" "mime" "percent-encoding" "httpdate" "set-status" "futures-util/alloc" "tracing" ]; "full" = [ "add-extension" "auth" "catch-panic" "compression-full" "cors" "decompression-full" "follow-redirect" "fs" "limit" "map-request-body" "map-response-body" "metrics" "normalize-path" "propagate-header" "redirect" "request-id" "sensitive-headers" "set-header" "set-status" "timeout" "trace" "util" "validate-request" ]; "futures-core" = [ "dep:futures-core" ]; "futures-util" = [ "dep:futures-util" ]; "httpdate" = [ "dep:httpdate" ]; "iri-string" = [ "dep:iri-string" ]; - "metrics" = [ "tokio/time" ]; + "limit" = [ "dep:http-body" "dep:http-body-util" ]; + "metrics" = [ "dep:http-body" "tokio/time" ]; "mime" = [ "dep:mime" ]; "mime_guess" = [ "dep:mime_guess" ]; "percent-encoding" = [ "dep:percent-encoding" ]; "request-id" = [ "uuid" ]; - "timeout" = [ "tokio/time" ]; + "timeout" = [ "dep:http-body" "tokio/time" ]; "tokio" = [ "dep:tokio" ]; "tokio-util" = [ "dep:tokio-util" ]; "tower" = [ "dep:tower" ]; - "trace" = [ "tracing" ]; + "trace" = [ "dep:http-body" "tracing" ]; "tracing" = [ "dep:tracing" ]; "util" = [ "tower" ]; "uuid" = [ "dep:uuid" ]; @@ -7995,7 +8170,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; usesDefaultFeatures = false; features = [ "full" "parsing" "printing" "visit-mut" "clone-impls" "extra-traits" "proc-macro" ]; } @@ -8634,7 +8809,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; features = [ "full" ]; } { @@ -8693,7 +8868,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; features = [ "visit" "visit-mut" "full" ]; } { @@ -9549,7 +9724,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.79"; + packageId = "syn 2.0.82"; } ]; diff --git a/crate-hashes.json b/crate-hashes.json index 4ac79c7f..562fb18b 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,5 +1,6 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.76.0#stackable-operator-derive@0.3.1": "1cyyyn6lizd0wdq79fc9fjnksnzx073ipydxmh7llciq5si5dnq6", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.76.0#stackable-operator@0.76.0": "1cyyyn6lizd0wdq79fc9fjnksnzx073ipydxmh7llciq5si5dnq6", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.80.0#stackable-operator-derive@0.3.1": "16jrq3wdwz63210jgmqbx3snrr15wxw6l1smqhzv7b7jpq8qvya3", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.80.0#stackable-operator@0.80.0": "16jrq3wdwz63210jgmqbx3snrr15wxw6l1smqhzv7b7jpq8qvya3", + "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.80.0#stackable-shared@0.0.1": "16jrq3wdwz63210jgmqbx3snrr15wxw6l1smqhzv7b7jpq8qvya3", "git+https://github.com/stackabletech/product-config.git?tag=0.7.0#product-config@0.7.0": "0gjsm80g6r75pm3824dcyiz4ysq1ka4c1if6k1mjm9cnd5ym0gny" } \ No newline at end of file diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index 32f98214..afa8785e 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -452,13 +452,27 @@ spec: properties: authentication: default: [] - description: The Airflow [authentication](https://docs.stackable.tech/home/nightly/airflow/usage-guide/security.html) settings. Currently the underlying Flask App Builder only supports one authentication mechanism at a time. This means the operator will error out if multiple references to an AuthenticationClass are provided. items: properties: authenticationClass: - description: Name of the [AuthenticationClass](https://docs.stackable.tech/home/nightly/concepts/authentication.html#authenticationclass) used to authenticate the users. At the moment only LDAP is supported. If not specified the default authentication (AUTH_DB) will be used. - nullable: true + description: Name of the [AuthenticationClass](https://docs.stackable.tech/home/nightly/concepts/authentication) used to authenticate users. type: string + oidc: + description: This field contains OIDC-specific configuration. It is only required in case OIDC is used. + nullable: true + properties: + clientCredentialsSecret: + description: A reference to the OIDC client credentials secret. The secret contains the client id and secret. + type: string + extraScopes: + default: [] + description: An optional list of extra scopes which get merged with the scopes defined in the [`AuthenticationClass`]. + items: + type: string + type: array + required: + - clientCredentialsSecret + type: object syncRolesAt: default: Registration description: If we should replace ALL the user's roles each login, or only on registration. Gets mapped to `AUTH_ROLES_SYNC_AT_LOGIN` @@ -474,6 +488,8 @@ spec: default: Public description: This role will be given in addition to any AUTH_ROLES_MAPPING. Gets mapped to `AUTH_USER_REGISTRATION_ROLE` type: string + required: + - authenticationClass type: object type: array credentialsSecret: From 963518ec35cb1b43e35ca6b47a57df5fc92bff54 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Tue, 29 Oct 2024 10:39:08 +0100 Subject: [PATCH 16/25] making pre-commit happy --- .../airflow/pages/usage-guide/security.adoc | 2 +- tests/templates/kuttl/oidc/login.py | 36 ++++++++++--------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/docs/modules/airflow/pages/usage-guide/security.adoc b/docs/modules/airflow/pages/usage-guide/security.adoc index 13b9afd2..80091fc4 100644 --- a/docs/modules/airflow/pages/usage-guide/security.adoc +++ b/docs/modules/airflow/pages/usage-guide/security.adoc @@ -13,7 +13,7 @@ Users need to authenticate themselves before using Airflow, and there are severa [IMPORTANT] .Multiple authentication methods ==== -Only one authentication method is supported at a time, and in case of LDAP, only one authentication class is allowed. +Only one authentication method is supported at a time, and in case of LDAP, only one authentication class is allowed. This means, it is not possible to configure both LDAP and OIDC authentication methods at the same time, but *it is* possible to configure multiple OIDC classes *or* one LDAP authentication class. ==== diff --git a/tests/templates/kuttl/oidc/login.py b/tests/templates/kuttl/oidc/login.py index 4fa243d9..e4479566 100644 --- a/tests/templates/kuttl/oidc/login.py +++ b/tests/templates/kuttl/oidc/login.py @@ -6,9 +6,8 @@ from bs4 import BeautifulSoup logging.basicConfig( - level='DEBUG', - format="%(asctime)s %(levelname)s: %(message)s", - stream=sys.stdout) + level='DEBUG', format="%(asctime)s %(levelname)s: %(message)s", stream=sys.stdout +) session = requests.Session() @@ -16,35 +15,40 @@ login_page = session.get("http://airflow-webserver:8080/login/keycloak?next=") assert login_page.ok, "Redirection from Airflow to Keycloak failed" -assert login_page.url.startswith("https://keycloak1.$NAMESPACE.svc.cluster.local:8443/realms/test1/protocol/openid-connect/auth?response_type=code&client_id=airflow1"), \ - "Redirection to the Keycloak login page expected" +assert login_page.url.startswith( + "https://keycloak1.$NAMESPACE.svc.cluster.local:8443/realms/test1/protocol/openid-connect/auth?response_type=code&client_id=airflow1" + ), "Redirection to the Keycloak login page expected" # Enter username and password into the Keycloak login page and click on "Sign In" login_page_html = BeautifulSoup(login_page.text, 'html.parser') authenticate_url = login_page_html.form['action'] -welcome_page = session.post(authenticate_url, data={ 'username': "jane.doe", 'password': "T8mn72D9" }) +welcome_page = session.post( + authenticate_url, data={ 'username': "jane.doe", 'password': "T8mn72D9" } +) assert welcome_page.ok, "Login failed" -assert welcome_page.url == "http://airflow-webserver:8080/home", \ - "Redirection to the Airflow home page expected" +assert ( + welcome_page.url == "http://airflow-webserver:8080/home" + ), "Redirection to the Airflow home page expected" # Open the user information page in Airflow userinfo_page = session.get("http://airflow-webserver:8080/users/userinfo/") assert userinfo_page.ok, "Retrieving user information failed" -assert userinfo_page.url == "http://airflow-webserver:8080/users/userinfo/", \ - "Redirection to the Airflow user info page expected" +assert ( + userinfo_page.url == "http://airflow-webserver:8080/users/userinfo/" +), "Redirection to the Airflow user info page expected" # Expect the user data provided by Keycloak in Airflow -userinfo_page_html = BeautifulSoup(userinfo_page.text, 'html.parser') -table_rows = userinfo_page_html.find_all('tr') -user_data = {tr.find('th').text:tr.find('td').text for tr in table_rows} +userinfo_page_html = BeautifulSoup(userinfo_page.text, "html.parser") +table_rows = userinfo_page_html.find_all("tr") +user_data = {tr.find("th").text:tr.find("td").text for tr in table_rows} -assert user_data['First Name'] == "Jane", \ +assert user_data["First Name"] == "Jane", \ "The first name of the user in Airflow should match the one provided by Keycloak" -assert user_data['Last Name'] == "Doe", \ +assert user_data["Last Name"] == "Doe", \ "The last name of the user in Airflow should match the one provided by Keycloak" -assert user_data['Email'] == "jane.doe@stackable.tech", \ +assert user_data["Email"] == "jane.doe@stackable.tech", \ "The email of the user in Airflow should match the one provided by Keycloak" # TODO Use different OIDC providers (currently only Keycloak is From f5e55f7b2c7c50d457fde798f65fc6d22f17f3eb Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Tue, 29 Oct 2024 10:41:35 +0100 Subject: [PATCH 17/25] fixing typo --- docs/modules/airflow/pages/usage-guide/security.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/airflow/pages/usage-guide/security.adoc b/docs/modules/airflow/pages/usage-guide/security.adoc index 80091fc4..e09a5be2 100644 --- a/docs/modules/airflow/pages/usage-guide/security.adoc +++ b/docs/modules/airflow/pages/usage-guide/security.adoc @@ -26,7 +26,7 @@ image::airflow_security.png[Airflow Security menu] === LDAP -Airflow supports xref:concepts:authentication.adoc[user authentication] againts a single LDAP server. +Airflow supports xref:concepts:authentication.adoc[user authentication] against a single LDAP server. Set up an AuthenticationClass for the LDAP server and reference it in the Airflow Stacklet resource as shown: [source,yaml] From fcda6eaeb8644aeb43f6e7a3ae6a099d20d31881 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Tue, 29 Oct 2024 11:07:10 +0100 Subject: [PATCH 18/25] again pre-commit --- tests/templates/kuttl/oidc/login.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/templates/kuttl/oidc/login.py b/tests/templates/kuttl/oidc/login.py index e4479566..768ce778 100644 --- a/tests/templates/kuttl/oidc/login.py +++ b/tests/templates/kuttl/oidc/login.py @@ -6,7 +6,7 @@ from bs4 import BeautifulSoup logging.basicConfig( - level='DEBUG', format="%(asctime)s %(levelname)s: %(message)s", stream=sys.stdout + level="DEBUG", format="%(asctime)s %(levelname)s: %(message)s", stream=sys.stdout ) session = requests.Session() @@ -17,19 +17,19 @@ assert login_page.ok, "Redirection from Airflow to Keycloak failed" assert login_page.url.startswith( "https://keycloak1.$NAMESPACE.svc.cluster.local:8443/realms/test1/protocol/openid-connect/auth?response_type=code&client_id=airflow1" - ), "Redirection to the Keycloak login page expected" +), "Redirection to the Keycloak login page expected" # Enter username and password into the Keycloak login page and click on "Sign In" login_page_html = BeautifulSoup(login_page.text, 'html.parser') authenticate_url = login_page_html.form['action'] welcome_page = session.post( - authenticate_url, data={ 'username': "jane.doe", 'password': "T8mn72D9" } + authenticate_url, data={ "username": "jane.doe", 'password': "T8mn72D9" } ) assert welcome_page.ok, "Login failed" assert ( welcome_page.url == "http://airflow-webserver:8080/home" - ), "Redirection to the Airflow home page expected" +), "Redirection to the Airflow home page expected" # Open the user information page in Airflow userinfo_page = session.get("http://airflow-webserver:8080/users/userinfo/") @@ -44,12 +44,15 @@ table_rows = userinfo_page_html.find_all("tr") user_data = {tr.find("th").text:tr.find("td").text for tr in table_rows} -assert user_data["First Name"] == "Jane", \ - "The first name of the user in Airflow should match the one provided by Keycloak" -assert user_data["Last Name"] == "Doe", \ - "The last name of the user in Airflow should match the one provided by Keycloak" -assert user_data["Email"] == "jane.doe@stackable.tech", \ - "The email of the user in Airflow should match the one provided by Keycloak" +assert ( + user_data["First Name"] == "Jane" +), "The first name of the user in Airflow should match the one provided by Keycloak" +assert ( + user_data["Last Name"] == "Doe" +), "The last name of the user in Airflow should match the one provided by Keycloak" +assert ( + user_data["Email"] == "jane.doe@stackable.tech" +), "The email of the user in Airflow should match the one provided by Keycloak" # TODO Use different OIDC providers (currently only Keycloak is # supported) From 39d4c23d07eff641ea0944e9223571012d4249ce Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Tue, 29 Oct 2024 11:15:54 +0100 Subject: [PATCH 19/25] pre-commit nr3 --- tests/templates/kuttl/oidc/login.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/templates/kuttl/oidc/login.py b/tests/templates/kuttl/oidc/login.py index 768ce778..771c8eaf 100644 --- a/tests/templates/kuttl/oidc/login.py +++ b/tests/templates/kuttl/oidc/login.py @@ -20,10 +20,10 @@ ), "Redirection to the Keycloak login page expected" # Enter username and password into the Keycloak login page and click on "Sign In" -login_page_html = BeautifulSoup(login_page.text, 'html.parser') -authenticate_url = login_page_html.form['action'] +login_page_html = BeautifulSoup(login_page.text, "html.parser") +authenticate_url = login_page_html.form["action"] welcome_page = session.post( - authenticate_url, data={ "username": "jane.doe", 'password': "T8mn72D9" } + authenticate_url, data={"username": "jane.doe", 'password': "T8mn72D9" } ) assert welcome_page.ok, "Login failed" @@ -42,7 +42,7 @@ # Expect the user data provided by Keycloak in Airflow userinfo_page_html = BeautifulSoup(userinfo_page.text, "html.parser") table_rows = userinfo_page_html.find_all("tr") -user_data = {tr.find("th").text:tr.find("td").text for tr in table_rows} +user_data = {tr.find("th").text: tr.find("td").text for tr in table_rows} assert ( user_data["First Name"] == "Jane" From 2205779e5e0eb3757a48bcc58d615d5348a7f0d3 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Tue, 29 Oct 2024 11:46:02 +0100 Subject: [PATCH 20/25] pre-commit nr4 --- tests/templates/kuttl/oidc/login.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/oidc/login.py b/tests/templates/kuttl/oidc/login.py index 771c8eaf..5ab93efc 100644 --- a/tests/templates/kuttl/oidc/login.py +++ b/tests/templates/kuttl/oidc/login.py @@ -23,7 +23,7 @@ login_page_html = BeautifulSoup(login_page.text, "html.parser") authenticate_url = login_page_html.form["action"] welcome_page = session.post( - authenticate_url, data={"username": "jane.doe", 'password': "T8mn72D9" } + authenticate_url, data={"username": "jane.doe", "password": "T8mn72D9" } ) assert welcome_page.ok, "Login failed" From 52de18a9d919587fd139697f1d809c6f82ebcadd Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Tue, 29 Oct 2024 11:51:14 +0100 Subject: [PATCH 21/25] pre-commit nr5 --- tests/templates/kuttl/oidc/login.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/oidc/login.py b/tests/templates/kuttl/oidc/login.py index 5ab93efc..ba037ba0 100644 --- a/tests/templates/kuttl/oidc/login.py +++ b/tests/templates/kuttl/oidc/login.py @@ -23,7 +23,7 @@ login_page_html = BeautifulSoup(login_page.text, "html.parser") authenticate_url = login_page_html.form["action"] welcome_page = session.post( - authenticate_url, data={"username": "jane.doe", "password": "T8mn72D9" } + authenticate_url, data={"username": "jane.doe", "password": "T8mn72D9"} ) assert welcome_page.ok, "Login failed" From 9d7e2e71b25c8707d207746146af9daf0375a012 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Tue, 29 Oct 2024 18:28:42 +0100 Subject: [PATCH 22/25] Env var set by operator rather then envoverrides --- rust/operator-binary/src/env_vars.rs | 8 ++++++++ tests/templates/kuttl/oidc/install-airflow.yaml.j2 | 2 -- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/rust/operator-binary/src/env_vars.rs b/rust/operator-binary/src/env_vars.rs index e1e77a5b..5d581090 100644 --- a/rust/operator-binary/src/env_vars.rs +++ b/rust/operator-binary/src/env_vars.rs @@ -200,6 +200,14 @@ pub fn build_airflow_statefulset_envs( AirflowRole::Webserver => { let auth_vars = authentication_env_vars(auth_config); env.extend(auth_vars.into_iter().map(|var| (var.name.to_owned(), var))); + env.insert( + "REQUESTS_CA_BUNDLE".into(), + EnvVar { + name: "REQUESTS_CA_BUNDLE".to_string(), + value: Some("/stackable/secrets/tls/ca.crt".to_string()), + ..Default::default() + }, + ); } _ => {} } diff --git a/tests/templates/kuttl/oidc/install-airflow.yaml.j2 b/tests/templates/kuttl/oidc/install-airflow.yaml.j2 index 868b83ab..49e46e46 100644 --- a/tests/templates/kuttl/oidc/install-airflow.yaml.j2 +++ b/tests/templates/kuttl/oidc/install-airflow.yaml.j2 @@ -61,8 +61,6 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} - envOverrides: - REQUESTS_CA_BUNDLE: /stackable/secrets/tls/ca.crt roleGroups: default: replicas: 1 From 908f1114fbf4f062dab266ffe620eb2f011351a0 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 30 Oct 2024 10:21:36 +0100 Subject: [PATCH 23/25] Removing TODO, sort toml alphabetically --- Cargo.toml | 2 +- rust/crd/Cargo.toml | 6 +++--- rust/crd/src/lib.rs | 3 +-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e76c5e35..38123e38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ built = { version = "0.7", features = ["chrono", "git2"] } clap = "4.5" fnv = "1.0" futures = { version = "0.3", features = ["compat"] } +indoc = "2.0" product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" } rstest = "0.23" semver = "1.0" @@ -26,7 +27,6 @@ stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", strum = { version = "0.26", features = ["derive"] } tokio = { version = "1.40", features = ["full"] } tracing = "0.1" -indoc = "2.0" # [patch."https://github.com/stackabletech/operator-rs.git"] # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml index c3c7f31d..42cdff8c 100644 --- a/rust/crd/Cargo.toml +++ b/rust/crd/Cargo.toml @@ -9,15 +9,15 @@ repository.workspace = true publish = false [dependencies] +indoc.workspace = true +product-config.workspace = true serde.workspace = true serde_json.workspace = true snafu.workspace = true stackable-operator.workspace = true -product-config.workspace = true strum.workspace = true -tracing.workspace = true -indoc.workspace = true tokio.workspace = true +tracing.workspace = true [dev-dependencies] serde_yaml.workspace = true diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 40951267..d51e4e6a 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -86,7 +86,6 @@ pub enum Error { NoRoleForExecutorFailure, } -// TODO: Doku! airflow #[derive(Display, EnumIter, EnumString)] #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] pub enum AirflowConfigOptions { @@ -111,7 +110,7 @@ pub enum AirflowConfigOptions { AuthLdapTlsCacertfile, AuthLdapAllowSelfSigned, } -// TODO: Doku! airflow + impl FlaskAppConfigOptions for AirflowConfigOptions { fn python_type(&self) -> PythonType { match self { From 467c8c69da6778583bfae090a44a80bd0e41b0b2 Mon Sep 17 00:00:00 2001 From: Maximilian Wittich <56642549+Maleware@users.noreply.github.com> Date: Thu, 31 Oct 2024 14:10:39 +0100 Subject: [PATCH 24/25] Andrew comments Co-authored-by: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> --- tests/templates/kuttl/oidc/login.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/oidc/login.py b/tests/templates/kuttl/oidc/login.py index ba037ba0..e0e22a70 100644 --- a/tests/templates/kuttl/oidc/login.py +++ b/tests/templates/kuttl/oidc/login.py @@ -54,7 +54,7 @@ user_data["Email"] == "jane.doe@stackable.tech" ), "The email of the user in Airflow should match the one provided by Keycloak" -# TODO Use different OIDC providers (currently only Keycloak is +# Later this can be extended to use different OIDC providers (currently only Keycloak is # supported) # # It would be beneficial if the second OAuth provider keycloak2 could From f28c3827abcd6b676268999b0f469d046907e922 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 31 Oct 2024 14:16:05 +0100 Subject: [PATCH 25/25] Specify every arm rather then just matching all --- rust/crd/src/authentication.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rust/crd/src/authentication.rs b/rust/crd/src/authentication.rs index d6743800..d769f163 100644 --- a/rust/crd/src/authentication.rs +++ b/rust/crd/src/authentication.rs @@ -214,9 +214,11 @@ impl AirflowClientAuthenticationDetailsResolved { ); } resolved_auth_classes.push(resolved_auth_class); + //`&Static(_)`, `&Tls(_)` and `&Kerberos(_)` not covered } - - _ => { + AuthenticationClassProvider::Kerberos(_) + | AuthenticationClassProvider::Static(_) + | AuthenticationClassProvider::Tls(_) => { return Err(Error::AuthenticationProviderNotSupported { auth_class_name: auth_class_name.to_owned(), provider: auth_class.spec.provider.to_string(),