From beb49282e9d4260790fad74335af00672a7bd97b Mon Sep 17 00:00:00 2001 From: Techassi Date: Tue, 12 Aug 2025 15:46:07 +0200 Subject: [PATCH 01/51] feat: Add boil tool --- Cargo.lock | 319 ++++++++++++++++++--- Cargo.toml | 16 +- boil.toml | 5 + rust/boil/Cargo.toml | 19 ++ rust/boil/README.md | 31 ++ rust/boil/src/build/bakefile.rs | 478 +++++++++++++++++++++++++++++++ rust/boil/src/build/cli.rs | 122 ++++++++ rust/boil/src/build/docker.rs | 156 ++++++++++ rust/boil/src/build/image.rs | 163 +++++++++++ rust/boil/src/build/mod.rs | 105 +++++++ rust/boil/src/build/platform.rs | 67 +++++ rust/boil/src/cli.rs | 89 ++++++ rust/boil/src/completions/mod.rs | 10 + rust/boil/src/config.rs | 26 ++ rust/boil/src/main.rs | 94 ++++++ rust/boil/src/show/images.rs | 27 ++ rust/boil/src/show/mod.rs | 1 + rust/boil/src/utils.rs | 23 ++ 18 files changed, 1708 insertions(+), 43 deletions(-) create mode 100644 boil.toml create mode 100644 rust/boil/Cargo.toml create mode 100644 rust/boil/README.md create mode 100644 rust/boil/src/build/bakefile.rs create mode 100644 rust/boil/src/build/cli.rs create mode 100644 rust/boil/src/build/docker.rs create mode 100644 rust/boil/src/build/image.rs create mode 100644 rust/boil/src/build/mod.rs create mode 100644 rust/boil/src/build/platform.rs create mode 100644 rust/boil/src/cli.rs create mode 100644 rust/boil/src/completions/mod.rs create mode 100644 rust/boil/src/config.rs create mode 100644 rust/boil/src/main.rs create mode 100644 rust/boil/src/show/images.rs create mode 100644 rust/boil/src/show/mod.rs create mode 100644 rust/boil/src/utils.rs diff --git a/Cargo.lock b/Cargo.lock index 2b20caacd..5181b2141 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.3" @@ -47,7 +62,7 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -58,7 +73,7 @@ checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" dependencies = [ "anstyle", "once_cell", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -67,18 +82,58 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "backtrace" +version = "0.3.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets", +] + [[package]] name = "bitflags" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" +[[package]] +name = "boil" +version = "0.0.1" +dependencies = [ + "clap", + "clap_complete", + "git2", + "glob", + "semver", + "serde", + "serde_json", + "snafu", + "strum", + "time", + "tokio", + "toml", + "url", +] + [[package]] name = "bumpalo" version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + [[package]] name = "cc" version = "1.2.11" @@ -98,9 +153,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.5.27" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "769b0145982b4b48713e01ec42d61614425f27b7058bda7180a3a41f30104796" +checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9" dependencies = [ "clap_builder", "clap_derive", @@ -108,9 +163,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.27" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7" +checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d" dependencies = [ "anstream", "anstyle", @@ -118,11 +173,20 @@ dependencies = [ "strsim", ] +[[package]] +name = "clap_complete" +version = "4.5.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5abde44486daf70c5be8b8f8f1b66c49f86236edf6fa2abadb4d961c4c6229a" +dependencies = [ + "clap", +] + [[package]] name = "clap_derive" -version = "4.5.24" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" +checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" dependencies = [ "heck", "proc-macro2", @@ -152,14 +216,14 @@ dependencies = [ "libc", "once_cell", "unicode-width 0.2.0", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] name = "deranged" -version = "0.3.11" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ "powerfmt", ] @@ -194,7 +258,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -220,10 +284,16 @@ checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.13.3+wasi-0.2.2", "windows-targets", ] +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + [[package]] name = "git2" version = "0.20.1" @@ -239,6 +309,12 @@ dependencies = [ "url", ] +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + [[package]] name = "hashbrown" version = "0.15.2" @@ -414,6 +490,17 @@ dependencies = [ "web-time", ] +[[package]] +name = "io-uring" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -530,6 +617,26 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -552,6 +659,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.20.2" @@ -690,6 +806,12 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "rustc-demangle" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" + [[package]] name = "rustix" version = "0.38.44" @@ -700,7 +822,28 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "semver" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" +dependencies = [ + "serde", ] [[package]] @@ -723,11 +866,23 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + [[package]] name = "serde_spanned" -version = "0.6.8" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +checksum = "40734c41988f7306bb04f0ecf60ec0f3f1caa34290e4e8ea471dcd3346483b83" dependencies = [ "serde", ] @@ -747,6 +902,21 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook-registry" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" + [[package]] name = "smallvec" version = "1.13.2" @@ -786,6 +956,28 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + [[package]] name = "syn" version = "2.0.96" @@ -819,7 +1011,7 @@ dependencies = [ "getrandom", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -834,11 +1026,12 @@ dependencies = [ [[package]] name = "time" -version = "0.3.37" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" dependencies = [ "deranged", + "itoa", "num-conv", "powerfmt", "serde", @@ -848,15 +1041,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" [[package]] name = "time-macros" -version = "0.2.19" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" dependencies = [ "num-conv", "time-core", @@ -872,40 +1065,74 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tokio" +version = "1.46.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" +dependencies = [ + "backtrace", + "bytes", + "io-uring", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "slab", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "toml" -version = "0.8.19" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +checksum = "ed0aee96c12fa71097902e0bb061a5e1ebd766a6636bb605ba401c45c1650eac" dependencies = [ + "indexmap", "serde", "serde_spanned", "toml_datetime", - "toml_edit", + "toml_parser", + "toml_writer", + "winnow", ] [[package]] name = "toml_datetime" -version = "0.6.8" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +checksum = "bade1c3e902f58d73d3f294cd7f20391c1cb2fbcb643b73566bc773971df91e3" dependencies = [ "serde", ] [[package]] -name = "toml_edit" -version = "0.22.23" +name = "toml_parser" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02a8b472d1a3d7c18e2d61a489aee3453fd9031c33e4f55bd533f4a7adca1bee" +checksum = "97200572db069e74c512a14117b296ba0a80a30123fbbb5aa1f4a348f639ca30" dependencies = [ - "indexmap", - "serde", - "serde_spanned", - "toml_datetime", "winnow", ] +[[package]] +name = "toml_writer" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b679217f2848de74cabd3e8fc5e6d66f40b7da40f8e1954d92054d9010690fd5" + [[package]] name = "tracing" version = "0.1.41" @@ -1071,6 +1298,12 @@ dependencies = [ "quote", ] +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "wasi" version = "0.13.3+wasi-0.2.2" @@ -1169,6 +1402,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.59.0" @@ -1244,12 +1486,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.7.0" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e49d2d35d3fad69b39b94139037ecfb4f359f08958b9c11e7315ce770462419" -dependencies = [ - "memchr", -] +checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95" [[package]] name = "wit-bindgen-rt" diff --git a/Cargo.toml b/Cargo.toml index 805dab66f..024ec4068 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,14 +2,24 @@ members = ["rust/*"] resolver = "2" +[workspace.package] +authors = ["Stackable "] + [workspace.dependencies] -clap = { version = "4.5.27", features = ["derive"] } +clap = { version = "4.5.41", features = ["derive"] } +clap_complete = "4.5.55" git2 = "0.20.1" +glob = "0.3.2" +semver = { version = "1.0.26", features = ["serde"] } serde = { version = "1.0.217", features = ["derive"] } +serde_json = "1.0.140" snafu = "0.8.5" +strum = { version = "0.27.1", features = ["derive"] } tempfile = "3.16.0" -time = { version = "0.3.37", features = ["parsing"] } -toml = "0.8.19" +time = { version = "0.3.41", features = ["parsing", "formatting"] } +tokio = { version = "1.46.1", features = ["rt", "macros", "process"] } +toml = "0.9.2" tracing = "0.1.41" tracing-indicatif = "0.3.9" tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } +url = "2.5.4" diff --git a/boil.toml b/boil.toml new file mode 100644 index 000000000..67cfac8be --- /dev/null +++ b/boil.toml @@ -0,0 +1,5 @@ +[build-arguments] +STACKABLE_USER_NAME = "stackable" +STACKABLE_USER_UID = "1000" +STACKABLE_USER_GID = "1000" +DELETE_CACHES = "true" diff --git a/rust/boil/Cargo.toml b/rust/boil/Cargo.toml new file mode 100644 index 000000000..cb68a0201 --- /dev/null +++ b/rust/boil/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "boil" +version = "0.0.1" +edition = "2024" + +[dependencies] +clap.workspace = true +clap_complete.workspace = true +git2.workspace = true +glob.workspace = true +semver.workspace = true +serde.workspace = true +serde_json.workspace = true +snafu.workspace = true +strum.workspace = true +time.workspace = true +tokio.workspace = true +toml.workspace = true +url.workspace = true diff --git a/rust/boil/README.md b/rust/boil/README.md new file mode 100644 index 000000000..e29b38986 --- /dev/null +++ b/rust/boil/README.md @@ -0,0 +1,31 @@ +# boil + +boil builds container images in parallel. + +- Define versions of container images and version specific values via the `boil-config.toml` file. +- Refer to local images in Containerfiles via `FROM local-image/...`. Nesting is supported. +- Structured output is provided for any potential follow-up tasks. + +## Quick Overview + +```shell +# Builds all version of the image located in the 'airflow' folder +boil build airflow + +# Builds the 3.0.1 version of the image located in the 'airflow' folder +boil build airflow=3.0.1 + +# Builds both the 3.0.1 and 2.10.5 versions of the image located in the +# 'airflow' folder +boil build airflow=3.0.1,2.10.5 + +# Builds all versions of the images located in the 'airflow' and 'opa' folder +boil build airflow opa + +# Display a list of all images and their declared versions as structured JSON +# output +boil show images + +# Soon (hopefully) implemented +boil show graph +``` diff --git a/rust/boil/src/build/bakefile.rs b/rust/boil/src/build/bakefile.rs new file mode 100644 index 000000000..e56896888 --- /dev/null +++ b/rust/boil/src/build/bakefile.rs @@ -0,0 +1,478 @@ +use std::{ + collections::BTreeMap, + fmt::Debug, + ops::{Deref, DerefMut}, + path::PathBuf, +}; + +use glob::glob; +use serde::Serialize; +use snafu::{ResultExt, Snafu}; +use time::format_description::well_known::Rfc3339; +use url::Host; + +use crate::{ + VersionExt, + build::{ + cli, + docker::{BuildArgument, BuildArguments}, + image::{Image, ImageConfig, ImageConfigError, ImageOptions, VersionOptionsPair}, + platform::TargetPlatform, + }, + config::Config, + utils::{format_image_manifest_uri, format_image_repository_uri}, +}; + +pub const OPEN_CONTAINER_IMAGE_REVISION: &str = "org.opencontainers.image.revision"; +pub const OPEN_CONTAINER_IMAGE_CREATED: &str = "org.opencontainers.image.created"; + +pub const ENTRY_TARGET_NAME_PREFIX: &str = "entry--"; + +#[derive(Debug, Snafu)] +pub enum GitError { + #[snafu(display("failed to open git repository"))] + OpenRepository { source: git2::Error }, + + #[snafu(display("failed to parse HEAD revision"))] + ParseHeadRevision { source: git2::Error }, +} + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("failed to format current datetime"))] + FormatTime { source: time::error::Format }, + + #[snafu(display("failed to get revision"))] + GetRevision { source: GitError }, + + #[snafu(display("failed to create target graph"))] + CreateGraph { source: TargetsError }, +} + +#[derive(Debug, Snafu)] +pub enum TargetsError { + #[snafu(display("encountered invalid product version"))] + InvalidProductVersion { source: ImageConfigError }, + + #[snafu(display("failed to read image config"))] + ReadImageConfig { source: ImageConfigError }, +} + +#[derive(Debug, Default)] +pub struct TargetsOptions { + pub only_entry: bool, +} + +/// Contains targets selected by the user. +/// +/// This is a map which uses the image/target name as the key. Each key points to another map, +/// which contains one entry per version of the target. Each value contains the image options and +/// a boolean flag to indicate if this target is an entry target. +#[derive(Debug, Default)] +pub struct Targets(BTreeMap>); + +impl Deref for Targets { + type Target = BTreeMap>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for Targets { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl IntoIterator for Targets { + type Item = (String, BTreeMap); + type IntoIter = + std::collections::btree_map::IntoIter>; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +impl Targets { + pub fn all(options: TargetsOptions) -> Result { + let image_config_paths = glob("./**/boil-config.toml") + .expect("glob pattern must be valid") + .filter_map(Result::ok); + + let mut targets = Self::default(); + + for image_config_path in image_config_paths { + let image_config = + ImageConfig::from_file(&image_config_path).context(ReadImageConfigSnafu)?; + + let image_name = image_config_path + .parent() + .expect("there must be a parent") + .to_string_lossy() + .into_owned(); + + let pairs = image_config.all(); + + targets.insert_targets(image_name.to_owned(), pairs, &options, true)?; + } + + println!("{targets:#?}"); + + Ok(targets) + } + + pub fn from_images(images: &[Image], options: TargetsOptions) -> Result { + let mut targets = Self::default(); + + for image in images { + // TODO (@Techassi): We should instead build the graph based on the Dockerfile(s), + // because this is the source of truth and what ultimately gets built. The boil config + // files are not a source a truth, but just provide data needed during the build. + let image_config_path = PathBuf::new().join(&image.name).join("boil-config.toml"); + + // Read the product config which defines supported product versions and their dependencies as + // well as other values. + let image_config = + ImageConfig::from_file(image_config_path).context(ReadImageConfigSnafu)?; + + // Create a list of product versions we need to generate targets for in the bakefile. + let pairs = image_config + .filter_by_version(&image.versions) + .context(InvalidProductVersionSnafu)?; + + targets.insert_targets(image.name.clone(), pairs, &options, true)?; + } + + Ok(targets) + } + + fn insert_targets( + &mut self, + image_name: String, + pairs: Vec, + options: &TargetsOptions, + is_entry: bool, + ) -> Result<(), TargetsError> { + for VersionOptionsPair { + version: image_version, + options: image_options, + } in pairs + { + if !options.only_entry { + // TODO (@Techassi): Add cycle detection + for (image_name, image_version) in &image_options.local_images { + if self + .get(image_name) + .is_some_and(|image_versions| image_versions.contains_key(image_version)) + { + continue; + } + + let product_config_path = + PathBuf::new().join(image_name).join("boil-config.toml"); + + let product_config = ImageConfig::from_file(product_config_path) + .context(ReadImageConfigSnafu)?; + + let pairs = product_config + .filter_by_version(&[image_version]) + .context(InvalidProductVersionSnafu)?; + + // Wowzers, recursion! + self.insert_targets(image_name.clone(), pairs, options, false)?; + } + } + + self.entry(image_name.clone()) + .or_default() + .insert(image_version, (image_options, is_entry)); + } + + Ok(()) + } +} + +#[derive(Debug, Default, Serialize)] +pub struct Bakefile { + #[serde(rename = "group")] + pub groups: BTreeMap, + + #[serde(rename = "target")] + pub targets: BTreeMap, +} + +impl Bakefile { + /// Create a bakefile from the [`BuildArguments`](cli::BuildArguments) provided via the CLI. + /// + /// This will only create targets for selected entry images and their dependencies. There is no + /// need to filter anything out afterwards. The filtering is done automatically internally. + pub fn from_args(args: &cli::BuildArguments, config: Config) -> Result { + let graph = Targets::from_images(&args.images, TargetsOptions::default()) + .context(CreateGraphSnafu)?; + Self::from_targets(graph, args, config) + } + + /// Returns all image manifest URIs for entry images. + pub fn image_manifest_uris(&self) -> Vec<&str> { + self.targets + .iter() + // We only care about the entry targets, because those are the primary images boil + // builds. + .filter(|(target_name, _)| target_name.starts_with(ENTRY_TARGET_NAME_PREFIX)) + // The image manifest URIs file only contains the image tags + .flat_map(|(_, target)| &target.tags) + // Flatten multiple tags (boil currently only ever writes a single one, but the data + // structure can accept a list). + .map(|s| s.as_str()) + .collect() + } + + fn from_targets( + targets: Targets, + args: &cli::BuildArguments, + config: Config, + ) -> Result { + let mut bakefile_targets = BTreeMap::new(); + let mut groups: BTreeMap = BTreeMap::new(); + + // TODO (@Techassi): Can we somehow optimize this to come by with minimal amount of + // cloning, because we also need to clone on every loop iteration below. + let mut docker_build_arguments = config.build_arguments; + docker_build_arguments.extend(args.docker_build_arguments.clone()); + docker_build_arguments.insert(BuildArgument::new( + "RELEASE_VERSION".to_owned(), + args.image_version.base_prerelease(), + )); + + for (image_name, image_versions) in targets.0.into_iter() { + for (image_version, (image_options, is_entry)) in image_versions { + // TODO (@Techassi): Clean this up + // TODO (@Techassi): Move the arg formatting into functions + let mut docker_build_arguments = docker_build_arguments.clone(); + + let local_version_docker_args: Vec<_> = image_options + .local_images + .iter() + .map(|(image_name, image_version)| { + BuildArgument::new( + format!( + "{image_name}_VERSION", + image_name = image_name.to_uppercase().replace('-', "_") + ), + image_version.to_string(), + ) + }) + .collect(); + + docker_build_arguments.extend(image_options.build_arguments.clone()); + docker_build_arguments.extend(local_version_docker_args); + docker_build_arguments.insert(BuildArgument::new( + "PRODUCT_VERSION".to_owned(), + image_version.to_string(), + )); + + // The image registry, eg. `oci.stackable.tech` or `localhost` + let image_registry = if args.use_localhost_registry { + &Host::Domain(String::from("localhost")) + } else { + &args.registry + }; + + let image_repository_uri = format_image_repository_uri( + image_registry, + &args.registry_namespace, + &image_name, + ); + + let image_manifest_uri = format_image_manifest_uri( + &image_repository_uri, + &image_version, + &args.image_version, + args.target_platform.architecture(), + ); + + let dockerfile = PathBuf::new().join(&image_name).join("Dockerfile"); + let revision = Self::git_head_revision().context(GetRevisionSnafu)?; + let date_time = Self::now()?; + + let target_name = if is_entry { + Self::format_entry_target_name(&image_name, &image_version) + } else { + Self::format_target_name(&image_name, &image_version) + }; + + let contexts: BTreeMap<_, _> = image_options + .local_images + .iter() + .map(|(image_name, image_version)| { + let context_name = Self::format_context_name(image_name); + let context_target = Self::format_context_target(image_name, image_version); + + (context_name, context_target) + }) + .collect(); + + let target = BakefileTarget { + annotations: BakefileTarget::annotations(&date_time, &revision), + labels: BakefileTarget::labels(date_time, revision), + tags: vec![image_manifest_uri], + arguments: docker_build_arguments, + platforms: vec![args.target_platform.clone()], + context: PathBuf::from("."), + dockerfile, + contexts, + }; + + bakefile_targets.insert(target_name, target); + + // Add the target to the default group if it is an entry + if is_entry { + groups + .entry("default".to_owned()) + .or_default() + .targets + .push(Self::format_entry_target_name(&image_name, &image_version)); + } + } + } + + Ok(Self { + targets: bakefile_targets, + groups, + }) + } + + /// Formats and returns the entry target name, eg. `entry--opa-1_4_2`. + fn format_entry_target_name(image_name: &str, image_version: &str) -> String { + let target_name = Self::format_target_name(image_name, image_version); + format!("{ENTRY_TARGET_NAME_PREFIX}{target_name}") + } + + /// Formats and returns the target name, eg. `stackable-base-1_0_0`. + fn format_target_name(image_name: &str, image_version: &str) -> String { + // Replace any slashes from nested product names, eg. shared/protobuf, because docker buildx + // has this weird restriction (because it also supports push, which we do on our own). We + // are therefore artificially limited what target names we can use: [a-zA-Z0-9_-]+ + let product_name = image_name.replace('/', "__"); + + // The dots in the semantic version also need to be replaced. + let product_version = image_version.to_string().replace('.', "_"); + + format!("{product_name}-{product_version}") + } + + /// Formats and return the context name, eg. `stackable/image/stackable-base-1_0_0`. + fn format_context_name(name: &str) -> String { + format!("local-image/{name}") + // format!("stackable/image/{name}") + } + + /// Formats and returns the context target name, eg. `target:stackable-base-1_0_0`. + fn format_context_target(name: &str, version: &str) -> String { + let target_name = Self::format_target_name(name, version); + format!("target:{target_name}") + } + + fn now() -> Result { + time::UtcDateTime::now() + .format(&Rfc3339) + .context(FormatTimeSnafu) + } + + fn git_head_revision() -> Result { + let repo = git2::Repository::open(".").context(OpenRepositorySnafu)?; + let rev = repo.revparse("HEAD").context(ParseHeadRevisionSnafu)?; + + Ok(rev.from().unwrap().id().to_string()) + } +} + +// TODO (@Techassi): Figure out of we can use borrowed data in here. This would avoid a whole bunch +// of cloning. +#[derive(Debug, Serialize)] +pub struct BakefileTarget { + pub annotations: Vec, + pub context: PathBuf, + + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub contexts: BTreeMap, + pub dockerfile: PathBuf, + + #[serde(rename = "args", skip_serializing_if = "BuildArguments::is_empty")] + pub arguments: BuildArguments, + + pub labels: BTreeMap, + pub tags: Vec, + pub platforms: Vec, +} + +impl BakefileTarget { + fn annotations(date_time: &str, revision: &str) -> Vec { + vec![ + format!("{OPEN_CONTAINER_IMAGE_CREATED}={date_time}"), + format!("{OPEN_CONTAINER_IMAGE_REVISION}={revision}"), + ] + } + + fn labels(date_time: String, revision: String) -> BTreeMap { + BTreeMap::from([ + (OPEN_CONTAINER_IMAGE_CREATED.to_owned(), date_time.clone()), + (OPEN_CONTAINER_IMAGE_REVISION.to_owned(), revision), + ("build-date".to_owned(), date_time), + ]) + } +} + +#[derive(Debug, Default, Serialize)] +pub struct BakefileGroup { + targets: Vec, +} + +// #[derive(Debug, Default)] +// pub struct Graph { +// targets: BTreeMap>, +// } + +// impl Graph { +// pub fn all() -> Self { +// let image_config_paths: Vec<_> = glob("./**/boil-config.toml") +// .expect("glob pattern must be valid") +// .filter_map(Result::ok) +// .collect(); + +// let mut targets = Self::default(); + +// for image_config_path in image_config_paths { +// let image_config = ImageConfig::from_file(&image_config_path).unwrap(); + +// let (image_name, _) = image_config_path +// .to_str() +// .unwrap() +// .rsplit_once('/') +// .unwrap(); + +// let pairs = image_config.filter_by_version(None).unwrap(); + +// targets.insert_targets(image_name.to_owned(), pairs); +// } + +// targets +// } + +// fn insert_targets( +// &mut self, +// image_name: String, +// pairs: Vec, +// ) -> Vec { +// let mut nodes = Vec::new(); + +// for VersionOptionsPair { version, options } in pairs { +// let key = format!("{image_name}:{version}"); +// let child_nodes = Vec::new(); + +// // let nodes = self.insert_targets(image_name, pairs); +// } +// } +// } diff --git a/rust/boil/src/build/cli.rs b/rust/boil/src/build/cli.rs new file mode 100644 index 000000000..871923941 --- /dev/null +++ b/rust/boil/src/build/cli.rs @@ -0,0 +1,122 @@ +use std::path::PathBuf; + +use clap::{Args, ValueHint, value_parser}; +use semver::Version; +use url::Host; + +use crate::{ + build::{ + docker::BuildArgument, + image::Image, + platform::{Architecture, TargetPlatform}, + }, + cli::parse_image_version, +}; + +#[derive(Debug, Args)] +pub struct BuildArguments { + /// The image(s) which should be build. The format is name[=version,...]. + #[arg(help_heading = "Image Options", required = true)] + pub images: Vec, + + // The action currently does the wrong thing here. It includes the + // architecture even though it should come from the --target-platform arg. + // The release arg is NOT needed, because this version IS the release version. + /// The image version being built. + #[arg( + short, long, + value_parser = parse_image_version, + default_value_t = Self::default_image_version(), + help_heading = "Image Options" + )] + pub image_version: Version, + + /// Target platform of the image. + #[arg( + short, long, + short_alias = 'a', alias = "architecture", + default_value_t = Self::default_architecture(), + help_heading = "Image Options" + )] + pub target_platform: TargetPlatform, + + /// Image registry used in image manifests, URIs, and tags. + #[arg( + short, long, + default_value_t = Self::default_registry(), + value_parser = Host::parse, + value_hint = ValueHint::Hostname, + help_heading = "Registry Options" + )] + pub registry: Host, + + /// The namespace within the given registry. + #[arg( + short = 'n', + long = "registry-namespace", + alias = "organization", + default_value = "sdp", + help_heading = "Registry Options" + )] + pub registry_namespace: String, + + /// Use 'localhost' as the registry instead of to avoid any accidental interactions + /// with remote registries. + /// + /// This is especially useful in CI, which can re-tag the image before pushing it. + #[arg(long, help_heading = "Registry Options")] + pub use_localhost_registry: bool, + + /// Override the target containerfile used, points to /. + #[arg( + long, + default_value_os_t = Self::default_target_containerfile(), + value_hint = ValueHint::FilePath, + help_heading = "Build Options" + )] + pub target_containerfile: PathBuf, + + /// Override build arguments, in key=value format. The key is case insensitive. This argument + /// can be supplied multiple times. + #[arg( + long = "build-argument", + alias = "build-arg", + help_heading = "Build Options" + )] + pub docker_build_arguments: Vec, + + /// Write target image tags to . Useful for signing or other follow-up CI steps. + #[arg( + long, + alias = "export-tags-file", + help_heading = "Build Options", + value_name = "EXPORT_FILE", + value_hint = ValueHint::FilePath, + value_parser = value_parser!(PathBuf), + default_missing_value = "boil-target-tags", + num_args(0..=1) + )] + pub export_image_manifest_uris: Option, + + /// Dry run. This does not build the image(s) but instead prints out the bakefile. + #[arg(short, long, alias = "dry")] + pub dry_run: bool, +} + +impl BuildArguments { + fn default_image_version() -> Version { + "0.0.0-dev".parse().expect("must be a valid SemVer") + } + + fn default_architecture() -> TargetPlatform { + TargetPlatform::Linux(Architecture::Amd64) + } + + fn default_registry() -> Host { + Host::Domain(String::from("oci.stackable.tech")) + } + + fn default_target_containerfile() -> PathBuf { + PathBuf::from("Dockerfile") + } +} diff --git a/rust/boil/src/build/docker.rs b/rust/boil/src/build/docker.rs new file mode 100644 index 000000000..38f10e3cb --- /dev/null +++ b/rust/boil/src/build/docker.rs @@ -0,0 +1,156 @@ +use std::{ + collections::BTreeSet, + fmt::Display, + ops::{Deref, DerefMut}, + str::FromStr, +}; + +use serde::{Deserialize, Serialize, de::Visitor, ser::SerializeMap}; +use snafu::{Snafu, ensure}; + +#[derive(Debug, Snafu)] +pub enum ParseBuildArgumentError { + NonAscii, +} + +// TODO (@Techassi): Unify parsing/casing in one place +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct BuildArgument((String, String)); + +impl BuildArgument { + pub fn new(key: String, value: String) -> Self { + Self((key.replace(['-', '/'], "_").to_uppercase(), value)) + } +} + +impl FromStr for BuildArgument { + type Err = ParseBuildArgumentError; + + fn from_str(s: &str) -> Result { + ensure!(s.is_ascii(), NonAsciiSnafu); + + let (key, value) = s.split_once('=').unwrap(); + let key = key.replace(['-', '/'], "_").to_uppercase(); + + Ok(Self((key, value.to_owned()))) + } +} + +impl<'de> Deserialize<'de> for BuildArgument { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct BuildArgumentVisitor; + + impl Visitor<'_> for BuildArgumentVisitor { + type Value = BuildArgument; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(formatter, "a valid build argument") + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + BuildArgument::from_str(v).map_err(serde::de::Error::custom) + } + } + + deserializer.deserialize_str(BuildArgumentVisitor) + } +} + +impl Display for BuildArgument { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let (key, value) = &self.0; + write!(f, "{key}={value}") + } +} + +#[derive(Clone, Debug, Default)] +pub struct BuildArguments(BTreeSet); + +impl Deref for BuildArguments { + type Target = BTreeSet; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for BuildArguments { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl Extend for BuildArguments { + fn extend>(&mut self, iter: T) { + self.0.extend(iter); + } +} + +impl IntoIterator for BuildArguments { + type Item = BuildArgument; + + type IntoIter = std::collections::btree_set::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +impl<'de> Deserialize<'de> for BuildArguments { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_map(BuildArgumentsVisitor) + } +} + +struct BuildArgumentsVisitor; + +impl<'de> Visitor<'de> for BuildArgumentsVisitor { + type Value = BuildArguments; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(formatter, "a map of valid build arguments") + } + + fn visit_map(self, mut map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + let mut args = BTreeSet::new(); + + while let Some((key, value)) = map.next_entry()? { + args.insert(BuildArgument::new(key, value)); + } + + Ok(BuildArguments(args)) + } +} + +impl Serialize for BuildArguments { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut map = serializer.serialize_map(Some(self.len()))?; + + for BuildArgument((key, value)) in &self.0 { + map.serialize_entry(&key, &value)?; + } + + map.end() + } +} + +impl BuildArguments { + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} diff --git a/rust/boil/src/build/image.rs b/rust/boil/src/build/image.rs new file mode 100644 index 000000000..4f84e127b --- /dev/null +++ b/rust/boil/src/build/image.rs @@ -0,0 +1,163 @@ +use std::{ + collections::BTreeMap, + fmt::Display, + ops::Deref, + path::{Path, PathBuf}, + str::FromStr, +}; + +use serde::Deserialize; +use snafu::{ResultExt as _, Snafu}; + +use crate::{IfContext, build::docker::BuildArguments}; + +#[derive(Debug, Snafu)] +pub enum ParseImageError { + #[snafu(display("encountered invalid format, expected name[=version,...]"))] + InvalidFormat, +} + +#[derive(Clone, Debug)] +pub struct Image { + pub name: String, + pub versions: Vec, +} + +impl FromStr for Image { + type Err = ParseImageError; + + fn from_str(s: &str) -> Result { + let parts: Vec<_> = s.split('=').collect(); + + match parts.len() { + 1 => Ok(Self::new_unversioned(parts[0].to_owned())), + 2 => { + let versions: Vec<_> = parts[1].split(',').map(ToOwned::to_owned).collect(); + Ok(Self::new(parts[0].to_owned(), versions)) + } + _ => InvalidFormatSnafu.fail(), + } + } +} + +impl Display for Image { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.versions.is_empty() { + f.write_str(&self.name) + } else { + write!( + f, + "{name}={versions}", + name = self.name, + versions = self.versions.join(",") + ) + } + } +} + +impl Image { + fn new(name: String, versions: Vec) -> Self { + Self { name, versions } + } + + fn new_unversioned(name: String) -> Self { + Self { + name, + versions: vec![], + } + } +} + +#[derive(Debug, Snafu)] +pub enum ImageConfigError { + #[snafu(display("failed to read config file at {path}", path = path.display()))] + ReadFile { + source: std::io::Error, + path: PathBuf, + }, + + #[snafu(display("failed to deserialize config file from TOML"))] + Deserialize { source: toml::de::Error }, + + #[snafu(display("provided filter version yielded empty list"))] + EmptyFilter, +} + +#[derive(Debug, Deserialize)] +pub struct ImageConfig { + pub versions: ImageVersions, +} + +impl ImageConfig { + pub fn filter_by_version( + self, + versions: &[V], + ) -> Result, ImageConfigError> + where + V: AsRef + PartialEq, + { + let versions: Vec<_> = self + .pairs() + .filter(|(image_version, _)| { + versions.is_empty() || versions.iter().any(|v| v.as_ref() == image_version) + }) + .map(Into::into) + .collect(); + + versions.if_context(|v| !v.is_empty(), EmptyFilterSnafu) + } + + pub fn all(self) -> Vec { + self.pairs().map(Into::into).collect() + } + + fn pairs(self) -> impl Iterator { + self.versions.0.into_iter() + } +} + +impl ImageConfig { + pub fn from_file(path: impl AsRef) -> Result { + let path = path.as_ref(); + let contents = std::fs::read_to_string(path).with_context(|_| ReadFileSnafu { path })?; + toml::from_str(&contents).context(DeserializeSnafu) + } +} + +#[derive(Debug, Deserialize)] +pub struct ImageVersions(BTreeMap); + +impl Deref for ImageVersions { + type Target = BTreeMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct ImageOptions { + #[serde(default)] + pub local_images: BTreeMap, + + // NOTE (@Techassi): Potentially add a dependencies field here which will be automatically be + // suffixed with _VERSION. + #[serde(default)] + pub build_arguments: BuildArguments, +} + +#[derive(Debug)] +pub struct VersionOptionsPair { + pub version: String, + pub options: ImageOptions, +} + +impl From<(String, ImageOptions)> for VersionOptionsPair { + fn from(value: (String, ImageOptions)) -> Self { + VersionOptionsPair { + version: value.0, + options: value.1, + } + } +} diff --git a/rust/boil/src/build/mod.rs b/rust/boil/src/build/mod.rs new file mode 100644 index 000000000..aa3753f4b --- /dev/null +++ b/rust/boil/src/build/mod.rs @@ -0,0 +1,105 @@ +use std::{ + fmt::Debug, + process::{Command, Stdio}, +}; + +use snafu::{OptionExt, ResultExt, Snafu, ensure}; + +use crate::{ + build::{bakefile::Bakefile, cli::BuildArguments}, + config::Config, +}; + +pub mod bakefile; +pub mod cli; +pub mod docker; +pub mod image; +pub mod platform; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("failed to create bakefile"))] + CreateBakefile { source: bakefile::Error }, + + #[snafu(display("failed to write image manifest URIs to file"))] + WriteImageManifestUrisFile { source: std::io::Error }, + + #[snafu(display("failed to serialize bakefile as JSON"))] + SerializeBakefile { source: serde_json::Error }, + + #[snafu(display("failed to acquire stdin handle"))] + AcquireStdinHandle, + + #[snafu(display("failed to run child process"))] + RunChildProcess { source: std::io::Error }, + + #[snafu(display("encountered invalid image version, must not include any build metadata"))] + InvalidImageVersion, +} + +pub fn run_command(args: BuildArguments, config: Config) -> Result<(), Error> { + // TODO (@Techassi): Parse Dockerfile instead to build the target graph + // let pattern = format!("**/{}/boil-config.toml", arguments.product.name); + + // Validation + ensure!( + args.image_version.build.is_empty(), + InvalidImageVersionSnafu + ); + + // Create bakefile + let bakefile = Bakefile::from_args(&args, config).context(CreateBakefileSnafu)?; + let image_manifest_uris = bakefile.image_manifest_uris(); + let count = image_manifest_uris.len(); + + // Write the image manifest URIs to file if requested + if let Some(path) = args.export_image_manifest_uris { + std::fs::write(path, image_manifest_uris.join("\n")) + .context(WriteImageManifestUrisFileSnafu)?; + } + + // Output the bakefile contents if in dry-run mode + if args.dry_run { + return serde_json::to_writer_pretty(std::io::stdout(), &bakefile) + .context(SerializeBakefileSnafu); + } + + // TODO (@Techassi): Invoke this directly using the Docker daemon via bollard + // Finally invoke the docker buildx bake command + let mut child = Command::new("docker") + .arg("buildx") + .arg("bake") + // .arg("--no-cache") + .arg("--file") + .arg("-") + .stdin(Stdio::piped()) + .spawn() + .unwrap(); + + let stdin_handle = child.stdin.take().with_context(|| { + child + .kill() + .expect("killing the child process must succeed"); + AcquireStdinHandleSnafu + })?; + + serde_json::to_writer(stdin_handle, &bakefile).with_context(|_| { + child + .kill() + .expect("killing the child process must succeed"); + SerializeBakefileSnafu + })?; + + let status = child.wait().context(RunChildProcessSnafu)?; + + // TODO (@Techassi): Return an error if the status was not a success + if status.success() { + println!( + "Successfully built {count} image{plural}:\n{images}", + plural = if count > 1 { "s" } else { "" }, + images = image_manifest_uris.join("\n") + ); + } + + Ok(()) +} diff --git a/rust/boil/src/build/platform.rs b/rust/boil/src/build/platform.rs new file mode 100644 index 000000000..857e4b532 --- /dev/null +++ b/rust/boil/src/build/platform.rs @@ -0,0 +1,67 @@ +use std::{fmt::Display, str::FromStr}; + +use serde::Serialize; +use snafu::{OptionExt as _, ResultExt, Snafu}; + +#[derive(Debug, Snafu)] +pub enum ParseArchitecturePairError { + #[snafu(display("encountered invalid format, expected platform/architecture"))] + InvalidFormat, + + #[snafu(display("failed to parse architecture"))] + ParseArchitecture { source: strum::ParseError }, + + #[snafu(display("unsupported, target platform"))] + UnsupportedPlatform, +} + +#[derive(Clone, Debug)] +pub enum TargetPlatform { + Linux(Architecture), +} + +impl Serialize for TargetPlatform { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&self.to_string()) + } +} + +impl FromStr for TargetPlatform { + type Err = ParseArchitecturePairError; + + fn from_str(s: &str) -> Result { + let (platform, architecture) = s.split_once('/').context(InvalidFormatSnafu)?; + let architecture = Architecture::from_str(architecture).context(ParseArchitectureSnafu)?; + + match platform { + "linux" => Ok(Self::Linux(architecture)), + _ => UnsupportedPlatformSnafu.fail(), + } + } +} + +impl Display for TargetPlatform { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self { + TargetPlatform::Linux(architecture) => write!(f, "linux/{architecture}"), + } + } +} + +impl TargetPlatform { + pub fn architecture(&self) -> &Architecture { + match self { + TargetPlatform::Linux(architecture) => architecture, + } + } +} + +#[derive(Copy, Clone, Debug, strum::Display, strum::EnumString, strum::AsRefStr)] +#[strum(serialize_all = "lowercase")] +pub enum Architecture { + Amd64, + Arm64, +} diff --git a/rust/boil/src/cli.rs b/rust/boil/src/cli.rs new file mode 100644 index 000000000..5914fc3c2 --- /dev/null +++ b/rust/boil/src/cli.rs @@ -0,0 +1,89 @@ +use std::{path::PathBuf, str::FromStr}; + +use clap::{Args, Parser, Subcommand}; +use clap_complete::Shell; +use semver::Version; +use snafu::{ResultExt, Snafu, ensure}; + +use crate::build::cli::BuildArguments; + +#[derive(Debug, Parser)] +#[command(author, version, about)] +pub struct Cli { + /// Path to the configuration file. + #[arg(short = 'c', long = "configuration", default_value_os_t = Self::default_config_path())] + pub config_path: PathBuf, + + /// Path to the OpenShift configuration file. + #[arg(long, default_value_os_t = Self::default_openshift_config_path())] + pub openshift_config_path: PathBuf, + + #[arg(short, long, default_value_os_t = Self::default_base_path())] + pub base_path: PathBuf, + + #[command(subcommand)] + pub command: Command, +} + +impl Cli { + fn default_config_path() -> PathBuf { + PathBuf::from("./boil.toml") + } + + fn default_openshift_config_path() -> PathBuf { + PathBuf::from("./openshift.toml") + } + + fn default_base_path() -> PathBuf { + PathBuf::from(".") + } +} + +#[derive(Debug, Subcommand)] +pub enum Command { + /// Build one or more product images. + /// + /// Requires docker with the buildx extension. + #[command(alias = "some-chicken")] + Build(BuildArguments), + + /// Display various structured outputs in JSON format. + Show(ShowArguments), + + /// Generate shell completions. + Completions(CompletionsArguments), +} + +#[derive(Debug, Args)] +pub struct ShowArguments { + #[command(subcommand)] + pub commands: ShowCommand, +} + +#[derive(Debug, Subcommand)] +pub enum ShowCommand { + Images, + Tree, +} + +#[derive(Debug, Args)] +pub struct CompletionsArguments { + /// Shell to generate completions for. + pub shell: Shell, +} + +#[derive(Debug, Snafu)] +pub enum ParseImageVersionError { + #[snafu(display("failed to parse semantic version"))] + ParseVersion { source: semver::Error }, + + #[snafu(display("semantic version must not contain build metadata"))] + ContainsBuildMetadata, +} + +pub fn parse_image_version(input: &str) -> Result { + let version = Version::from_str(input).context(ParseVersionSnafu)?; + ensure!(version.build.is_empty(), ContainsBuildMetadataSnafu); + + Ok(version) +} diff --git a/rust/boil/src/completions/mod.rs b/rust/boil/src/completions/mod.rs new file mode 100644 index 000000000..14518365a --- /dev/null +++ b/rust/boil/src/completions/mod.rs @@ -0,0 +1,10 @@ +use clap::CommandFactory; + +use crate::cli::{Cli, CompletionsArguments}; + +pub fn run_command(arguments: CompletionsArguments) { + let mut cli = Cli::command(); + let bin_name = cli.get_bin_name().unwrap_or("boil").to_owned(); + + clap_complete::generate(arguments.shell, &mut cli, bin_name, &mut std::io::stdout()); +} diff --git a/rust/boil/src/config.rs b/rust/boil/src/config.rs new file mode 100644 index 000000000..64e16aee9 --- /dev/null +++ b/rust/boil/src/config.rs @@ -0,0 +1,26 @@ +use std::path::Path; + +use serde::Deserialize; +use snafu::{ResultExt, Snafu}; + +use crate::build::docker::BuildArguments; + +#[derive(Debug, Snafu)] +pub enum ConfigError { + ReadFile { source: std::io::Error }, + + Deserialize { source: toml::de::Error }, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct Config { + pub build_arguments: BuildArguments, +} + +impl Config { + pub fn from_file(path: impl AsRef) -> Result { + let contents = std::fs::read_to_string(path).context(ReadFileSnafu)?; + toml::from_str(&contents).context(DeserializeSnafu) + } +} diff --git a/rust/boil/src/main.rs b/rust/boil/src/main.rs new file mode 100644 index 000000000..fadb093d6 --- /dev/null +++ b/rust/boil/src/main.rs @@ -0,0 +1,94 @@ +use clap::Parser; +use semver::Version; +use snafu::{ResultExt, Snafu}; + +use crate::{ + cli::{Cli, Command, ShowCommand}, + config::Config, +}; + +// Common modules +mod cli; +mod config; +mod utils; + +// Command modules +mod build; +mod completions; +mod show; + +pub trait IfContext: Sized { + fn if_context(self, predicate: P, context: C) -> Result + where + P: Fn(&Self) -> bool, + C: snafu::IntoError, + E: std::error::Error + snafu::ErrorCompat; +} + +impl IfContext for T { + fn if_context(self, predicate: P, context: C) -> Result + where + P: Fn(&Self) -> bool, + C: snafu::IntoError, + E: std::error::Error + snafu::ErrorCompat, + { + match predicate(&self) { + true => Ok(self), + false => Err(context.into_error(snafu::NoneError)), + } + } +} + +pub trait VersionExt { + /// Returns the base of a [`Version`] as a string, eg. `1.2.3`. + fn base(&self) -> String; + + /// Returns the base and prerelease of a [`Version`] as a string, eg. `1.2.3-rc.1`. + fn base_prerelease(&self) -> String; +} + +impl VersionExt for Version { + fn base(&self) -> String { + format!("{}.{}.{}", self.major, self.minor, self.patch) + } + + fn base_prerelease(&self) -> String { + let mut base = self.base(); + base.push('-'); + base.push_str(&self.pre); + base + } +} + +#[derive(Debug, Snafu)] +enum Error { + #[snafu(display("failed to run build command"))] + Build { source: build::Error }, + + #[snafu(display("failed to run show command"))] + Show { source: show::images::Error }, + + #[snafu(display("failed to read config"))] + ReadConfig { source: config::ConfigError }, +} + +#[tokio::main(flavor = "current_thread")] +#[snafu::report] +async fn main() -> Result<(), Error> { + let cli = Cli::parse(); + + match cli.command { + Command::Build(arguments) => { + let config = Config::from_file(&cli.config_path).context(ReadConfigSnafu)?; + build::run_command(arguments, config).context(BuildSnafu) + } + Command::Show(arguments) => match arguments.commands { + ShowCommand::Images => show::images::run_command().context(ShowSnafu), + ShowCommand::Tree => todo!(), + }, + Command::Completions(arguments) => { + completions::run_command(arguments); + Ok(()) + } + } +} diff --git a/rust/boil/src/show/images.rs b/rust/boil/src/show/images.rs new file mode 100644 index 000000000..3baaa3929 --- /dev/null +++ b/rust/boil/src/show/images.rs @@ -0,0 +1,27 @@ +use std::collections::BTreeMap; + +use snafu::{ResultExt, Snafu}; + +use crate::build::bakefile::{Targets, TargetsOptions}; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("failed to serialize list as JSON"))] + SerializeList { source: serde_json::Error }, +} + +pub fn run_command() -> Result<(), Error> { + let list: BTreeMap<_, _> = Targets::all(TargetsOptions { only_entry: true }) + .unwrap() + .into_iter() + .map(|(image_name, image_versions)| { + let versions: Vec<_> = image_versions + .into_iter() + .map(|(image_version, (_, _))| image_version) + .collect(); + (image_name, versions) + }) + .collect(); + + serde_json::to_writer_pretty(std::io::stdout(), &list).context(SerializeListSnafu) +} diff --git a/rust/boil/src/show/mod.rs b/rust/boil/src/show/mod.rs new file mode 100644 index 000000000..8f0da9f8e --- /dev/null +++ b/rust/boil/src/show/mod.rs @@ -0,0 +1 @@ +pub mod images; diff --git a/rust/boil/src/utils.rs b/rust/boil/src/utils.rs new file mode 100644 index 000000000..561a949d6 --- /dev/null +++ b/rust/boil/src/utils.rs @@ -0,0 +1,23 @@ +use semver::Version; +use url::Host; + +use crate::build::platform::Architecture; + +/// Formats and returns the image repository URI, eg. `oci.stackable.tech/sdp/opa`. +pub fn format_image_repository_uri( + image_registry: &Host, + registry_namespace: &str, + image_name: &str, +) -> String { + format!("{image_registry}/{registry_namespace}/{image_name}") +} + +/// Formats and returns the image manifest URI, eg. `oci.stackable.tech/sdp/opa:1.4.2-stackable25.7.0-amd64`. +pub fn format_image_manifest_uri( + image_repository_uri: &str, + image_version: &str, + sdp_image_version: &Version, + architecture: &Architecture, +) -> String { + format!("{image_repository_uri}:{image_version}-stackable{sdp_image_version}-{architecture}") +} From 4e96bba3cff4a4b0b2d72903e0ab37d45a941488 Mon Sep 17 00:00:00 2001 From: Techassi Date: Tue, 12 Aug 2025 15:46:57 +0200 Subject: [PATCH 02/51] chore: Add boil configs and adjust Dockerfiles --- airflow/Dockerfile | 77 +++++---- airflow/boil-config.toml | 55 +++++++ druid/Dockerfile | 56 +++---- druid/boil-config.toml | 26 +++ hadoop/Dockerfile | 74 ++++----- hadoop/boil-config.toml | 22 +++ hadoop/hadoop/Dockerfile | 24 +-- hadoop/hadoop/boil-config.toml | 11 ++ hbase/Dockerfile | 86 +++++----- hbase/boil-config.toml | 25 +++ hbase/hbase-opa-authorizer/Dockerfile | 14 +- hbase/hbase-opa-authorizer/boil-config.toml | 5 + hbase/hbase-operator-tools/Dockerfile | 26 +-- hbase/hbase-operator-tools/boil-config.toml | 19 +++ hbase/hbase/Dockerfile | 32 ++-- hbase/hbase/boil-config.toml | 17 ++ hbase/phoenix/Dockerfile | 37 ++--- hbase/phoenix/boil-config.toml | 19 +++ hive/Dockerfile | 86 +++++----- hive/boil-config.toml | 38 +++++ java-base/Dockerfile | 16 +- java-base/boil-config.toml | 20 +++ java-devel/Dockerfile | 8 +- java-devel/boil-config.toml | 20 +++ jdk-base/Dockerfile | 16 +- jdk-base/boil-config.toml | 20 +++ kafka-testing-tools/Dockerfile | 20 +-- kafka-testing-tools/boil-config.toml | 4 + kafka/Dockerfile | 78 ++++----- kafka/boil-config.toml | 39 +++++ kafka/kafka-opa-plugin/Dockerfile | 10 +- kafka/kafka-opa-plugin/boil-config.toml | 2 + kafka/kcat/Dockerfile | 12 +- kafka/kcat/boil-config.toml | 3 + krb5/Dockerfile | 6 +- krb5/boil-config.toml | 1 + nifi/Dockerfile | 90 +++++----- nifi/boil-config.toml | 24 +++ omid/Dockerfile | 42 ++--- omid/boil-config.toml | 13 ++ opa/Dockerfile | 32 ++-- opa/boil-config.toml | 15 ++ opensearch/Dockerfile | 86 +++++----- opensearch/boil-config.toml | 4 + opensearch/security-plugin/Dockerfile | 12 +- opensearch/security-plugin/boil-config.toml | 2 + shared/statsd-exporter/Dockerfile | 14 +- shared/statsd-exporter/boil-config.toml | 2 + spark-connect-client/Dockerfile | 16 +- spark-connect-client/boil-config.toml | 6 + spark-k8s/Dockerfile | 172 ++++++++++---------- spark-k8s/boil-config.toml | 37 +++++ stackable-base/Dockerfile | 6 +- stackable-base/boil-config.toml | 2 + stackable-devel/Dockerfile | 2 +- stackable-devel/boil-config.toml | 1 + superset/Dockerfile | 66 ++++---- superset/boil-config.toml | 36 ++++ testing-tools/Dockerfile | 10 +- testing-tools/boil-config.toml | 2 + tools/Dockerfile | 10 +- tools/boil-config.toml | 7 + trino-cli/Dockerfile | 14 +- trino-cli/boil-config.toml | 2 + trino/Dockerfile | 32 ++-- trino/boil-config.toml | 23 +++ trino/storage-connector/Dockerfile | 18 +- trino/storage-connector/boil-config.toml | 11 ++ trino/trino/Dockerfile | 18 +- trino/trino/boil-config.toml | 8 + vector/Dockerfile | 8 +- vector/boil-config.toml | 6 + zookeeper/Dockerfile | 44 ++--- zookeeper/boil-config.toml | 6 + 74 files changed, 1237 insertions(+), 686 deletions(-) create mode 100644 airflow/boil-config.toml create mode 100644 druid/boil-config.toml create mode 100644 hadoop/boil-config.toml create mode 100644 hadoop/hadoop/boil-config.toml create mode 100644 hbase/boil-config.toml create mode 100644 hbase/hbase-opa-authorizer/boil-config.toml create mode 100644 hbase/hbase-operator-tools/boil-config.toml create mode 100644 hbase/hbase/boil-config.toml create mode 100644 hbase/phoenix/boil-config.toml create mode 100644 hive/boil-config.toml create mode 100644 java-base/boil-config.toml create mode 100644 java-devel/boil-config.toml create mode 100644 jdk-base/boil-config.toml create mode 100644 kafka-testing-tools/boil-config.toml create mode 100644 kafka/boil-config.toml create mode 100644 kafka/kafka-opa-plugin/boil-config.toml create mode 100644 kafka/kcat/boil-config.toml create mode 100644 krb5/boil-config.toml create mode 100644 nifi/boil-config.toml create mode 100644 omid/boil-config.toml create mode 100644 opa/boil-config.toml create mode 100644 opensearch/boil-config.toml create mode 100644 opensearch/security-plugin/boil-config.toml create mode 100644 shared/statsd-exporter/boil-config.toml create mode 100644 spark-connect-client/boil-config.toml create mode 100644 spark-k8s/boil-config.toml create mode 100644 stackable-base/boil-config.toml create mode 100644 stackable-devel/boil-config.toml create mode 100644 superset/boil-config.toml create mode 100644 testing-tools/boil-config.toml create mode 100644 tools/boil-config.toml create mode 100644 trino-cli/boil-config.toml create mode 100644 trino/boil-config.toml create mode 100644 trino/storage-connector/boil-config.toml create mode 100644 trino/trino/boil-config.toml create mode 100644 vector/boil-config.toml create mode 100644 zookeeper/boil-config.toml diff --git a/airflow/Dockerfile b/airflow/Dockerfile index 837c50066..164edb912 100644 --- a/airflow/Dockerfile +++ b/airflow/Dockerfile @@ -3,19 +3,19 @@ # - SecretsUsedInArgOrEnv : OPA_AUTH_MANAGER is a false positive and breaks the build. # check=error=true;skip=InvalidDefaultArgInFrom,SecretsUsedInArgOrEnv -ARG GIT_SYNC +ARG GIT_SYNC_VERSION # For updated versions check https://github.com/kubernetes/git-sync/releases # which should contain a image location (e.g. registry.k8s.io/git-sync/git-sync:v3.6.8) -FROM oci.stackable.tech/sdp/git-sync/git-sync:${GIT_SYNC} AS gitsync-image +FROM oci.stackable.tech/sdp/git-sync/git-sync:${GIT_SYNC_VERSION} AS gitsync-image -FROM stackable/image/shared/statsd-exporter AS statsd_exporter-builder +FROM local-image/shared/statsd-exporter AS statsd_exporter-builder -FROM stackable/image/vector AS opa-auth-manager-builder +FROM local-image/vector AS opa-auth-manager-builder ARG OPA_AUTH_MANAGER -ARG PYTHON -ARG UV +ARG PYTHON_VERSION +ARG UV_VERSION COPY airflow/opa-auth-manager/${OPA_AUTH_MANAGER} /tmp/opa-auth-manager @@ -23,10 +23,10 @@ WORKDIR /tmp/opa-auth-manager RUN < /stackable/app/airflow-${PRODUCT}.cdx.json +end)' /tmp/sbom.json > /stackable/app/airflow-${PRODUCT_VERSION}.cdx.json EOF COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter /stackable/statsd_exporter -COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter-${SHARED_STATSD_EXPORTER}.cdx.json /stackable/statsd_exporter-${SHARED_STATSD_EXPORTER}.cdx.json +COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter-${SHARED_STATSD_EXPORTER_VERSION}.cdx.json /stackable/statsd_exporter-${SHARED_STATSD_EXPORTER_VERSION}.cdx.json COPY --from=gitsync-image --chown=${STACKABLE_USER_UID}:0 /git-sync /stackable/git-sync RUN < /tmp/DRUID_SOURCE_DIR +RUN /stackable/patchable --images-repo-root=src checkout druid ${PRODUCT_VERSION} > /tmp/DRUID_SOURCE_DIR -RUN --mount=type=cache,id=maven-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository \ - --mount=type=cache,id=npm-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.npm \ - --mount=type=cache,id=cache-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.cache \ +RUN --mount=type=cache,id=maven-${PRODUCT_VERSION},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository \ + --mount=type=cache,id=npm-${PRODUCT_VERSION},uid=${STACKABLE_USER_UID},target=/stackable/.npm \ + --mount=type=cache,id=cache-${PRODUCT_VERSION},uid=${STACKABLE_USER_UID},target=/stackable/.cache \ < /stackable/package_manifest.txt +rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt chmod g=u /stackable/package_manifest.txt rm -rf /var/cache/yum -ln -sf /stackable/apache-druid-${PRODUCT}-stackable${RELEASE} /stackable/druid +ln -sf /stackable/apache-druid-${PRODUCT_VERSION}-stackable${RELEASE_VERSION} /stackable/druid chown -h ${STACKABLE_USER_UID}:0 stackable/druid # Force to overwrite the existing 'run-druid' @@ -159,7 +159,7 @@ chown -h ${STACKABLE_USER_UID}:0 /stackable/druid/bin/run-druid # fix missing permissions chmod -R g=u /stackable/bin -chmod g=u /stackable/apache-druid-${PRODUCT}-stackable${RELEASE} /stackable/druid-${PRODUCT}-stackable${RELEASE}-src.tar.gz +chmod g=u /stackable/apache-druid-${PRODUCT_VERSION}-stackable${RELEASE_VERSION} /stackable/druid-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-src.tar.gz EOF # ---------------------------------------- diff --git a/druid/boil-config.toml b/druid/boil-config.toml new file mode 100644 index 000000000..8a30bc28e --- /dev/null +++ b/druid/boil-config.toml @@ -0,0 +1,26 @@ +[versions."30.0.1".local-images] +# https://druid.apache.org/docs/30.0.1/operations/java/ +java-base = "17" +java-devel = "17" +"hadoop/hadoop" = "3.3.6" + +[versions."30.0.1".build-arguments] +authorizer-version = "0.7.0" + +[versions."31.0.1".local-images] +# https://druid.apache.org/docs/31.0.1/operations/java/ +java-base = "17" +java-devel = "17" +"hadoop/hadoop" = "3.3.6" + +[versions."31.0.1".build-arguments] +authorizer-version = "0.7.0" + +[versions."33.0.0".local-images] +# https://druid.apache.org/docs/33.0.0/operations/java/ +java-base = "17" +java-devel = "17" +"hadoop/hadoop" = "3.3.6" + +[versions."33.0.0".build-arguments] +authorizer-version = "0.7.0" diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index 5ca3cd6b1..5a8892fc3 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -1,19 +1,19 @@ # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 # check=error=true -FROM stackable/image/hadoop/hadoop AS hadoop-builder +FROM local-image/hadoop/hadoop AS hadoop-builder -FROM stackable/image/java-devel AS hdfs-utils-builder +FROM local-image/java-devel AS hdfs-utils-builder -ARG HDFS_UTILS -ARG PRODUCT -ARG RELEASE +ARG HDFS_UTILS_VERSION +ARG PRODUCT_VERSION +ARG RELEASE_VERSION ARG STACKABLE_USER_UID -ARG HADOOP_HADOOP +ARG HADOOP_HADOOP_VERSION # Reassign the arg to `HADOOP_VERSION` for better readability. -# It is passed as `HADOOP_HADOOP`, because versions.py has to contain `hadoop/hadoop` to establish a dependency on the Hadoop builder. -# The value of `hadoop/hadoop` is transformed by `bake` and automatically passed as `HADOOP_HADOOP` arg. -ENV HADOOP_VERSION=${HADOOP_HADOOP} +# It is passed as `HADOOP_HADOOP_VERSION`, because versions.py has to contain `hadoop/hadoop` to establish a dependency on the Hadoop builder. +# The value of `hadoop/hadoop` is transformed by `bake` and automatically passed as `HADOOP_HADOOP_VERSION` arg. +ENV HADOOP_VERSION=${HADOOP_HADOOP_VERSION} # Starting with hdfs-utils 0.4.0 we need to use Java 17 for compilation. # We can not simply use java-devel with Java 17, as it is also used to compile Hadoop in this @@ -33,7 +33,7 @@ USER ${STACKABLE_USER_UID} WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 hadoop/hdfs-utils/stackable/patches/patchable.toml /stackable/src/hadoop/hdfs-utils/stackable/patches/patchable.toml -COPY --chown=${STACKABLE_USER_UID}:0 hadoop/hdfs-utils/stackable/patches/${HDFS_UTILS} /stackable/src/hadoop/hdfs-utils/stackable/patches/${HDFS_UTILS} +COPY --chown=${STACKABLE_USER_UID}:0 hadoop/hdfs-utils/stackable/patches/${HDFS_UTILS_VERSION} /stackable/src/hadoop/hdfs-utils/stackable/patches/${HDFS_UTILS_VERSION} COPY --from=hadoop-builder --chown=${STACKABLE_USER_UID}:0 /stackable/patched-libs /stackable/patched-libs @@ -42,40 +42,40 @@ COPY --from=hadoop-builder --chown=${STACKABLE_USER_UID}:0 /stackable/patched-li # labels to build a rackID from. # Starting with hdfs-utils version 0.3.0 the topology provider is not a standalone jar anymore and included in hdfs-utils. RUN < /stackable/package_manifest.txt +rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt chmod g=u /stackable/package_manifest.txt rm -rf /var/cache/yum @@ -119,21 +119,21 @@ rm -rf /var/cache/yum # It is so non-root users (as we are) can mount a FUSE device and let other users access it echo "user_allow_other" > /etc/fuse.conf -ln -s "/stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE}" /stackable/hadoop +ln -s "/stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION}" /stackable/hadoop # async-profiler ARCH="${TARGETARCH/amd64/x64}" -curl "https://repo.stackable.tech/repository/packages/async-profiler/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}.tar.gz" | tar -xzC /stackable -ln -s "/stackable/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}" /stackable/async-profiler +curl "https://repo.stackable.tech/repository/packages/async-profiler/async-profiler-${ASYNC_PROFILER_VERSION}-${TARGETOS}-${ARCH}.tar.gz" | tar -xzC /stackable +ln -s "/stackable/async-profiler-${ASYNC_PROFILER_VERSION}-${TARGETOS}-${ARCH}" /stackable/async-profiler # JMX Exporter -curl "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" -o "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" -chmod -x "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" -ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar +curl "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" -o "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" +chmod -x "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" +ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar # Set correct permissions and ownerships -chown --recursive ${STACKABLE_USER_UID}:0 /stackable/hadoop /stackable/jmx /stackable/async-profiler "/stackable/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}" -chmod --recursive g=u /stackable/jmx /stackable/async-profiler "/stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE}" +chown --recursive ${STACKABLE_USER_UID}:0 /stackable/hadoop /stackable/jmx /stackable/async-profiler "/stackable/async-profiler-${ASYNC_PROFILER_VERSION}-${TARGETOS}-${ARCH}" +chmod --recursive g=u /stackable/jmx /stackable/async-profiler "/stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION}" # Workaround for https://issues.apache.org/jira/browse/HADOOP-12845 # The problem is that our stackable-devel image does contain the openssl-devel package diff --git a/hadoop/boil-config.toml b/hadoop/boil-config.toml new file mode 100644 index 000000000..b0d0641f2 --- /dev/null +++ b/hadoop/boil-config.toml @@ -0,0 +1,22 @@ +# Not part of SDP 25.7.0, but still required for hbase, hive, spark-k8s +[versions."3.3.6".local-images] +"hadoop/hadoop" = "3.3.6" +java-base = "11" +java-devel = "11" + +[versions."3.3.6".build-arguments] +async-profiler-version = "2.9" +jmx-exporter-version = "1.3.0" +protobuf-version = "3.7.1" +hdfs-utils-version = "0.4.0" + +[versions."3.4.1".local-images] +"hadoop/hadoop" = "3.4.1" +java-base = "11" +java-devel = "11" + +[versions."3.4.1".build-arguments] +async-profiler-version = "2.9" +jmx-exporter-version = "1.3.0" +protobuf-version = "3.7.1" +hdfs-utils-version = "0.4.1" diff --git a/hadoop/hadoop/Dockerfile b/hadoop/hadoop/Dockerfile index 342eef556..a17feec73 100644 --- a/hadoop/hadoop/Dockerfile +++ b/hadoop/hadoop/Dockerfile @@ -1,17 +1,17 @@ # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 # check=error=true -FROM stackable/image/java-devel AS hadoop-builder +FROM local-image/java-devel AS hadoop-builder -ARG PRODUCT -ARG RELEASE -ARG PROTOBUF +ARG PRODUCT_VERSION +ARG RELEASE_VERSION +ARG PROTOBUF_VERSION ARG STACKABLE_USER_UID WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/patchable.toml /stackable/src/shared/protobuf/stackable/patches/patchable.toml -COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/${PROTOBUF} /stackable/src/shared/protobuf/stackable/patches/${PROTOBUF} +COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/${PROTOBUF_VERSION} /stackable/src/shared/protobuf/stackable/patches/${PROTOBUF_VERSION} RUN < /stackable/bin/export-snapshot-to-s3 +envsubst '${HBASE_VERSION}:${RELEASE_VERSION}:${LIBS}' < /stackable/bin/export-snapshot-to-s3.env > /stackable/bin/export-snapshot-to-s3 chmod +x /stackable/bin/export-snapshot-to-s3 rm /stackable/bin/export-snapshot-to-s3.env @@ -58,35 +58,35 @@ chmod --recursive g=u /stackable EOF # Final Image -FROM stackable/image/java-base AS final +FROM local-image/java-base AS final -ARG PRODUCT -ARG RELEASE -ARG HADOOP_HADOOP +ARG PRODUCT_VERSION +ARG RELEASE_VERSION +ARG HADOOP_HADOOP_VERSION # Reassign the arg to `HADOOP_VERSION` for better readability. -ENV HADOOP_VERSION=${HADOOP_HADOOP} +ENV HADOOP_VERSION=${HADOOP_HADOOP_VERSION} ARG HBASE_PROFILE -ARG HBASE_HBASE +ARG HBASE_HBASE_VERSION # Reassign the arg to `HBASE_VERSION` for better readability. -ENV HBASE_VERSION=${HBASE_HBASE} -ARG HBASE_HBASE_OPERATOR_TOOLS -ARG HBASE_HBASE_OPA_AUTHORIZER -ARG HBASE_PHOENIX +ENV HBASE_VERSION=${HBASE_HBASE_VERSION} +ARG HBASE_HBASE_OPERATOR_TOOLS_VERSION +ARG HBASE_HBASE_OPA_AUTHORIZER_VERSION +ARG HBASE_PHOENIX_VERSION ARG STACKABLE_USER_UID ARG NAME="Apache HBase" ARG DESCRIPTION="This image is deployed by the Stackable Operator for Apache HBase" LABEL name="${NAME}" -LABEL version="${PRODUCT}" -LABEL release="${RELEASE}" +LABEL version="${PRODUCT_VERSION}" +LABEL release="${RELEASE_VERSION}" LABEL summary="The Stackable image for Apache HBase" LABEL description="${DESCRIPTION}" # https://github.com/opencontainers/image-spec/blob/036563a4a268d7c08b51a08f05a02a0fe74c7268/annotations.md#annotations LABEL org.opencontainers.image.documentation="https://docs.stackable.tech/home/stable/hbase/" -LABEL org.opencontainers.image.version="${PRODUCT}" -LABEL org.opencontainers.image.revision="${RELEASE}" +LABEL org.opencontainers.image.version="${PRODUCT_VERSION}" +LABEL org.opencontainers.image.revision="${RELEASE_VERSION}" LABEL org.opencontainers.image.title="${NAME}" LABEL org.opencontainers.image.description="${DESCRIPTION}" @@ -96,17 +96,17 @@ LABEL io.openshift.tags="ubi9,stackable,hbase,sdp,nosql" LABEL io.k8s.description="${DESCRIPTION}" LABEL io.k8s.display-name="${NAME}" -COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE} /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE}/ -COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE}-src.tar.gz /stackable +COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE_VERSION} /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE_VERSION}/ +COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE_VERSION}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/async-profiler /stackable/async-profiler/ -COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS}-stackable${RELEASE} /stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS}-stackable${RELEASE}/ -COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS}-stackable${RELEASE}-src.tar.gz /stackable +COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS_VERSION}-stackable${RELEASE_VERSION} /stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS_VERSION}-stackable${RELEASE_VERSION}/ +COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS_VERSION}-stackable${RELEASE_VERSION}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/bin/hbck2 /stackable/bin/hbck2 -COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/bin/hbase-entrypoint.sh /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE}/bin/hbase-entrypoint.sh +COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/bin/hbase-entrypoint.sh /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE_VERSION}/bin/hbase-entrypoint.sh COPY --chown=${STACKABLE_USER_UID}:0 --from=phoenix /stackable/phoenix /stackable/phoenix/ -COPY --chown=${STACKABLE_USER_UID}:0 --from=phoenix /stackable/phoenix-${HBASE_PHOENIX}-stackable${RELEASE}-src.tar.gz /stackable +COPY --chown=${STACKABLE_USER_UID}:0 --from=phoenix /stackable/phoenix-${HBASE_PHOENIX_VERSION}-stackable${RELEASE_VERSION}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-s3-builder /stackable/bin/export-snapshot-to-s3 /stackable/bin/export-snapshot-to-s3 COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-s3-builder /stackable/hadoop/share/hadoop/tools/lib/ /stackable/hadoop/share/hadoop/tools/lib/ @@ -116,11 +116,11 @@ COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-s3-builder /stackable/hadoop/ # hadoop-azure-${HADOOP}.jar contains the AzureBlobFileSystem which is required # by hadoop-common-${HADOOP}.jar if the scheme of a file system is "abfs://". COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder \ - /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP_VERSION}-stackable${RELEASE}.jar \ - /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE}/lib/ + /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP_VERSION}-stackable${RELEASE_VERSION}.jar \ + /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE_VERSION}/lib/ -COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-opa-authorizer /stackable/hbase-opa-authorizer-${HBASE_HBASE_OPA_AUTHORIZER}-src.tar.gz /stackable -COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-opa-authorizer /stackable/hbase-opa-authorizer/target/hbase-opa-authorizer*.jar /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE}/lib +COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-opa-authorizer /stackable/hbase-opa-authorizer-${HBASE_HBASE_OPA_AUTHORIZER_VERSION}-src.tar.gz /stackable +COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-opa-authorizer /stackable/hbase-opa-authorizer/target/hbase-opa-authorizer*.jar /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE_VERSION}/lib RUN < /stackable/package_manifest.txt +rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt chmod g=u /stackable/package_manifest.txt rm -rf /var/cache/yum -ln --symbolic --logical --verbose "/stackable/hbase-${HBASE_VERSION}-stackable${RELEASE}" /stackable/hbase +ln --symbolic --logical --verbose "/stackable/hbase-${HBASE_VERSION}-stackable${RELEASE_VERSION}" /stackable/hbase chown --no-dereference ${STACKABLE_USER_UID}:0 /stackable/hbase chmod g=u /stackable/hbase -ln --symbolic --logical --verbose "/stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS}-stackable${RELEASE}" /stackable/hbase-operator-tools +ln --symbolic --logical --verbose "/stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS_VERSION}-stackable${RELEASE_VERSION}" /stackable/hbase-operator-tools chown --no-dereference ${STACKABLE_USER_UID}:0 /stackable/hbase-operator-tools chmod g=u /stackable/hbase-operator-tools diff --git a/hbase/boil-config.toml b/hbase/boil-config.toml new file mode 100644 index 000000000..2aa8bd793 --- /dev/null +++ b/hbase/boil-config.toml @@ -0,0 +1,25 @@ +[versions."2.6.1".local-images] +"hbase/hbase" = "2.6.1" +"hbase/hbase-operator-tools" = "1.3.0-fd5a5fb-hbase2.6.1" +"hbase/phoenix" = "5.2.1-hbase2.6.1" +"hbase/hbase-opa-authorizer" = "0.1.0" # only for HBase 2.6.1 +"hadoop/hadoop" = "3.3.6" +java-base = "11" +java-devel = "11" + +[versions."2.6.1".build-arguments] +hbase-profile = "2.6" +delete-caches = "true" + +[versions."2.6.2".local-images] +"hbase/hbase" = "2.6.2" +"hbase/hbase-operator-tools" = "1.3.0-fd5a5fb-hbase2.6.2" +"hbase/phoenix" = "5.2.1-hbase2.6.2" +"hbase/hbase-opa-authorizer" = "0.1.0" # only for HBase 2.6.1 +"hadoop/hadoop" = "3.4.1" +java-base = "11" +java-devel = "11" + +[versions."2.6.2".build-arguments] +hbase-profile = "2.6" +delete-caches = "true" diff --git a/hbase/hbase-opa-authorizer/Dockerfile b/hbase/hbase-opa-authorizer/Dockerfile index 78f5a7115..c43e4990a 100644 --- a/hbase/hbase-opa-authorizer/Dockerfile +++ b/hbase/hbase-opa-authorizer/Dockerfile @@ -1,6 +1,6 @@ -FROM stackable/image/java-devel +FROM local-image/java-devel -ARG PRODUCT +ARG PRODUCT_VERSION ARG DELETE_CACHES ARG STACKABLE_USER_UID @@ -8,17 +8,17 @@ USER ${STACKABLE_USER_UID} WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-opa-authorizer/stackable/patches/patchable.toml /stackable/src/hbase/hbase-opa-authorizer/stackable/patches/patchable.toml -COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-opa-authorizer/stackable/patches/${PRODUCT} /stackable/src/hbase/hbase-opa-authorizer/stackable/patches/${PRODUCT} +COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-opa-authorizer/stackable/patches/${PRODUCT_VERSION} /stackable/src/hbase/hbase-opa-authorizer/stackable/patches/${PRODUCT_VERSION} RUN --mount=type=cache,id=maven-opa,uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository < /stackable/package_manifest.txt +rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt chmod g=u /stackable/package_manifest.txt rm -rf /var/cache/yum -chmod g=u /stackable/apache-hive-metastore-${PRODUCT}-stackable${RELEASE}-bin/bin/start-metastore +chmod g=u /stackable/apache-hive-metastore-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-bin/bin/start-metastore -ln -s /stackable/apache-hive-metastore-${PRODUCT}-stackable${RELEASE}-bin /stackable/hive-metastore +ln -s /stackable/apache-hive-metastore-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-bin /stackable/hive-metastore chown -h ${STACKABLE_USER_UID}:0 /stackable/hive-metastore chmod g=u /stackable/hive-metastore -ln -s /stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE} /stackable/hadoop +ln -s /stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION} /stackable/hadoop chown -h ${STACKABLE_USER_UID}:0 /stackable/hadoop chmod g=u /stackable/hadoop chmod g=u /stackable/*-src.tar.gz diff --git a/hive/boil-config.toml b/hive/boil-config.toml new file mode 100644 index 000000000..7bbcca500 --- /dev/null +++ b/hive/boil-config.toml @@ -0,0 +1,38 @@ +[versions."3.1.3".local-images] +# Hive 3 must be built with Java 8 but will run on Java 11 +java-base = "11" +java-devel = "8" +"hadoop/hadoop" = "3.3.6" + +[versions."3.1.3".build-arguments] +jmx-exporter-version = "1.3.0" +# Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 +aws-java-sdk-bundle-version = "1.12.367" +azure-storage-version = "7.0.1" +azure-keyvault-core-version = "1.0.0" + +[versions."4.0.0".local-images] +# Hive 4 must be built with Java 8 (according to GitHub README) but seems to run on Java 11 +java-base = "11" +java-devel = "8" +"hadoop/hadoop" = "3.3.6" + +[versions."4.0.0".build-arguments] +jmx-exporter-version = "1.3.0" +# Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 +aws-java-sdk-bundle-version = "1.12.367" +azure-storage-version = "7.0.1" +azure-keyvault-core-version = "1.0.0" + +[versions."4.0.1".local-images] +# Hive 4 must be built with Java 8 (according to GitHub README) but seems to run on Java 11 +java-base = "11" +java-devel = "8" +"hadoop/hadoop" = "3.3.6" + +[versions."4.0.1".build-arguments] +jmx-exporter-version = "1.3.0" +# Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 +aws-java-sdk-bundle-version = "1.12.367" +azure-storage-version = "7.0.1" +azure-keyvault-core-version = "1.0.0" diff --git a/java-base/Dockerfile b/java-base/Dockerfile index 397587d2f..610eb5a46 100644 --- a/java-base/Dockerfile +++ b/java-base/Dockerfile @@ -4,16 +4,16 @@ # # Provides the common Java Runtime for SDP products # -FROM stackable/image/vector +FROM local-image/vector -ARG PRODUCT -ARG RELEASE="1" +ARG PRODUCT_VERSION +ARG RELEASE_VERSION="1" LABEL name="Stackable image for OpenJDK" \ maintainer="info@stackable.tech" \ vendor="Stackable GmbH" \ - version="${PRODUCT}" \ - release="${RELEASE}" \ + version="${PRODUCT_VERSION}" \ + release="${RELEASE_VERSION}" \ summary="The Stackable OpenJDK base image." \ description="This image is the base image for all Stackable Java product images." @@ -32,7 +32,7 @@ EOF RUN microdnf update && \ microdnf install \ # Needed to run Java programs - "temurin-${PRODUCT}-jre" \ + "temurin-${PRODUCT_VERSION}-jre" \ # Needed, because otherwise e.g. Zookeeper fails with # Caused by: java.io.FileNotFoundException: /usr/lib/jvm/java-11-openjdk-11.0.20.0.8-2.el8.x86_64/lib/tzdb.dat (No such file or directory) tzdata-java \ @@ -43,7 +43,7 @@ RUN microdnf update && \ COPY java-base/licenses /licenses -ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT}-jre" +ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT_VERSION}-jre" # This image doesn't include the development packages for Java. # For images that need the devel package (ex. Spark) use this env variable to @@ -51,7 +51,7 @@ ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT}-jre" # # microdnf install java-${JAVA_VERSION}-openjdk-devel # -ENV JAVA_VERSION=$PRODUCT +ENV JAVA_VERSION=$PRODUCT_VERSION # Mitigation for CVE-2021-44228 (Log4Shell) # This variable is supported as of Log4j version 2.10 and diff --git a/java-base/boil-config.toml b/java-base/boil-config.toml new file mode 100644 index 000000000..c7968ebf0 --- /dev/null +++ b/java-base/boil-config.toml @@ -0,0 +1,20 @@ +[versions."8".local-images] +vector = "0.47.0" + +[versions."11".local-images] +vector = "0.47.0" + +[versions."17".local-images] +vector = "0.47.0" + +[versions."21".local-images] +vector = "0.47.0" + +[versions."22".local-images] +vector = "0.47.0" + +[versions."23".local-images] +vector = "0.47.0" + +[versions."24".local-images] +vector = "0.47.0" diff --git a/java-devel/Dockerfile b/java-devel/Dockerfile index 8a4343aed..b88e26498 100644 --- a/java-devel/Dockerfile +++ b/java-devel/Dockerfile @@ -5,9 +5,9 @@ # Base image for builder stages in Java based products # -FROM stackable/image/stackable-devel +FROM local-image/stackable-devel -ARG PRODUCT +ARG PRODUCT_VERSION ARG STACKABLE_USER_UID # Find the latest version here: https://github.com/apache/maven @@ -46,7 +46,7 @@ microdnf install \ `# Needed by the maven ant run plugin for the "set-hostname-property" step in zookeeper` \ hostname \ `# Needed for compiling Java projects` \ - "temurin-${PRODUCT}-jdk" \ + "temurin-${PRODUCT_VERSION}-jdk" \ krb5-devel \ libcurl-devel \ make \ @@ -66,7 +66,7 @@ ln -s /opt/maven/bin/mvn /usr/bin/mvn EOF -ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT}-jdk" +ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT_VERSION}-jdk" ENV MAVEN_ARGS="--batch-mode --no-transfer-progress" COPY --chown=${STACKABLE_USER_UID}:0 java-devel/stackable/settings.xml /stackable/.m2/settings.xml diff --git a/java-devel/boil-config.toml b/java-devel/boil-config.toml new file mode 100644 index 000000000..b6d750b42 --- /dev/null +++ b/java-devel/boil-config.toml @@ -0,0 +1,20 @@ +[versions."8".local-images] +stackable-devel = "1.0.0" + +[versions."11".local-images] +stackable-devel = "1.0.0" + +[versions."17".local-images] +stackable-devel = "1.0.0" + +[versions."21".local-images] +stackable-devel = "1.0.0" + +[versions."22".local-images] +stackable-devel = "1.0.0" + +[versions."23".local-images] +stackable-devel = "1.0.0" + +[versions."24".local-images] +stackable-devel = "1.0.0" diff --git a/jdk-base/Dockerfile b/jdk-base/Dockerfile index 776804c1d..914ab6cf9 100644 --- a/jdk-base/Dockerfile +++ b/jdk-base/Dockerfile @@ -4,16 +4,16 @@ # # Provides the common Java Development Kit for SDP products # -FROM stackable/image/vector +FROM local-image/vector -ARG PRODUCT -ARG RELEASE="1" +ARG PRODUCT_VERSION +ARG RELEASE_VERSION="1" LABEL name="Stackable image for OpenJDK" \ maintainer="info@stackable.tech" \ vendor="Stackable GmbH" \ - version="${PRODUCT}" \ - release="${RELEASE}" \ + version="${PRODUCT_VERSION}" \ + release="${RELEASE_VERSION}" \ summary="The Stackable OpenJDK base image." \ description="This image is the base image for all Stackable Java product images which require a JDK." @@ -32,7 +32,7 @@ EOF RUN microdnf update && \ microdnf install \ # Needed to run Java programs - "temurin-${PRODUCT}-jdk" \ + "temurin-${PRODUCT_VERSION}-jdk" \ # Needed, because otherwise e.g. Zookeeper fails with # Caused by: java.io.FileNotFoundException: /usr/lib/jvm/java-11-openjdk-11.0.20.0.8-2.el8.x86_64/lib/tzdb.dat (No such file or directory) tzdata-java \ @@ -43,7 +43,7 @@ RUN microdnf update && \ COPY java-base/licenses /licenses -ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT}-jdk" +ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT_VERSION}-jdk" # This image doesn't include the development packages for Java. # For images that need the devel package (ex. Spark) use this env variable to @@ -51,7 +51,7 @@ ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT}-jdk" # # microdnf install java-${JAVA_VERSION}-openjdk-devel # -ENV JAVA_VERSION=$PRODUCT +ENV JAVA_VERSION=$PRODUCT_VERSION # Mitigation for CVE-2021-44228 (Log4Shell) # This variable is supported as of Log4j version 2.10 and diff --git a/jdk-base/boil-config.toml b/jdk-base/boil-config.toml new file mode 100644 index 000000000..c7968ebf0 --- /dev/null +++ b/jdk-base/boil-config.toml @@ -0,0 +1,20 @@ +[versions."8".local-images] +vector = "0.47.0" + +[versions."11".local-images] +vector = "0.47.0" + +[versions."17".local-images] +vector = "0.47.0" + +[versions."21".local-images] +vector = "0.47.0" + +[versions."22".local-images] +vector = "0.47.0" + +[versions."23".local-images] +vector = "0.47.0" + +[versions."24".local-images] +vector = "0.47.0" diff --git a/kafka-testing-tools/Dockerfile b/kafka-testing-tools/Dockerfile index cbfb4299c..7f1e563d7 100644 --- a/kafka-testing-tools/Dockerfile +++ b/kafka-testing-tools/Dockerfile @@ -1,20 +1,20 @@ # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 # check=error=true -FROM stackable/image/kafka/kcat AS kcat +FROM local-image/kafka/kcat AS kcat -FROM stackable/image/stackable-base AS final +FROM local-image/stackable-base AS final -ARG PRODUCT -ARG KAFKA_KCAT -ARG RELEASE +ARG PRODUCT_VERSION +ARG KAFKA_KCAT_VERSION +ARG RELEASE_VERSION ARG STACKABLE_USER_UID LABEL name="Kafka Testing Tools" \ maintainer="info@stackable.tech" \ vendor="Stackable GmbH" \ - version="${PRODUCT}" \ - release="${RELEASE}" \ + version="${PRODUCT_VERSION}" \ + release="${RELEASE_VERSION}" \ summary="The Stackable image for the kcat tool." \ description="Used for integration testing" @@ -30,9 +30,9 @@ RUN microdnf install \ && rm -rf /var/cache/yum # Store kcat version with binary name and add softlink -COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat /stackable/kcat-${KAFKA_KCAT} -COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KAFKA_KCAT}-src.tar.gz /stackable -RUN ln -s /stackable/kcat-${KAFKA_KCAT} /stackable/kcat +COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat /stackable/kcat-${KAFKA_KCAT_VERSION} +COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KAFKA_KCAT_VERSION}-src.tar.gz /stackable +RUN ln -s /stackable/kcat-${KAFKA_KCAT_VERSION} /stackable/kcat COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /licenses /licenses COPY --chown=${STACKABLE_USER_UID}:0 kafka-testing-tools/licenses /licenses diff --git a/kafka-testing-tools/boil-config.toml b/kafka-testing-tools/boil-config.toml new file mode 100644 index 000000000..ca78f8116 --- /dev/null +++ b/kafka-testing-tools/boil-config.toml @@ -0,0 +1,4 @@ +[versions."1.0.0".local-images] +stackable-base = "1.0.0" +"kafka/kcat" = "1.7.0" +java-base = "11" diff --git a/kafka/Dockerfile b/kafka/Dockerfile index e01f09fda..f9dffb9dd 100644 --- a/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -1,15 +1,15 @@ # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 # check=error=true -FROM stackable/image/kafka/kcat AS kcat -FROM stackable/image/kafka/kafka-opa-plugin AS kafka-opa-plugin +FROM local-image/kafka/kcat AS kcat +FROM local-image/kafka/kafka-opa-plugin AS kafka-opa-plugin -FROM stackable/image/java-devel AS kafka-builder +FROM local-image/java-devel AS kafka-builder -ARG PRODUCT -ARG RELEASE -ARG SCALA -ARG JMX_EXPORTER +ARG PRODUCT_VERSION +ARG RELEASE_VERSION +ARG SCALA_VERSION +ARG JMX_EXPORTER_VERSION ARG STACKABLE_USER_UID USER ${STACKABLE_USER_UID} @@ -17,15 +17,15 @@ WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/jmx/ /stackable/jmx/ COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/patchable.toml /stackable/src/kafka/stackable/patches/patchable.toml -COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/${PRODUCT} /stackable/src/kafka/stackable/patches/${PRODUCT} +COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/${PRODUCT_VERSION} /stackable/src/kafka/stackable/patches/${PRODUCT_VERSION} RUN < /stackable/package_manifest.txt +rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt chmod g=u /stackable/package_manifest.txt rm -rf /var/cache/yum -ln -s /stackable/bin/kcat-${KAFKA_KCAT} /stackable/bin/kcat +ln -s /stackable/bin/kcat-${KAFKA_KCAT_VERSION} /stackable/bin/kcat chown -h ${STACKABLE_USER_UID}:0 /stackable/bin/kcat # kcat was located in /stackable/kcat - legacy ln -s /stackable/bin/kcat /stackable/kcat chown -h ${STACKABLE_USER_UID}:0 /stackable/kcat -ln -s /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE} /stackable/kafka +ln -s /stackable/kafka_${SCALA_VERSION}-${PRODUCT_VERSION}-stackable${RELEASE_VERSION} /stackable/kafka chown -h ${STACKABLE_USER_UID}:0 /stackable/kafka # fix missing permissions chmod g=u /stackable/bin chmod g=u /stackable/jmx -chmod g=u /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE} -chmod g=u /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE}/libs/opa-authorizer-${KAFKA_KAFKA_OPA_PLUGIN}-all.jar +chmod g=u /stackable/kafka_${SCALA_VERSION}-${PRODUCT_VERSION}-stackable${RELEASE_VERSION} +chmod g=u /stackable/kafka_${SCALA_VERSION}-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/libs/opa-authorizer-${KAFKA_KAFKA_OPA_PLUGIN_VERSION}-all.jar chmod g=u /stackable/*-src.tar.gz EOF diff --git a/kafka/boil-config.toml b/kafka/boil-config.toml new file mode 100644 index 000000000..5347e10e4 --- /dev/null +++ b/kafka/boil-config.toml @@ -0,0 +1,39 @@ +[versions."3.7.2".local-images] +java-base = "21" +java-devel = "21" +"kafka/kcat" = "1.7.0" +"kafka/kafka-opa-plugin" = "1.5.1" + +[versions."3.7.2".build-arguments] +scala-version = "2.13" +jmx-exporter-version = "1.3.0" + +[versions."3.9.0".local-images] +java-base = "21" +java-devel = "21" +"kafka/kcat" = "1.7.0" +"kafka/kafka-opa-plugin" = "1.5.1" + +[versions."3.9.0".build-arguments] +scala-version = "2.13" +jmx-exporter-version = "1.3.0" + +[versions."3.9.1".local-images] +java-base = "21" +java-devel = "21" +"kafka/kcat" = "1.7.0" +"kafka/kafka-opa-plugin" = "1.5.1" + +[versions."3.9.1".build-arguments] +scala-version = "2.13" +jmx-exporter-version = "1.3.0" + +[versions."4.0.0".local-images] +java-base = "21" +java-devel = "21" +"kafka/kcat" = "1.7.0" +"kafka/kafka-opa-plugin" = "1.5.1" + +[versions."4.0.0".build-arguments] +scala-version = "2.13" +jmx-exporter-version = "1.3.0" diff --git a/kafka/kafka-opa-plugin/Dockerfile b/kafka/kafka-opa-plugin/Dockerfile index 2a5df3166..f963a2cb3 100644 --- a/kafka/kafka-opa-plugin/Dockerfile +++ b/kafka/kafka-opa-plugin/Dockerfile @@ -1,22 +1,22 @@ # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 # check=error=true -FROM stackable/image/java-devel +FROM local-image/java-devel -ARG PRODUCT +ARG PRODUCT_VERSION ARG STACKABLE_USER_UID USER ${STACKABLE_USER_UID} WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 kafka/kafka-opa-plugin/stackable/patches/patchable.toml /stackable/src/kafka/kafka-opa-plugin/stackable/patches/patchable.toml -COPY --chown=${STACKABLE_USER_UID}:0 kafka/kafka-opa-plugin/stackable/patches/${PRODUCT} /stackable/src/kafka/kafka-opa-plugin/stackable/patches/${PRODUCT} +COPY --chown=${STACKABLE_USER_UID}:0 kafka/kafka-opa-plugin/stackable/patches/${PRODUCT_VERSION} /stackable/src/kafka/kafka-opa-plugin/stackable/patches/${PRODUCT_VERSION} RUN <= 1.23.1) go install github.com/CycloneDX/cyclonedx-gomod/cmd/cyclonedx-gomod@v1.7.0 -cd "$(/stackable/patchable --images-repo-root=src checkout opa ${PRODUCT})" +cd "$(/stackable/patchable --images-repo-root=src checkout opa ${PRODUCT_VERSION})" -ORIGINAL_VERSION=${PRODUCT} -NEW_VERSION="${PRODUCT}-stackable${RELEASE}" +ORIGINAL_VERSION=${PRODUCT_VERSION} +NEW_VERSION="${PRODUCT_VERSION}-stackable${RELEASE_VERSION}" sed -i 's/var Version = "'${ORIGINAL_VERSION}'"/var Version = "'${NEW_VERSION}'"/g' v1/version/version.go @@ -102,24 +100,24 @@ mv opa /stackable/opa/ chmod -R g=u /stackable/opa /stackable/opa-${NEW_VERSION}-src.tar.gz EOF -FROM stackable/image/vector +FROM local-image/vector -ARG PRODUCT -ARG RELEASE +ARG PRODUCT_VERSION +ARG RELEASE_VERSION ARG STACKABLE_USER_UID LABEL name="Open Policy Agent" \ maintainer="info@stackable.tech" \ vendor="Stackable GmbH" \ - version="${PRODUCT}" \ - release="${RELEASE}" \ + version="${PRODUCT_VERSION}" \ + release="${RELEASE_VERSION}" \ summary="The Stackable image for Open Policy Agent (OPA)." \ description="This image is deployed by the Stackable Operator for OPA." COPY --chown=${STACKABLE_USER_UID}:0 opa/licenses /licenses COPY --from=opa-builder --chown=${STACKABLE_USER_UID}:0 /stackable/opa /stackable/opa -COPY --from=opa-builder --chown=${STACKABLE_USER_UID}:0 /stackable/opa-${PRODUCT}-stackable${RELEASE}-src.tar.gz /stackable/opa-${PRODUCT}-stackable${RELEASE}-src.tar.gz +COPY --from=opa-builder --chown=${STACKABLE_USER_UID}:0 /stackable/opa-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-src.tar.gz /stackable/opa-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-src.tar.gz COPY --from=multilog-builder --chown=${STACKABLE_USER_UID}:0 /daemontools/admin/daemontools/command/multilog /stackable/multilog RUN <= 1.23.1) go install github.com/CycloneDX/cyclonedx-gomod/cmd/cyclonedx-gomod@v1.7.0 -curl "https://repo.stackable.tech/repository/packages/statsd_exporter/statsd_exporter-${PRODUCT}.src.tar.gz" | tar -xzC . +curl "https://repo.stackable.tech/repository/packages/statsd_exporter/statsd_exporter-${PRODUCT_VERSION}.src.tar.gz" | tar -xzC . ( - cd "statsd_exporter-${PRODUCT}" || exit + cd "statsd_exporter-${PRODUCT_VERSION}" || exit # Unfortunately, we need to create a dummy Git repository to allow cyclonedx-gomod to determine the version of statsd_exporter git init git config user.email "fake.commiter@stackable.tech" git config user.name "Fake commiter" git commit --allow-empty --message "Fake commit, so that we can create a tag" - git tag "v${PRODUCT}" + git tag "v${PRODUCT_VERSION}" go build -o ../statsd_exporter - $GOPATH/bin/cyclonedx-gomod app -json -output-version 1.5 -output ../statsd_exporter-${PRODUCT}.cdx.json -packages -files + $GOPATH/bin/cyclonedx-gomod app -json -output-version 1.5 -output ../statsd_exporter-${PRODUCT_VERSION}.cdx.json -packages -files ) -rm -rf "statsd_exporter-${PRODUCT}" +rm -rf "statsd_exporter-${PRODUCT_VERSION}" EOF diff --git a/shared/statsd-exporter/boil-config.toml b/shared/statsd-exporter/boil-config.toml new file mode 100644 index 000000000..c2f9afcb2 --- /dev/null +++ b/shared/statsd-exporter/boil-config.toml @@ -0,0 +1,2 @@ +[versions."0.28.0".local-images] +stackable-base = "1.0.0" diff --git a/spark-connect-client/Dockerfile b/spark-connect-client/Dockerfile index 4750a071e..fd2c9e950 100644 --- a/spark-connect-client/Dockerfile +++ b/spark-connect-client/Dockerfile @@ -1,18 +1,18 @@ # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 # spark-builder: provides client libs for spark-connect -FROM stackable/image/spark-k8s AS spark-builder +FROM local-image/spark-k8s AS spark-builder -ARG PRODUCT -ARG PYTHON -ARG RELEASE +ARG PRODUCT_VERSION +ARG PYTHON_VERSION +ARG RELEASE_VERSION ARG STACKABLE_USER_UID LABEL name="Stackable Spark Connect Examples" \ maintainer="info@stackable.tech" \ vendor="Stackable GmbH" \ - version="${PRODUCT}" \ - release="${RELEASE}" \ + version="${PRODUCT_VERSION}" \ + release="${RELEASE_VERSION}" \ summary="Spark Connect Examples" \ description="Spark Connect client libraries for Python and the JVM, including some examples." @@ -26,7 +26,7 @@ RUN < /stackable/package_manifest.txt +rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt chmod g=u /stackable/package_manifest.txt rm -rf /var/cache/yum -# Add link pointing from /stackable/zookeeper to /stackable/apache-zookeeper-${PRODUCT}-stackable${RELEASE}-bin/ +# Add link pointing from /stackable/zookeeper to /stackable/apache-zookeeper-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-bin/ # to preserve the folder name with the version. -ln -s /stackable/apache-zookeeper-${PRODUCT}-stackable${RELEASE}-bin/ /stackable/zookeeper +ln -s /stackable/apache-zookeeper-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-bin/ /stackable/zookeeper chown -h ${STACKABLE_USER_UID}:0 /stackable/zookeeper # fix missing permissions chmod g=u /stackable/jmx -chmod g=u /stackable/apache-zookeeper-${PRODUCT}-stackable${RELEASE}-bin/ +chmod g=u /stackable/apache-zookeeper-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-bin/ EOF # ---------------------------------------- diff --git a/zookeeper/boil-config.toml b/zookeeper/boil-config.toml new file mode 100644 index 000000000..64ab0416d --- /dev/null +++ b/zookeeper/boil-config.toml @@ -0,0 +1,6 @@ +[versions."3.9.3".local-images] +java-base = "17" +java-devel = "11" + +[versions."3.9.3".build-arguments] +jmx-exporter-version = "1.3.0" From 19738fd5434f3b558cdcc7ac1b792a9cb8849d92 Mon Sep 17 00:00:00 2001 From: Techassi Date: Tue, 12 Aug 2025 15:48:34 +0200 Subject: [PATCH 03/51] chore: Remove old Python config files --- airflow/versions.py | 54 ----------- conf.py | 128 ------------------------- druid/versions.py | 26 ----- hadoop/hadoop/versions.py | 13 --- hadoop/versions.py | 21 ---- hbase/hbase-opa-authorizer/versions.py | 7 -- hbase/hbase-operator-tools/versions.py | 20 ---- hbase/hbase/versions.py | 20 ---- hbase/phoenix/versions.py | 20 ---- hbase/versions.py | 28 ------ hive/versions.py | 38 -------- java-base/versions.py | 30 ------ java-devel/versions.py | 30 ------ jdk-base/versions.py | 30 ------ kafka-testing-tools/versions.py | 8 -- kafka/kafka-opa-plugin/versions.py | 6 -- kafka/kcat/versions.py | 7 -- kafka/versions.py | 38 -------- krb5/versions.py | 5 - nifi/versions.py | 24 ----- omid/versions.py | 14 --- opa/versions.py | 14 --- opensearch/security-plugin/versions.py | 6 -- opensearch/versions.py | 8 -- shared/statsd-exporter/versions.py | 6 -- spark-connect-client/versions.py | 8 -- spark-k8s/versions.py | 38 -------- stackable-base/versions.py | 6 -- stackable-devel/versions.py | 5 - superset/versions.py | 32 ------- testing-tools/versions.py | 6 -- tools/versions.py | 9 -- trino-cli/versions.py | 8 -- trino/storage-connector/versions.py | 17 ---- trino/trino/versions.py | 14 --- trino/versions.py | 23 ----- vector/versions.py | 8 -- zookeeper/versions.py | 12 --- 38 files changed, 787 deletions(-) delete mode 100644 airflow/versions.py delete mode 100644 conf.py delete mode 100644 druid/versions.py delete mode 100644 hadoop/hadoop/versions.py delete mode 100644 hadoop/versions.py delete mode 100644 hbase/hbase-opa-authorizer/versions.py delete mode 100644 hbase/hbase-operator-tools/versions.py delete mode 100644 hbase/hbase/versions.py delete mode 100644 hbase/phoenix/versions.py delete mode 100644 hbase/versions.py delete mode 100644 hive/versions.py delete mode 100644 java-base/versions.py delete mode 100644 java-devel/versions.py delete mode 100644 jdk-base/versions.py delete mode 100644 kafka-testing-tools/versions.py delete mode 100644 kafka/kafka-opa-plugin/versions.py delete mode 100644 kafka/kcat/versions.py delete mode 100644 kafka/versions.py delete mode 100644 krb5/versions.py delete mode 100644 nifi/versions.py delete mode 100644 omid/versions.py delete mode 100644 opa/versions.py delete mode 100644 opensearch/security-plugin/versions.py delete mode 100644 opensearch/versions.py delete mode 100644 shared/statsd-exporter/versions.py delete mode 100644 spark-connect-client/versions.py delete mode 100644 spark-k8s/versions.py delete mode 100644 stackable-base/versions.py delete mode 100644 stackable-devel/versions.py delete mode 100644 superset/versions.py delete mode 100644 testing-tools/versions.py delete mode 100644 tools/versions.py delete mode 100644 trino-cli/versions.py delete mode 100644 trino/storage-connector/versions.py delete mode 100644 trino/trino/versions.py delete mode 100644 trino/versions.py delete mode 100644 vector/versions.py delete mode 100644 zookeeper/versions.py diff --git a/airflow/versions.py b/airflow/versions.py deleted file mode 100644 index 9788be818..000000000 --- a/airflow/versions.py +++ /dev/null @@ -1,54 +0,0 @@ -versions = [ - { - "product": "2.9.3", - "python": "3.9", - "git_sync": "v4.4.1", - "s3fs": "2024.9.0", - "cyclonedx_bom": "6.0.0", - "shared/statsd-exporter": "0.28.0", - "tini": "0.19.0", - "vector": "0.47.0", - "uv": "0.7.8", - "extras": "async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,ldap,google,google_auth,microsoft.azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv,trino", - "opa_auth_manager": "airflow-2", - }, - { - "product": "2.10.4", - "python": "3.12", - "git_sync": "v4.4.1", - "s3fs": "2024.9.0", - "cyclonedx_bom": "6.0.0", - "shared/statsd-exporter": "0.28.0", - "tini": "0.19.0", - "vector": "0.47.0", - "uv": "0.7.8", - "extras": "async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,ldap,google,google_auth,microsoft.azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv,trino", - "opa_auth_manager": "airflow-2", - }, - { - "product": "2.10.5", - "python": "3.12", - "git_sync": "v4.4.1", - "s3fs": "2024.9.0", - "cyclonedx_bom": "6.0.0", - "shared/statsd-exporter": "0.28.0", - "tini": "0.19.0", - "vector": "0.47.0", - "uv": "0.7.8", - "extras": "async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,ldap,google,google_auth,microsoft.azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv,trino", - "opa_auth_manager": "airflow-2", - }, - { - "product": "3.0.1", - "python": "3.12", - "git_sync": "v4.4.1", - "s3fs": "2024.9.0", - "cyclonedx_bom": "6.0.0", - "shared/statsd-exporter": "0.28.0", - "tini": "0.19.0", - "vector": "0.47.0", - "uv": "0.7.8", - "extras": "async,amazon,celery,cncf-kubernetes,docker,elasticsearch,fab,ftp,grpc,hashicorp,http,ldap,google,microsoft-azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,trino", - "opa_auth_manager": "airflow-3", - }, -] diff --git a/conf.py b/conf.py deleted file mode 100644 index fa4c83017..000000000 --- a/conf.py +++ /dev/null @@ -1,128 +0,0 @@ -""" -Configuration file for the Stackable image-tools: https://github.com/stackabletech/image-tools. - -Application images will be created for products and associated versions configured here. -""" - -# NOTE: The .scripts/enumerate-product-versions.py script (used in the release workflow as of 2024-07-23) imports this file and it relies on conf.py being in its parent folder. Should this file be moved or the structure changed in any way remember to update that script as well! - -# NOTE (@NickLarsenNZ): Unfortunately, some directories have hyphens, so they need -# importing in a special way. For consistency, we'll do them all the same way. -import importlib - -airflow = importlib.import_module("airflow.versions") -druid = importlib.import_module("druid.versions") -hadoop = importlib.import_module("hadoop.versions") -hadoop_jars = importlib.import_module("hadoop.hadoop.versions") -hbase = importlib.import_module("hbase.versions") -hbase_jars = importlib.import_module("hbase.hbase.versions") -hbase_phoenix = importlib.import_module("hbase.phoenix.versions") -hbase_opa_authorizer = importlib.import_module("hbase.hbase-opa-authorizer.versions") -hbase_operator_tools = importlib.import_module("hbase.hbase-operator-tools.versions") -hive = importlib.import_module("hive.versions") -java_base = importlib.import_module("java-base.versions") -java_devel = importlib.import_module("java-devel.versions") -jdk_base = importlib.import_module("jdk-base.versions") -kafka = importlib.import_module("kafka.versions") -krb5 = importlib.import_module("krb5.versions") -vector = importlib.import_module("vector.versions") -nifi = importlib.import_module("nifi.versions") -omid = importlib.import_module("omid.versions") -opa = importlib.import_module("opa.versions") -opensearch = importlib.import_module("opensearch.versions") -opensearch_security_plugin = importlib.import_module( - "opensearch.security-plugin.versions" -) -spark_k8s = importlib.import_module("spark-k8s.versions") -stackable_base = importlib.import_module("stackable-base.versions") -stackable_devel = importlib.import_module("stackable-devel.versions") -superset = importlib.import_module("superset.versions") -trino_cli = importlib.import_module("trino-cli.versions") -trino = importlib.import_module("trino.versions") -trino_jars = importlib.import_module("trino.trino.versions") -trino_storage_connector = importlib.import_module("trino.storage-connector.versions") -kafka_testing_tools = importlib.import_module("kafka-testing-tools.versions") -kcat = importlib.import_module("kafka.kcat.versions") -kafka_opa_plugin = importlib.import_module("kafka.kafka-opa-plugin.versions") -testing_tools = importlib.import_module("testing-tools.versions") -zookeeper = importlib.import_module("zookeeper.versions") -tools = importlib.import_module("tools.versions") -statsd_exporter = importlib.import_module("shared.statsd-exporter.versions") -spark_connect_client = importlib.import_module("spark-connect-client.versions") - -products = [ - {"name": "airflow", "versions": airflow.versions}, - {"name": "druid", "versions": druid.versions}, - {"name": "hadoop", "versions": hadoop.versions}, - {"name": "hadoop/hadoop", "versions": hadoop_jars.versions}, - {"name": "hbase", "versions": hbase.versions}, - {"name": "hbase/hbase", "versions": hbase_jars.versions}, - {"name": "hbase/phoenix", "versions": hbase_phoenix.versions}, - {"name": "hbase/hbase-opa-authorizer", "versions": hbase_opa_authorizer.versions}, - {"name": "hbase/hbase-operator-tools", "versions": hbase_operator_tools.versions}, - {"name": "hive", "versions": hive.versions}, - {"name": "java-base", "versions": java_base.versions}, - {"name": "java-devel", "versions": java_devel.versions}, - {"name": "jdk-base", "versions": jdk_base.versions}, - {"name": "kafka", "versions": kafka.versions}, - {"name": "krb5", "versions": krb5.versions}, - {"name": "vector", "versions": vector.versions}, - {"name": "nifi", "versions": nifi.versions}, - {"name": "omid", "versions": omid.versions}, - {"name": "opa", "versions": opa.versions}, - {"name": "opensearch", "versions": opensearch.versions}, - { - "name": "opensearch/security-plugin", - "versions": opensearch_security_plugin.versions, - }, - {"name": "spark-k8s", "versions": spark_k8s.versions}, - {"name": "stackable-base", "versions": stackable_base.versions}, - {"name": "stackable-devel", "versions": stackable_devel.versions}, - {"name": "superset", "versions": superset.versions}, - {"name": "trino-cli", "versions": trino_cli.versions}, - {"name": "trino", "versions": trino.versions}, - {"name": "trino/trino", "versions": trino_jars.versions}, - {"name": "trino/storage-connector", "versions": trino_storage_connector.versions}, - {"name": "kafka-testing-tools", "versions": kafka_testing_tools.versions}, - {"name": "kafka/kcat", "versions": kcat.versions}, - {"name": "kafka/kafka-opa-plugin", "versions": kafka_opa_plugin.versions}, - {"name": "testing-tools", "versions": testing_tools.versions}, - {"name": "zookeeper", "versions": zookeeper.versions}, - {"name": "tools", "versions": tools.versions}, - {"name": "shared/statsd-exporter", "versions": statsd_exporter.versions}, - {"name": "spark-connect-client", "versions": spark_connect_client.versions}, -] - -open_shift_projects = { - "airflow": {"id": "62613f498ccb9938ba3cfde6"}, - "druid": {"id": "626140028ccb9938ba3cfde7"}, - "hadoop": {"id": "6261407f887d6e0b8614660c"}, - "hbase": {"id": "62614109992bac3f9a4a24b8"}, - "hive": {"id": "626140806812078a392dceaa"}, - "kafka": {"id": "625ff25b91bdcd4b49c823a4"}, - "nifi": {"id": "625586a32e9e14bc8118e203"}, - "opa": {"id": "6255838bea1feb8bec4aaaa3"}, - "opensearch": {"id": "6880fe690db664aa303d3a28"}, - "spark-k8s": {"id": "62613e81f8ce82a2f247dda5"}, - "superset": {"id": "62557e5fea1feb8bec4aaaa0"}, - "tools": {"id": "62557cd575ab7e30884aaaa0"}, - "trino": {"id": "62557c4a0030f6483318e203"}, - "zookeeper": {"id": "62552b0aadd9d54d56cda11d"}, -} - -cache = [ - { - "type": "registry", - "ref_prefix": "build-repo.stackable.tech:8083/sandbox/cache", - "mode": "max", - "compression": "zstd", - "ignore-error": "true", - }, -] - -args = { - "STACKABLE_USER_NAME": "stackable", - "STACKABLE_USER_UID": "1000", - "STACKABLE_USER_GID": "1000", - "DELETE_CACHES": "true", -} diff --git a/druid/versions.py b/druid/versions.py deleted file mode 100644 index e380f7c81..000000000 --- a/druid/versions.py +++ /dev/null @@ -1,26 +0,0 @@ -versions = [ - { - "product": "30.0.1", - # https://druid.apache.org/docs/30.0.1/operations/java/ - "java-base": "17", - "java-devel": "17", - "hadoop/hadoop": "3.3.6", - "authorizer": "0.7.0", - }, - { - "product": "31.0.1", - # https://druid.apache.org/docs/31.0.1/operations/java/ - "java-base": "17", - "java-devel": "17", - "hadoop/hadoop": "3.3.6", - "authorizer": "0.7.0", - }, - { - "product": "33.0.0", - # https://druid.apache.org/docs/33.0.0/operations/java/ - "java-base": "17", - "java-devel": "17", - "hadoop/hadoop": "3.3.6", - "authorizer": "0.7.0", - }, -] diff --git a/hadoop/hadoop/versions.py b/hadoop/hadoop/versions.py deleted file mode 100644 index d6907a86f..000000000 --- a/hadoop/hadoop/versions.py +++ /dev/null @@ -1,13 +0,0 @@ -versions = [ - { - # Not part of SDP 25.7.0, but still required for hbase, hive, spark-k8s - "product": "3.3.6", - "java-devel": "11", - "protobuf": "3.7.1", - }, - { - "product": "3.4.1", - "java-devel": "11", - "protobuf": "3.7.1", - }, -] diff --git a/hadoop/versions.py b/hadoop/versions.py deleted file mode 100644 index e88be1aca..000000000 --- a/hadoop/versions.py +++ /dev/null @@ -1,21 +0,0 @@ -versions = [ - { - # Not part of SDP 25.7.0, but still required for hbase, hive, spark-k8s - "product": "3.3.6", - "hadoop/hadoop": "3.3.6", - "java-base": "11", - "java-devel": "11", - "async_profiler": "2.9", - "jmx_exporter": "1.3.0", - "hdfs_utils": "0.4.0", - }, - { - "product": "3.4.1", - "hadoop/hadoop": "3.4.1", - "java-base": "11", - "java-devel": "11", - "async_profiler": "2.9", - "jmx_exporter": "1.3.0", - "hdfs_utils": "0.4.1", - }, -] diff --git a/hbase/hbase-opa-authorizer/versions.py b/hbase/hbase-opa-authorizer/versions.py deleted file mode 100644 index cd1f2934f..000000000 --- a/hbase/hbase-opa-authorizer/versions.py +++ /dev/null @@ -1,7 +0,0 @@ -versions = [ - { - "product": "0.1.0", - "java-devel": "11", - "delete_caches": "true", - }, -] diff --git a/hbase/hbase-operator-tools/versions.py b/hbase/hbase-operator-tools/versions.py deleted file mode 100644 index 248ecd343..000000000 --- a/hbase/hbase-operator-tools/versions.py +++ /dev/null @@ -1,20 +0,0 @@ -versions = [ - { - "product": "1.3.0-fd5a5fb-hbase2.6.1", - "hbase_operator_tools_version": "1.3.0-fd5a5fb", - "hadoop/hadoop": "3.3.6", - "hbase_thirdparty": "4.1.9", - "hbase/hbase": "2.6.1", - "java-devel": "11", - "delete_caches": "true", - }, - { - "product": "1.3.0-fd5a5fb-hbase2.6.2", - "hbase_operator_tools_version": "1.3.0-fd5a5fb", - "hadoop/hadoop": "3.4.1", - "hbase_thirdparty": "4.1.9", - "hbase/hbase": "2.6.2", - "java-devel": "11", - "delete_caches": "true", - }, -] diff --git a/hbase/hbase/versions.py b/hbase/hbase/versions.py deleted file mode 100644 index 205cedacb..000000000 --- a/hbase/hbase/versions.py +++ /dev/null @@ -1,20 +0,0 @@ -versions = [ - # Also do not merge java-base with java below as "JAVA-BASE is not a valid identifier" in Dockerfiles, it's unfortunate but to fix this would require a bigger refactoring of names or the image tools - # hbase-thirdparty is used to build the hbase-operator-tools and should be set to the version defined in the POM of HBase. - { - "product": "2.6.1", - "hadoop/hadoop": "3.3.6", - "java-base": "11", - "java-devel": "11", - "async_profiler": "2.9", - "delete_caches": "true", - }, - { - "product": "2.6.2", - "hadoop/hadoop": "3.4.1", - "java-base": "11", - "java-devel": "11", - "async_profiler": "2.9", - "delete_caches": "true", - }, -] diff --git a/hbase/phoenix/versions.py b/hbase/phoenix/versions.py deleted file mode 100644 index ed0e304e2..000000000 --- a/hbase/phoenix/versions.py +++ /dev/null @@ -1,20 +0,0 @@ -versions = [ - { - "product": "5.2.1-hbase2.6.1", - "phoenix_version": "5.2.1", - "hbase/hbase": "2.6.1", - "hadoop/hadoop": "3.3.6", - "java-devel": "11", - "hbase_profile": "2.6", - "delete_caches": "true", - }, - { - "product": "5.2.1-hbase2.6.2", - "phoenix_version": "5.2.1", - "hbase/hbase": "2.6.2", - "hadoop/hadoop": "3.4.1", - "java-devel": "11", - "hbase_profile": "2.6", - "delete_caches": "true", - }, -] diff --git a/hbase/versions.py b/hbase/versions.py deleted file mode 100644 index 3ea6dcdbf..000000000 --- a/hbase/versions.py +++ /dev/null @@ -1,28 +0,0 @@ -versions = [ - # Also do not merge java-base with java below as "JAVA-BASE is not a valid identifier" in Dockerfiles, it's unfortunate but to fix this would require a bigger refactoring of names or the image tools - # hbase-thirdparty is used to build the hbase-operator-tools and should be set to the version defined in the POM of HBase. - { - "product": "2.6.1", - "hbase/hbase": "2.6.1", - "hbase/hbase-operator-tools": "1.3.0-fd5a5fb-hbase2.6.1", - "hbase/phoenix": "5.2.1-hbase2.6.1", - "hbase/hbase-opa-authorizer": "0.1.0", # only for HBase 2.6.1 - "hadoop/hadoop": "3.3.6", - "java-base": "11", - "java-devel": "11", - "hbase_profile": "2.6", - "delete_caches": "true", - }, - { - "product": "2.6.2", - "hbase/hbase": "2.6.2", - "hbase/hbase-operator-tools": "1.3.0-fd5a5fb-hbase2.6.2", - "hbase/phoenix": "5.2.1-hbase2.6.2", - "hbase/hbase-opa-authorizer": "0.1.0", # only for HBase 2.6.1 - "hadoop/hadoop": "3.4.1", - "java-base": "11", - "java-devel": "11", - "hbase_profile": "2.6", - "delete_caches": "true", - }, -] diff --git a/hive/versions.py b/hive/versions.py deleted file mode 100644 index d28fa84be..000000000 --- a/hive/versions.py +++ /dev/null @@ -1,38 +0,0 @@ -versions = [ - { - "product": "3.1.3", - "jmx_exporter": "1.3.0", - # Hive 3 must be built with Java 8 but will run on Java 11 - "java-base": "11", - "java-devel": "8", - "hadoop/hadoop": "3.3.6", - # Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 - "aws_java_sdk_bundle": "1.12.367", - "azure_storage": "7.0.1", - "azure_keyvault_core": "1.0.0", - }, - { - "product": "4.0.0", - "jmx_exporter": "1.3.0", - # Hive 4 must be built with Java 8 (according to GitHub README) but seems to run on Java 11 - "java-base": "11", - "java-devel": "8", - "hadoop/hadoop": "3.3.6", - # Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 - "aws_java_sdk_bundle": "1.12.367", - "azure_storage": "7.0.1", - "azure_keyvault_core": "1.0.0", - }, - { - "product": "4.0.1", - "jmx_exporter": "1.3.0", - # Hive 4 must be built with Java 8 (according to GitHub README) but seems to run on Java 11 - "java-base": "11", - "java-devel": "8", - "hadoop/hadoop": "3.3.6", - # Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 - "aws_java_sdk_bundle": "1.12.367", - "azure_storage": "7.0.1", - "azure_keyvault_core": "1.0.0", - }, -] diff --git a/java-base/versions.py b/java-base/versions.py deleted file mode 100644 index 036d18203..000000000 --- a/java-base/versions.py +++ /dev/null @@ -1,30 +0,0 @@ -versions = [ - { - "product": "8", - "vector": "0.47.0", - }, - { - "product": "11", - "vector": "0.47.0", - }, - { - "product": "17", - "vector": "0.47.0", - }, - { - "product": "21", - "vector": "0.47.0", - }, - { - "product": "22", - "vector": "0.47.0", - }, - { - "product": "23", - "vector": "0.47.0", - }, - { - "product": "24", - "vector": "0.47.0", - }, -] diff --git a/java-devel/versions.py b/java-devel/versions.py deleted file mode 100644 index cd0838e4f..000000000 --- a/java-devel/versions.py +++ /dev/null @@ -1,30 +0,0 @@ -versions = [ - { - "product": "8", - "stackable-devel": "1.0.0", - }, - { - "product": "11", - "stackable-devel": "1.0.0", - }, - { - "product": "17", - "stackable-devel": "1.0.0", - }, - { - "product": "21", - "stackable-devel": "1.0.0", - }, - { - "product": "22", - "stackable-devel": "1.0.0", - }, - { - "product": "23", - "stackable-devel": "1.0.0", - }, - { - "product": "24", - "stackable-devel": "1.0.0", - }, -] diff --git a/jdk-base/versions.py b/jdk-base/versions.py deleted file mode 100644 index 036d18203..000000000 --- a/jdk-base/versions.py +++ /dev/null @@ -1,30 +0,0 @@ -versions = [ - { - "product": "8", - "vector": "0.47.0", - }, - { - "product": "11", - "vector": "0.47.0", - }, - { - "product": "17", - "vector": "0.47.0", - }, - { - "product": "21", - "vector": "0.47.0", - }, - { - "product": "22", - "vector": "0.47.0", - }, - { - "product": "23", - "vector": "0.47.0", - }, - { - "product": "24", - "vector": "0.47.0", - }, -] diff --git a/kafka-testing-tools/versions.py b/kafka-testing-tools/versions.py deleted file mode 100644 index b706d446e..000000000 --- a/kafka-testing-tools/versions.py +++ /dev/null @@ -1,8 +0,0 @@ -versions = [ - { - "product": "1.0.0", - "kafka/kcat": "1.7.0", - "java-base": "11", - "stackable-base": "1.0.0", - } -] diff --git a/kafka/kafka-opa-plugin/versions.py b/kafka/kafka-opa-plugin/versions.py deleted file mode 100644 index 602909cd2..000000000 --- a/kafka/kafka-opa-plugin/versions.py +++ /dev/null @@ -1,6 +0,0 @@ -versions = [ - { - "product": "1.5.1", - "java-devel": "11", - }, -] diff --git a/kafka/kcat/versions.py b/kafka/kcat/versions.py deleted file mode 100644 index 80f708acd..000000000 --- a/kafka/kcat/versions.py +++ /dev/null @@ -1,7 +0,0 @@ -versions = [ - { - "product": "1.7.0", - "java-devel": "11", - "stackable-devel": "1.0.0", - } -] diff --git a/kafka/versions.py b/kafka/versions.py deleted file mode 100644 index bf16adadb..000000000 --- a/kafka/versions.py +++ /dev/null @@ -1,38 +0,0 @@ -versions = [ - { - "product": "3.7.2", - "java-base": "21", - "java-devel": "21", - "scala": "2.13", - "kafka/kcat": "1.7.0", - "kafka/kafka-opa-plugin": "1.5.1", - "jmx_exporter": "1.3.0", - }, - { - "product": "3.9.0", - "java-base": "21", - "java-devel": "21", - "scala": "2.13", - "kafka/kcat": "1.7.0", - "kafka/kafka-opa-plugin": "1.5.1", - "jmx_exporter": "1.3.0", - }, - { - "product": "3.9.1", - "java-base": "21", - "java-devel": "21", - "scala": "2.13", - "kafka/kcat": "1.7.0", - "kafka/kafka-opa-plugin": "1.5.1", - "jmx_exporter": "1.3.0", - }, - { - "product": "4.0.0", - "java-base": "23", - "java-devel": "23", - "scala": "2.13", - "kafka/kcat": "1.7.0", - "kafka/kafka-opa-plugin": "1.5.1", - "jmx_exporter": "1.3.0", - }, -] diff --git a/krb5/versions.py b/krb5/versions.py deleted file mode 100644 index fcf6ef066..000000000 --- a/krb5/versions.py +++ /dev/null @@ -1,5 +0,0 @@ -versions = [ - { - "product": "1.21.1", - }, -] diff --git a/nifi/versions.py b/nifi/versions.py deleted file mode 100644 index ea9e26360..000000000 --- a/nifi/versions.py +++ /dev/null @@ -1,24 +0,0 @@ -versions = [ - { - "product": "1.27.0", - "java-base": "11", - "java-devel": "11", # There is an error when trying to use the jdk 21 (since nifi 1.26.0) - "git_sync": "v4.4.1", - "nifi_opa_authorizer_plugin": "0.1.0", - }, - { - "product": "1.28.1", - "java-base": "11", - "java-devel": "11", - "git_sync": "v4.4.1", - "nifi_opa_authorizer_plugin": "0.1.0", - }, - { - "product": "2.4.0", - "java-base": "21", - "java-devel": "21", - "git_sync": "v4.4.1", - "nifi_iceberg_bundle": "0.0.4", - "nifi_opa_authorizer_plugin": "0.1.0", - }, -] diff --git a/omid/versions.py b/omid/versions.py deleted file mode 100644 index 533cb91be..000000000 --- a/omid/versions.py +++ /dev/null @@ -1,14 +0,0 @@ -versions = [ - { - "product": "1.1.2", - "java-base": "11", - "java-devel": "11", - "jmx_exporter": "1.3.0", - }, - { - "product": "1.1.3", - "java-base": "11", - "java-devel": "11", - "jmx_exporter": "1.3.0", - }, -] diff --git a/opa/versions.py b/opa/versions.py deleted file mode 100644 index 64589586a..000000000 --- a/opa/versions.py +++ /dev/null @@ -1,14 +0,0 @@ -versions = [ - { - "product": "1.4.2", - "vector": "0.47.0", - "golang": "1.23.9", - "stackable-devel": "1.0.0", - }, - { - "product": "1.0.1", - "vector": "0.47.0", - "golang": "1.23.9", - "stackable-devel": "1.0.0", - }, -] diff --git a/opensearch/security-plugin/versions.py b/opensearch/security-plugin/versions.py deleted file mode 100644 index a46fa4c9f..000000000 --- a/opensearch/security-plugin/versions.py +++ /dev/null @@ -1,6 +0,0 @@ -versions = [ - { - "product": "3.1.0.0", - "java-devel": "21", - }, -] diff --git a/opensearch/versions.py b/opensearch/versions.py deleted file mode 100644 index 708e430d7..000000000 --- a/opensearch/versions.py +++ /dev/null @@ -1,8 +0,0 @@ -versions = [ - { - "product": "3.1.0", - "java-devel": "21", - "jdk-base": "21", - "opensearch/security-plugin": "3.1.0.0", - }, -] diff --git a/shared/statsd-exporter/versions.py b/shared/statsd-exporter/versions.py deleted file mode 100644 index adb26f18a..000000000 --- a/shared/statsd-exporter/versions.py +++ /dev/null @@ -1,6 +0,0 @@ -versions = [ - { - "product": "0.28.0", - "stackable-base": "1.0.0", - } -] diff --git a/spark-connect-client/versions.py b/spark-connect-client/versions.py deleted file mode 100644 index be7977a85..000000000 --- a/spark-connect-client/versions.py +++ /dev/null @@ -1,8 +0,0 @@ -versions = [ - { - "product": "3.5.6", - "spark-k8s": "3.5.6", - "java-base": "17", - "python": "3.11", - }, -] diff --git a/spark-k8s/versions.py b/spark-k8s/versions.py deleted file mode 100644 index fe6405702..000000000 --- a/spark-k8s/versions.py +++ /dev/null @@ -1,38 +0,0 @@ -versions = [ - { - "product": "3.5.5", - "java-base": "17", - "java-devel": "17", - "python": "3.11", - "hadoop/hadoop": "3.4.1", # Current Stackable LTS version. Source of the AWS and Azure artifacts to Spark's classpath. - "hbase": "2.6.2", # Current Stackable LTS version. Used to build the HBase connector. - "aws_java_sdk_bundle": "2.24.6", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.1 - "azure_storage": "7.0.1", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.1 - "azure_keyvault_core": "1.0.0", # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 - "jackson_dataformat_xml": "2.15.2", # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.5.2 - "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 - "woodstox_core": "6.5.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 - "vector": "0.47.0", - "jmx_exporter": "1.3.0", - "tini": "0.19.0", - "hbase_connector": "1.0.1", - }, - { - "product": "3.5.6", - "java-base": "17", - "java-devel": "17", - "python": "3.11", - "hadoop/hadoop": "3.4.1", # Current Stackable LTS version. Source of the AWS and Azure artifacts to Spark's classpath. - "hbase": "2.6.2", # Current Stackable LTS version. Used to build the HBase connector. - "aws_java_sdk_bundle": "2.24.6", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.1 - "azure_storage": "7.0.1", # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.1 - "azure_keyvault_core": "1.0.0", # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 - "jackson_dataformat_xml": "2.15.2", # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.5.2 - "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 - "woodstox_core": "6.5.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 - "vector": "0.47.0", - "jmx_exporter": "1.3.0", - "tini": "0.19.0", - "hbase_connector": "1.0.1", - }, -] diff --git a/stackable-base/versions.py b/stackable-base/versions.py deleted file mode 100644 index 6f6bf1ca7..000000000 --- a/stackable-base/versions.py +++ /dev/null @@ -1,6 +0,0 @@ -versions = [ - { - "product": "1.0.0", - "stackable-devel": "1.0.0", - }, -] diff --git a/stackable-devel/versions.py b/stackable-devel/versions.py deleted file mode 100644 index ed192d92f..000000000 --- a/stackable-devel/versions.py +++ /dev/null @@ -1,5 +0,0 @@ -versions = [ - { - "product": "1.0.0", - }, -] diff --git a/superset/versions.py b/superset/versions.py deleted file mode 100644 index 4ef1f3e6c..000000000 --- a/superset/versions.py +++ /dev/null @@ -1,32 +0,0 @@ -versions = [ - { - "product": "4.0.2", - "python": "3.9", - "cyclonedx_bom": "6.0.0", - "vector": "0.47.0", - "shared/statsd-exporter": "0.28.0", - "authlib": "1.2.1", # https://github.com/dpgaspar/Flask-AppBuilder/blob/release/4.4.1/requirements/extra.txt#L7 - "stackable-base": "1.0.0", - "uv": "0.7.3", - }, - { - "product": "4.1.1", - "python": "3.9", # 3.11 support was merged in January 2025 (two months after 4.1.1 release), 3.10 is not available in our UBI image, so we need to stay on 3.9 for now - "cyclonedx_bom": "6.0.0", - "vector": "0.47.0", - "shared/statsd-exporter": "0.28.0", - "authlib": "1.2.1", # https://github.com/dpgaspar/Flask-AppBuilder/blob/release/4.5.0/requirements/extra.txt#L7 - "stackable-base": "1.0.0", - "uv": "0.7.3", - }, - { - "product": "4.1.2", - "python": "3.9", - "cyclonedx_bom": "6.0.0", - "vector": "0.47.0", - "shared/statsd-exporter": "0.28.0", - "authlib": "1.2.1", # https://github.com/dpgaspar/Flask-AppBuilder/blob/release/4.5.0/requirements/extra.txt#L7 - "stackable-base": "1.0.0", - "uv": "0.7.3", - }, -] diff --git a/testing-tools/versions.py b/testing-tools/versions.py deleted file mode 100644 index 5231af218..000000000 --- a/testing-tools/versions.py +++ /dev/null @@ -1,6 +0,0 @@ -versions = [ - { - "product": "0.2.0", - "keycloak_version": "23.0.0", - } -] diff --git a/tools/versions.py b/tools/versions.py deleted file mode 100644 index 78269452c..000000000 --- a/tools/versions.py +++ /dev/null @@ -1,9 +0,0 @@ -versions = [ - { - "product": "1.0.0", - "kubectl_version": "1.33.0", - "jq_version": "1.7.1", - "stackable-base": "1.0.0", - "yq_version": "4.45.2", - }, -] diff --git a/trino-cli/versions.py b/trino-cli/versions.py deleted file mode 100644 index 828f6f625..000000000 --- a/trino-cli/versions.py +++ /dev/null @@ -1,8 +0,0 @@ -# This image is only used in integration tests and demos. -# It's therefore ok if we only support a single version at a time. -versions = [ - { - "product": "476", - "java-base": "24", - }, -] diff --git a/trino/storage-connector/versions.py b/trino/storage-connector/versions.py deleted file mode 100644 index d9fe35c51..000000000 --- a/trino/storage-connector/versions.py +++ /dev/null @@ -1,17 +0,0 @@ -versions = [ - { - "product": "451", - "trino/trino": "451", - "java-devel": "22", - }, - { - "product": "470", - "trino/trino": "470", - "java-devel": "23", - }, - { - "product": "476", - "trino/trino": "476", - "java-devel": "24", - }, -] diff --git a/trino/trino/versions.py b/trino/trino/versions.py deleted file mode 100644 index 3c67dc2b3..000000000 --- a/trino/trino/versions.py +++ /dev/null @@ -1,14 +0,0 @@ -versions = [ - { - "product": "451", - "java-devel": "22", - }, - { - "product": "470", - "java-devel": "23", - }, - { - "product": "476", - "java-devel": "24", - }, -] diff --git a/trino/versions.py b/trino/versions.py deleted file mode 100644 index bb9a9dac8..000000000 --- a/trino/versions.py +++ /dev/null @@ -1,23 +0,0 @@ -versions = [ - { - "product": "451", - "java-base": "22", - "trino/trino": "451", - "jmx_exporter": "1.3.0", - "trino/storage-connector": "451", - }, - { - "product": "470", - "java-base": "23", - "trino/trino": "470", - "jmx_exporter": "1.3.0", - "trino/storage-connector": "470", - }, - { - "product": "476", - "java-base": "24", - "trino/trino": "476", - "jmx_exporter": "1.3.0", - "trino/storage-connector": "476", - }, -] diff --git a/vector/versions.py b/vector/versions.py deleted file mode 100644 index 828d3fc2d..000000000 --- a/vector/versions.py +++ /dev/null @@ -1,8 +0,0 @@ -versions = [ - { - "product": "0.47.0", - "rpm_release": "1", - "stackable-base": "1.0.0", - "inotify_tools": "3.22.1.0-1.el9", - } -] diff --git a/zookeeper/versions.py b/zookeeper/versions.py deleted file mode 100644 index d63bfac5a..000000000 --- a/zookeeper/versions.py +++ /dev/null @@ -1,12 +0,0 @@ -versions = [ - { - "product": "3.9.3", - "java-base": "17", - # NOTE (@NickLarsenNZ): Builds fail on Java 17, with the output: - # [ERROR] Failed to execute goal com.github.spotbugs:spotbugs-maven-plugin:4.0.0:spotbugs (spotbugs) on project - # zookeeper: Execution spotbugs of goal com.github.spotbugs:spotbugs-maven-plugin:4.0.0:spotbugs failed: Java - # returned: 1 -> [Help 1] - "java-devel": "11", - "jmx_exporter": "1.3.0", - }, -] From 9a6ab65d8431bf1fe6c61f61d235ed3a09f8f2c4 Mon Sep 17 00:00:00 2001 From: Techassi Date: Tue, 12 Aug 2025 15:49:01 +0200 Subject: [PATCH 04/51] ci: Adjust notification condition Only send out notifications if the build failed, a build is re-run and wasn't cancelled. --- .github/workflows/reusable_build_image.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable_build_image.yaml b/.github/workflows/reusable_build_image.yaml index a11cb8c80..5ae205d65 100644 --- a/.github/workflows/reusable_build_image.yaml +++ b/.github/workflows/reusable_build_image.yaml @@ -102,7 +102,7 @@ jobs: name: Failure Notification needs: [generate_matrix, build, publish_manifests] runs-on: ubuntu-latest - if: failure() || github.run_attempt > 1 + if: failure() || (github.run_attempt > 1 && !cancelled()) steps: - name: Send Notification uses: stackabletech/actions/send-slack-notification@55d2f9fcbcd7884ac929ea65fd6f069e7b7a49d2 # v0.8.1 From 75194b270ad5c9231655dab07210280044b4ec28 Mon Sep 17 00:00:00 2001 From: Techassi Date: Wed, 13 Aug 2025 09:03:14 +0200 Subject: [PATCH 05/51] chore: Remove commented-out code --- rust/boil/src/build/bakefile.rs | 47 --------------------------------- 1 file changed, 47 deletions(-) diff --git a/rust/boil/src/build/bakefile.rs b/rust/boil/src/build/bakefile.rs index e56896888..4727dc50d 100644 --- a/rust/boil/src/build/bakefile.rs +++ b/rust/boil/src/build/bakefile.rs @@ -429,50 +429,3 @@ impl BakefileTarget { pub struct BakefileGroup { targets: Vec, } - -// #[derive(Debug, Default)] -// pub struct Graph { -// targets: BTreeMap>, -// } - -// impl Graph { -// pub fn all() -> Self { -// let image_config_paths: Vec<_> = glob("./**/boil-config.toml") -// .expect("glob pattern must be valid") -// .filter_map(Result::ok) -// .collect(); - -// let mut targets = Self::default(); - -// for image_config_path in image_config_paths { -// let image_config = ImageConfig::from_file(&image_config_path).unwrap(); - -// let (image_name, _) = image_config_path -// .to_str() -// .unwrap() -// .rsplit_once('/') -// .unwrap(); - -// let pairs = image_config.filter_by_version(None).unwrap(); - -// targets.insert_targets(image_name.to_owned(), pairs); -// } - -// targets -// } - -// fn insert_targets( -// &mut self, -// image_name: String, -// pairs: Vec, -// ) -> Vec { -// let mut nodes = Vec::new(); - -// for VersionOptionsPair { version, options } in pairs { -// let key = format!("{image_name}:{version}"); -// let child_nodes = Vec::new(); - -// // let nodes = self.insert_targets(image_name, pairs); -// } -// } -// } From 6db36f65e60088139c9371601562a4ffd2b64046 Mon Sep 17 00:00:00 2001 From: Techassi Date: Wed, 13 Aug 2025 09:04:54 +0200 Subject: [PATCH 06/51] chore: Remove unwraps --- rust/boil/src/build/bakefile.rs | 9 ++++++--- rust/boil/src/build/docker.rs | 8 ++++++-- rust/boil/src/build/mod.rs | 5 ++++- rust/boil/src/show/images.rs | 7 +++++-- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/rust/boil/src/build/bakefile.rs b/rust/boil/src/build/bakefile.rs index 4727dc50d..d903f2dc1 100644 --- a/rust/boil/src/build/bakefile.rs +++ b/rust/boil/src/build/bakefile.rs @@ -7,7 +7,7 @@ use std::{ use glob::glob; use serde::Serialize; -use snafu::{ResultExt, Snafu}; +use snafu::{OptionExt, ResultExt, Snafu}; use time::format_description::well_known::Rfc3339; use url::Host; @@ -35,6 +35,9 @@ pub enum GitError { #[snafu(display("failed to parse HEAD revision"))] ParseHeadRevision { source: git2::Error }, + + #[snafu(display("failed to find starting point of rev range"))] + InvalidRange, } #[derive(Debug, Snafu)] @@ -366,7 +369,6 @@ impl Bakefile { /// Formats and return the context name, eg. `stackable/image/stackable-base-1_0_0`. fn format_context_name(name: &str) -> String { format!("local-image/{name}") - // format!("stackable/image/{name}") } /// Formats and returns the context target name, eg. `target:stackable-base-1_0_0`. @@ -384,8 +386,9 @@ impl Bakefile { fn git_head_revision() -> Result { let repo = git2::Repository::open(".").context(OpenRepositorySnafu)?; let rev = repo.revparse("HEAD").context(ParseHeadRevisionSnafu)?; + let rev = rev.from().context(InvalidRangeSnafu)?.id().to_string(); - Ok(rev.from().unwrap().id().to_string()) + Ok(rev) } } diff --git a/rust/boil/src/build/docker.rs b/rust/boil/src/build/docker.rs index 38f10e3cb..70fa77657 100644 --- a/rust/boil/src/build/docker.rs +++ b/rust/boil/src/build/docker.rs @@ -6,10 +6,14 @@ use std::{ }; use serde::{Deserialize, Serialize, de::Visitor, ser::SerializeMap}; -use snafu::{Snafu, ensure}; +use snafu::{OptionExt, Snafu, ensure}; #[derive(Debug, Snafu)] pub enum ParseBuildArgumentError { + #[snafu(display("invalid format, expected ="))] + InvalidFormat, + + #[snafu(display("encountered non ASCII characters"))] NonAscii, } @@ -29,7 +33,7 @@ impl FromStr for BuildArgument { fn from_str(s: &str) -> Result { ensure!(s.is_ascii(), NonAsciiSnafu); - let (key, value) = s.split_once('=').unwrap(); + let (key, value) = s.split_once('=').context(InvalidFormatSnafu)?; let key = key.replace(['-', '/'], "_").to_uppercase(); Ok(Self((key, value.to_owned()))) diff --git a/rust/boil/src/build/mod.rs b/rust/boil/src/build/mod.rs index aa3753f4b..ef5cae5b9 100644 --- a/rust/boil/src/build/mod.rs +++ b/rust/boil/src/build/mod.rs @@ -33,6 +33,9 @@ pub enum Error { #[snafu(display("failed to run child process"))] RunChildProcess { source: std::io::Error }, + #[snafu(display("failed to spawn child process"))] + SpawnChildProcess { source: std::io::Error }, + #[snafu(display("encountered invalid image version, must not include any build metadata"))] InvalidImageVersion, } @@ -74,7 +77,7 @@ pub fn run_command(args: BuildArguments, config: Config) -> Result<(), Error> { .arg("-") .stdin(Stdio::piped()) .spawn() - .unwrap(); + .context(SpawnChildProcessSnafu)?; let stdin_handle = child.stdin.take().with_context(|| { child diff --git a/rust/boil/src/show/images.rs b/rust/boil/src/show/images.rs index 3baaa3929..5af7daf16 100644 --- a/rust/boil/src/show/images.rs +++ b/rust/boil/src/show/images.rs @@ -2,17 +2,20 @@ use std::collections::BTreeMap; use snafu::{ResultExt, Snafu}; -use crate::build::bakefile::{Targets, TargetsOptions}; +use crate::build::bakefile::{Targets, TargetsError, TargetsOptions}; #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("failed to serialize list as JSON"))] SerializeList { source: serde_json::Error }, + + #[snafu(display("failed to build list of targets"))] + BuildTargets { source: TargetsError }, } pub fn run_command() -> Result<(), Error> { let list: BTreeMap<_, _> = Targets::all(TargetsOptions { only_entry: true }) - .unwrap() + .context(BuildTargetsSnafu)? .into_iter() .map(|(image_name, image_versions)| { let versions: Vec<_> = image_versions From edaacb9266265d2230d7a3a03110d2f1463dad82 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 15 Aug 2025 11:40:08 +0200 Subject: [PATCH 07/51] chore: Add cargo alias for boil --- .cargo/config.toml | 1 + rust/boil/README.md | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.cargo/config.toml b/.cargo/config.toml index b6cf3ff25..54e07b717 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,2 +1,3 @@ [alias] patchable = ["run", "--bin", "patchable", "--"] +boil = ["run", "--bin", "boil", "--"] diff --git a/rust/boil/README.md b/rust/boil/README.md index e29b38986..a53c2f3af 100644 --- a/rust/boil/README.md +++ b/rust/boil/README.md @@ -8,6 +8,8 @@ boil builds container images in parallel. ## Quick Overview +Either compile and run the binary, or use the `cargo boil` alias. + ```shell # Builds all version of the image located in the 'airflow' folder boil build airflow From 84184ec79fa49a8465174e4184a09184b2717eb4 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 15 Aug 2025 11:42:10 +0200 Subject: [PATCH 08/51] chore: Update README to mention boil --- README.md | 43 +++++++++++++------------------------------ 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index a5f2602ea..fa313e425 100644 --- a/README.md +++ b/README.md @@ -15,45 +15,28 @@ This repository contains Dockerfiles and scripts to build base images for use wi ## Prerequisites -* [Stackable Image Tools](https://github.com/stackabletech/image-tools) (`pip install image-tools-stackabletech`) +* [boil](./rust/boil//README.md) (`cargo boil`) * Docker including the [`buildx` plugin](https://github.com/docker/buildx) * Optional: [OpenShift preflight tool](https://github.com/redhat-openshift-ecosystem/openshift-preflight) to verify an image for OpenShift -## Build Product Images +## Build Images -Product images are published to the `oci.stackable.tech` registry under the `sdp` organization by default. +Images are published to the `oci.stackable.tech` registry under the `sdp` organization by default. -### Build single products locally +### Build images locally -To build and push product images to the default repository use this command: +Consult the [boil README](./rust/boil//README.md) which contains a broad selection of different commands to build images locally. -```sh -bake --product zookeeper --image 0.0.0-dev --push -``` - -This will build images for Apache ZooKeeper versions as defined in the `conf.py` file, tag them with the `image-version` 0.0.0-dev and push them to the registry. - -You can select a specific version of a product to build using the syntax `product=version` e.g. to build Hive 3.1.3 you can use this command: - -```sh -bake --product hive=3.1.3 -i 0.0.0-dev -``` - -> [!NOTE] -> `-i` is the shorthand for `--image` (i.e. the resulting image tag) - -### Build all products locally - -To build all products in all versions locally you can use this command - -```sh -bake --image-version 0.0.0-dev -``` +### Build images via GitHub Actions -### Build everything in GitHub +There are individual GHA workflows (one for each image) which use a +[reusable workflow](.github/workflows/reusable_build_image.yaml) to build all specified versions for +both `amd64` and `arm64`. The workflow is triggered -The GitHub action called `Build (and optionally publish) 0.0.0-dev images` can be triggered manually to do build all images in all versions. -When triggered manually it will _not_ push the images to the registry. +* by pushes to `main` to produce `0.0.0-dev` versions of the images, +* by a regular schedule to rebuild `0.0.0-dev` versions of the images to avoid staleness, +* by tag pushes to produce (release candidate) images for a particular SDP, +* and by manual workflow dispatches. ## Patches From f439e99b1052ddb9408bc4e9dd821d54e53c1ebb Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 15 Aug 2025 14:04:46 +0200 Subject: [PATCH 09/51] chore: Update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fa313e425..efd84522c 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ both `amd64` and `arm64`. The workflow is triggered * by pushes to `main` to produce `0.0.0-dev` versions of the images, * by a regular schedule to rebuild `0.0.0-dev` versions of the images to avoid staleness, -* by tag pushes to produce (release candidate) images for a particular SDP, +* by tag pushes to produce (release candidate) images for a particular SDP release, * and by manual workflow dispatches. ## Patches From 65bab9c02c07465c40ed62bb75796a46419237f4 Mon Sep 17 00:00:00 2001 From: Techassi Date: Sun, 17 Aug 2025 14:23:35 +0200 Subject: [PATCH 10/51] feat: Add --load flag to build command --- rust/boil/src/build/cli.rs | 4 ++++ rust/boil/src/build/mod.rs | 7 ++++++- rust/boil/src/utils.rs | 18 ++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/rust/boil/src/build/cli.rs b/rust/boil/src/build/cli.rs index 871923941..31530355e 100644 --- a/rust/boil/src/build/cli.rs +++ b/rust/boil/src/build/cli.rs @@ -98,6 +98,10 @@ pub struct BuildArguments { )] pub export_image_manifest_uris: Option, + /// Loads the image into the local image store. + #[arg(long, help_heading = "Build Options")] + pub load: bool, + /// Dry run. This does not build the image(s) but instead prints out the bakefile. #[arg(short, long, alias = "dry")] pub dry_run: bool, diff --git a/rust/boil/src/build/mod.rs b/rust/boil/src/build/mod.rs index ef5cae5b9..10a2db7fc 100644 --- a/rust/boil/src/build/mod.rs +++ b/rust/boil/src/build/mod.rs @@ -8,6 +8,7 @@ use snafu::{OptionExt, ResultExt, Snafu, ensure}; use crate::{ build::{bakefile::Bakefile, cli::BuildArguments}, config::Config, + utils::CommandExt, }; pub mod bakefile; @@ -68,17 +69,20 @@ pub fn run_command(args: BuildArguments, config: Config) -> Result<(), Error> { } // TODO (@Techassi): Invoke this directly using the Docker daemon via bollard + // or by building the image ourself. + // Finally invoke the docker buildx bake command let mut child = Command::new("docker") .arg("buildx") .arg("bake") - // .arg("--no-cache") + .arg_if(args.load, "--load") .arg("--file") .arg("-") .stdin(Stdio::piped()) .spawn() .context(SpawnChildProcessSnafu)?; + // Acquire stdin handle to pipe the bakefile as JSON to it let stdin_handle = child.stdin.take().with_context(|| { child .kill() @@ -93,6 +97,7 @@ pub fn run_command(args: BuildArguments, config: Config) -> Result<(), Error> { SerializeBakefileSnafu })?; + // Wait for successful completion of the child process let status = child.wait().context(RunChildProcessSnafu)?; // TODO (@Techassi): Return an error if the status was not a success diff --git a/rust/boil/src/utils.rs b/rust/boil/src/utils.rs index 561a949d6..def0f0320 100644 --- a/rust/boil/src/utils.rs +++ b/rust/boil/src/utils.rs @@ -1,3 +1,5 @@ +use std::process::Command; + use semver::Version; use url::Host; @@ -21,3 +23,19 @@ pub fn format_image_manifest_uri( ) -> String { format!("{image_repository_uri}:{image_version}-stackable{sdp_image_version}-{architecture}") } + +pub trait CommandExt { + /// Adds an argument to the command if the `predicate` is `true`. + fn arg_if(&mut self, predicate: bool, arg: S) -> &mut Self + where + S: AsRef; +} + +impl CommandExt for Command { + fn arg_if(&mut self, predicate: bool, arg: S) -> &mut Self + where + S: AsRef, + { + if predicate { self.arg(arg) } else { self } + } +} From 077b2d4cf942dd09ededff1ac55e007f462f5b0c Mon Sep 17 00:00:00 2001 From: Techassi Date: Sun, 17 Aug 2025 14:24:52 +0200 Subject: [PATCH 11/51] feat: Add more annotations to the built image --- Cargo.lock | 177 ++++++++++++++++++++++++++++++++ Cargo.toml | 3 +- boil.toml | 9 ++ rust/boil/Cargo.toml | 1 + rust/boil/src/build/bakefile.rs | 64 +++++++++--- rust/boil/src/build/image.rs | 3 + rust/boil/src/config.rs | 16 +++ 7 files changed, 259 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5181b2141..92fc3723d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,6 +111,7 @@ dependencies = [ "clap_complete", "git2", "glob", + "oci-spec", "semver", "serde", "serde_json", @@ -219,6 +220,61 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "const_format" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "126f97965c8ad46d6d9163268ff28432e8f6a1196a55578867832e3049df63dd" +dependencies = [ + "const_format_proc_macros", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn", +] + [[package]] name = "deranged" version = "0.4.0" @@ -228,6 +284,37 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -267,6 +354,12 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -288,6 +381,18 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "getset" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf0fc11e47561d47397154977bc219f4cf809b2974facc3ccb3b89e2436f912" +dependencies = [ + "proc-macro-error2", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "gimli" version = "0.31.1" @@ -445,6 +550,12 @@ dependencies = [ "syn", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.0.3" @@ -668,6 +779,23 @@ dependencies = [ "memchr", ] +[[package]] +name = "oci-spec" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2078e2f6be932a4de9aca90a375a45590809dfb5a08d93ab1ee217107aceeb67" +dependencies = [ + "const_format", + "derive_builder", + "getset", + "regex", + "serde", + "serde_json", + "strum", + "strum_macros", + "thiserror", +] + [[package]] name = "once_cell" version = "1.20.2" @@ -744,6 +872,28 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.93" @@ -1014,6 +1164,26 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "thiserror" +version = "2.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d76d3f064b981389ecb4b6b7f45a0bf9fdac1d5b9204c7bd6714fecc302850" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d29feb33e986b6ea906bd9c3559a856983f92371b3eaa5e83782a351623de0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "thread_local" version = "1.1.8" @@ -1224,6 +1394,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "url" version = "2.5.4" @@ -1233,6 +1409,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 024ec4068..93dcfd2ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ clap = { version = "4.5.41", features = ["derive"] } clap_complete = "4.5.55" git2 = "0.20.1" glob = "0.3.2" +oci-spec = "0.8.2" semver = { version = "1.0.26", features = ["serde"] } serde = { version = "1.0.217", features = ["derive"] } serde_json = "1.0.140" @@ -22,4 +23,4 @@ toml = "0.9.2" tracing = "0.1.41" tracing-indicatif = "0.3.9" tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } -url = "2.5.4" +url = { version = "2.5.4", features = ["serde"] } diff --git a/boil.toml b/boil.toml index 67cfac8be..2325681a5 100644 --- a/boil.toml +++ b/boil.toml @@ -3,3 +3,12 @@ STACKABLE_USER_NAME = "stackable" STACKABLE_USER_UID = "1000" STACKABLE_USER_GID = "1000" DELETE_CACHES = "true" + +[metadata] +documentation = "https://docs.stackable.tech/home/stable/" +source = "https://github.com/stackabletech/docker-images/" +authors = "Stackable GmbH " +vendor = "Stackable GmbH" +licenses = "Apache-2.0" + +[docker-config] diff --git a/rust/boil/Cargo.toml b/rust/boil/Cargo.toml index cb68a0201..ac2f516ea 100644 --- a/rust/boil/Cargo.toml +++ b/rust/boil/Cargo.toml @@ -8,6 +8,7 @@ clap.workspace = true clap_complete.workspace = true git2.workspace = true glob.workspace = true +oci-spec.workspace = true semver.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/rust/boil/src/build/bakefile.rs b/rust/boil/src/build/bakefile.rs index d903f2dc1..7c0bed35d 100644 --- a/rust/boil/src/build/bakefile.rs +++ b/rust/boil/src/build/bakefile.rs @@ -6,6 +6,11 @@ use std::{ }; use glob::glob; +use oci_spec::image::{ + ANNOTATION_AUTHORS, ANNOTATION_CREATED, ANNOTATION_DOCUMENTATION, ANNOTATION_LICENSES, + ANNOTATION_REVISION, ANNOTATION_SOURCE, ANNOTATION_VENDOR, ANNOTATION_VERSION, +}; +use semver::Version; use serde::Serialize; use snafu::{OptionExt, ResultExt, Snafu}; use time::format_description::well_known::Rfc3339; @@ -19,13 +24,10 @@ use crate::{ image::{Image, ImageConfig, ImageConfigError, ImageOptions, VersionOptionsPair}, platform::TargetPlatform, }, - config::Config, + config::{self, Config}, utils::{format_image_manifest_uri, format_image_repository_uri}, }; -pub const OPEN_CONTAINER_IMAGE_REVISION: &str = "org.opencontainers.image.revision"; -pub const OPEN_CONTAINER_IMAGE_CREATED: &str = "org.opencontainers.image.created"; - pub const ENTRY_TARGET_NAME_PREFIX: &str = "entry--"; #[derive(Debug, Snafu)] @@ -240,6 +242,9 @@ impl Bakefile { let mut bakefile_targets = BTreeMap::new(); let mut groups: BTreeMap = BTreeMap::new(); + let revision = Self::git_head_revision().context(GetRevisionSnafu)?; + let date_time = Self::now()?; + // TODO (@Techassi): Can we somehow optimize this to come by with minimal amount of // cloning, because we also need to clone on every loop iteration below. let mut docker_build_arguments = config.build_arguments; @@ -297,8 +302,6 @@ impl Bakefile { ); let dockerfile = PathBuf::new().join(&image_name).join("Dockerfile"); - let revision = Self::git_head_revision().context(GetRevisionSnafu)?; - let date_time = Self::now()?; let target_name = if is_entry { Self::format_entry_target_name(&image_name, &image_version) @@ -317,15 +320,24 @@ impl Bakefile { }) .collect(); + let annotations = BakefileTarget::annotations( + &date_time, + &revision, + &image_version, + &args.image_version, + &config.metadata, + ); + let labels = BakefileTarget::labels(date_time.clone(), revision.clone()); + let target = BakefileTarget { - annotations: BakefileTarget::annotations(&date_time, &revision), - labels: BakefileTarget::labels(date_time, revision), tags: vec![image_manifest_uri], arguments: docker_build_arguments, platforms: vec![args.target_platform.clone()], context: PathBuf::from("."), + annotations, dockerfile, contexts, + labels, }; bakefile_targets.insert(target_name, target); @@ -412,17 +424,43 @@ pub struct BakefileTarget { } impl BakefileTarget { - fn annotations(date_time: &str, revision: &str) -> Vec { + fn annotations( + date_time: &str, + revision: &str, + image_version: &str, + sdp_image_version: &Version, + global_metadata: &config::Metadata, + ) -> Vec { + let config::Metadata { + documentation, + licenses, + authors, + source, + vendor, + } = global_metadata; + + // Annotations describe OCI image components. vec![ - format!("{OPEN_CONTAINER_IMAGE_CREATED}={date_time}"), - format!("{OPEN_CONTAINER_IMAGE_REVISION}={revision}"), + format!("{ANNOTATION_CREATED}={date_time}"), + format!("{ANNOTATION_AUTHORS}={authors}"), + format!("{ANNOTATION_DOCUMENTATION}={documentation}"), + format!("{ANNOTATION_SOURCE}={source}"), + // TODO (@Techassi): Move this version formatting into a function + // TODO (@Techassi): Make this vendor agnostic, don't hard-code stackable here + format!("{ANNOTATION_VERSION}={image_version}-stackable{sdp_image_version}"), + format!("{ANNOTATION_REVISION}={revision}"), + format!("{ANNOTATION_VENDOR}={vendor}"), + format!("{ANNOTATION_LICENSES}={licenses}"), ] } fn labels(date_time: String, revision: String) -> BTreeMap { + // Labels describe Docker resources, and con be considered legacy. We + // should use annotations instead. These labels are only added to be + // consistent with `bake`. BTreeMap::from([ - (OPEN_CONTAINER_IMAGE_CREATED.to_owned(), date_time.clone()), - (OPEN_CONTAINER_IMAGE_REVISION.to_owned(), revision), + (ANNOTATION_CREATED.to_owned(), date_time.clone()), + (ANNOTATION_REVISION.to_owned(), revision), ("build-date".to_owned(), date_time), ]) } diff --git a/rust/boil/src/build/image.rs b/rust/boil/src/build/image.rs index 4f84e127b..cd5a7c6f5 100644 --- a/rust/boil/src/build/image.rs +++ b/rust/boil/src/build/image.rs @@ -85,6 +85,9 @@ pub enum ImageConfigError { #[derive(Debug, Deserialize)] pub struct ImageConfig { + // TODO (@Techassi): Eventually support this + // #[serde(default)] + // pub metadata: ImageMetadata, pub versions: ImageVersions, } diff --git a/rust/boil/src/config.rs b/rust/boil/src/config.rs index 64e16aee9..e65a0360a 100644 --- a/rust/boil/src/config.rs +++ b/rust/boil/src/config.rs @@ -2,6 +2,7 @@ use std::path::Path; use serde::Deserialize; use snafu::{ResultExt, Snafu}; +use url::Url; use crate::build::docker::BuildArguments; @@ -16,6 +17,7 @@ pub enum ConfigError { #[serde(rename_all = "kebab-case")] pub struct Config { pub build_arguments: BuildArguments, + pub metadata: Metadata, } impl Config { @@ -24,3 +26,17 @@ impl Config { toml::from_str(&contents).context(DeserializeSnafu) } } + +// NOTE (@Techassi): Think about if these metadata fields should be required or optional. If they +// are optional, the appropriate annotations are only emitted if set. +#[derive(Debug, Deserialize)] +pub struct Metadata { + pub documentation: Url, + pub licenses: String, + pub authors: String, + pub vendor: String, + pub source: Url, +} + +#[derive(Debug, Deserialize)] +pub struct DockerConfig {} From fe22240cc49ffa38b654855b6a6d7b1e66fc5ec5 Mon Sep 17 00:00:00 2001 From: Techassi Date: Sun, 17 Aug 2025 14:33:46 +0200 Subject: [PATCH 12/51] fix: Apply corrections from code review There were a bunch of small errors in the new boil-config.toml files. These are now fixed and should contain the same data as the versions.py files before. Co-authored-by: Xenia --- airflow/boil-config.toml | 2 +- hadoop/boil-config.toml | 1 - kafka/boil-config.toml | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/airflow/boil-config.toml b/airflow/boil-config.toml index add960930..652ee1a70 100644 --- a/airflow/boil-config.toml +++ b/airflow/boil-config.toml @@ -51,5 +51,5 @@ s3fs-version = "2024.9.0" cyclonedx-bom-version = "6.0.0" tini-version = "0.19.0" uv-version = "0.7.8" -airflow-extras = "async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,ldap,google,google_auth,microsoft.azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv,trino" +airflow-extras = "async,amazon,celery,cncf-kubernetes,docker,elasticsearch,fab,ftp,grpc,hashicorp,http,ldap,google,microsoft-azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,trino" opa-auth-manager = "airflow-3" diff --git a/hadoop/boil-config.toml b/hadoop/boil-config.toml index b0d0641f2..cccf2aa78 100644 --- a/hadoop/boil-config.toml +++ b/hadoop/boil-config.toml @@ -7,7 +7,6 @@ java-devel = "11" [versions."3.3.6".build-arguments] async-profiler-version = "2.9" jmx-exporter-version = "1.3.0" -protobuf-version = "3.7.1" hdfs-utils-version = "0.4.0" [versions."3.4.1".local-images] diff --git a/kafka/boil-config.toml b/kafka/boil-config.toml index 5347e10e4..9d7fc2ad1 100644 --- a/kafka/boil-config.toml +++ b/kafka/boil-config.toml @@ -29,8 +29,8 @@ scala-version = "2.13" jmx-exporter-version = "1.3.0" [versions."4.0.0".local-images] -java-base = "21" -java-devel = "21" +java-base = "23" +java-devel = "23" "kafka/kcat" = "1.7.0" "kafka/kafka-opa-plugin" = "1.5.1" From 068784d02b9f6c28a2e3fdcc2bb372e56dc664d8 Mon Sep 17 00:00:00 2001 From: Techassi Date: Mon, 18 Aug 2025 10:27:48 +0200 Subject: [PATCH 13/51] chore: Remove protobuf-version arg from Hadoop boil config --- hadoop/boil-config.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop/boil-config.toml b/hadoop/boil-config.toml index cccf2aa78..6636ae943 100644 --- a/hadoop/boil-config.toml +++ b/hadoop/boil-config.toml @@ -17,5 +17,4 @@ java-devel = "11" [versions."3.4.1".build-arguments] async-profiler-version = "2.9" jmx-exporter-version = "1.3.0" -protobuf-version = "3.7.1" hdfs-utils-version = "0.4.1" From 186bd223ace953999e5a942ea40491ed1872b550 Mon Sep 17 00:00:00 2001 From: Techassi Date: Tue, 19 Aug 2025 22:05:00 +0200 Subject: [PATCH 14/51] feat: List images to show, move CLI arguments --- rust/boil/src/build/bakefile.rs | 10 ++-- rust/boil/src/build/cli.rs | 30 ++++++++---- rust/boil/src/cli.rs | 43 ++---------------- rust/boil/src/completions/mod.rs | 11 ++++- rust/boil/src/main.rs | 7 ++- rust/boil/src/show/images.rs | 30 ------------ rust/boil/src/show/images/cli.rs | 13 ++++++ rust/boil/src/show/images/mod.rs | 78 ++++++++++++++++++++++++++++++++ rust/boil/src/show/mod.rs | 16 +++++++ 9 files changed, 150 insertions(+), 88 deletions(-) delete mode 100644 rust/boil/src/show/images.rs create mode 100644 rust/boil/src/show/images/cli.rs create mode 100644 rust/boil/src/show/images/mod.rs diff --git a/rust/boil/src/build/bakefile.rs b/rust/boil/src/build/bakefile.rs index 7c0bed35d..67cef7c0f 100644 --- a/rust/boil/src/build/bakefile.rs +++ b/rust/boil/src/build/bakefile.rs @@ -123,12 +123,10 @@ impl Targets { targets.insert_targets(image_name.to_owned(), pairs, &options, true)?; } - println!("{targets:#?}"); - Ok(targets) } - pub fn from_images(images: &[Image], options: TargetsOptions) -> Result { + pub fn set(images: &[Image], options: TargetsOptions) -> Result { let mut targets = Self::default(); for image in images { @@ -214,9 +212,9 @@ impl Bakefile { /// This will only create targets for selected entry images and their dependencies. There is no /// need to filter anything out afterwards. The filtering is done automatically internally. pub fn from_args(args: &cli::BuildArguments, config: Config) -> Result { - let graph = Targets::from_images(&args.images, TargetsOptions::default()) - .context(CreateGraphSnafu)?; - Self::from_targets(graph, args, config) + let targets = + Targets::set(&args.images, TargetsOptions::default()).context(CreateGraphSnafu)?; + Self::from_targets(targets, args, config) } /// Returns all image manifest URIs for entry images. diff --git a/rust/boil/src/build/cli.rs b/rust/boil/src/build/cli.rs index 31530355e..5465c9e1a 100644 --- a/rust/boil/src/build/cli.rs +++ b/rust/boil/src/build/cli.rs @@ -1,16 +1,14 @@ -use std::path::PathBuf; +use std::{path::PathBuf, str::FromStr}; use clap::{Args, ValueHint, value_parser}; use semver::Version; +use snafu::{ResultExt, Snafu, ensure}; use url::Host; -use crate::{ - build::{ - docker::BuildArgument, - image::Image, - platform::{Architecture, TargetPlatform}, - }, - cli::parse_image_version, +use crate::build::{ + docker::BuildArgument, + image::Image, + platform::{Architecture, TargetPlatform}, }; #[derive(Debug, Args)] @@ -124,3 +122,19 @@ impl BuildArguments { PathBuf::from("Dockerfile") } } + +#[derive(Debug, Snafu)] +pub enum ParseImageVersionError { + #[snafu(display("failed to parse semantic version"))] + ParseVersion { source: semver::Error }, + + #[snafu(display("semantic version must not contain build metadata"))] + ContainsBuildMetadata, +} + +pub fn parse_image_version(input: &str) -> Result { + let version = Version::from_str(input).context(ParseVersionSnafu)?; + ensure!(version.build.is_empty(), ContainsBuildMetadataSnafu); + + Ok(version) +} diff --git a/rust/boil/src/cli.rs b/rust/boil/src/cli.rs index 5914fc3c2..04ea44efb 100644 --- a/rust/boil/src/cli.rs +++ b/rust/boil/src/cli.rs @@ -1,11 +1,8 @@ -use std::{path::PathBuf, str::FromStr}; +use std::path::PathBuf; -use clap::{Args, Parser, Subcommand}; -use clap_complete::Shell; -use semver::Version; -use snafu::{ResultExt, Snafu, ensure}; +use clap::{Parser, Subcommand}; -use crate::build::cli::BuildArguments; +use crate::{build::cli::BuildArguments, completions::CompletionsArguments, show::ShowArguments}; #[derive(Debug, Parser)] #[command(author, version, about)] @@ -53,37 +50,3 @@ pub enum Command { /// Generate shell completions. Completions(CompletionsArguments), } - -#[derive(Debug, Args)] -pub struct ShowArguments { - #[command(subcommand)] - pub commands: ShowCommand, -} - -#[derive(Debug, Subcommand)] -pub enum ShowCommand { - Images, - Tree, -} - -#[derive(Debug, Args)] -pub struct CompletionsArguments { - /// Shell to generate completions for. - pub shell: Shell, -} - -#[derive(Debug, Snafu)] -pub enum ParseImageVersionError { - #[snafu(display("failed to parse semantic version"))] - ParseVersion { source: semver::Error }, - - #[snafu(display("semantic version must not contain build metadata"))] - ContainsBuildMetadata, -} - -pub fn parse_image_version(input: &str) -> Result { - let version = Version::from_str(input).context(ParseVersionSnafu)?; - ensure!(version.build.is_empty(), ContainsBuildMetadataSnafu); - - Ok(version) -} diff --git a/rust/boil/src/completions/mod.rs b/rust/boil/src/completions/mod.rs index 14518365a..070d3a469 100644 --- a/rust/boil/src/completions/mod.rs +++ b/rust/boil/src/completions/mod.rs @@ -1,6 +1,13 @@ -use clap::CommandFactory; +use clap::{Args, CommandFactory}; +use clap_complete::Shell; -use crate::cli::{Cli, CompletionsArguments}; +use crate::cli::Cli; + +#[derive(Debug, Args)] +pub struct CompletionsArguments { + /// Shell to generate completions for. + pub shell: Shell, +} pub fn run_command(arguments: CompletionsArguments) { let mut cli = Cli::command(); diff --git a/rust/boil/src/main.rs b/rust/boil/src/main.rs index fadb093d6..25bf2c3ec 100644 --- a/rust/boil/src/main.rs +++ b/rust/boil/src/main.rs @@ -3,8 +3,9 @@ use semver::Version; use snafu::{ResultExt, Snafu}; use crate::{ - cli::{Cli, Command, ShowCommand}, + cli::{Cli, Command}, config::Config, + show::ShowCommand, }; // Common modules @@ -83,7 +84,9 @@ async fn main() -> Result<(), Error> { build::run_command(arguments, config).context(BuildSnafu) } Command::Show(arguments) => match arguments.commands { - ShowCommand::Images => show::images::run_command().context(ShowSnafu), + ShowCommand::Images(arguments) => { + show::images::run_command(arguments).context(ShowSnafu) + } ShowCommand::Tree => todo!(), }, Command::Completions(arguments) => { diff --git a/rust/boil/src/show/images.rs b/rust/boil/src/show/images.rs deleted file mode 100644 index 5af7daf16..000000000 --- a/rust/boil/src/show/images.rs +++ /dev/null @@ -1,30 +0,0 @@ -use std::collections::BTreeMap; - -use snafu::{ResultExt, Snafu}; - -use crate::build::bakefile::{Targets, TargetsError, TargetsOptions}; - -#[derive(Debug, Snafu)] -pub enum Error { - #[snafu(display("failed to serialize list as JSON"))] - SerializeList { source: serde_json::Error }, - - #[snafu(display("failed to build list of targets"))] - BuildTargets { source: TargetsError }, -} - -pub fn run_command() -> Result<(), Error> { - let list: BTreeMap<_, _> = Targets::all(TargetsOptions { only_entry: true }) - .context(BuildTargetsSnafu)? - .into_iter() - .map(|(image_name, image_versions)| { - let versions: Vec<_> = image_versions - .into_iter() - .map(|(image_version, (_, _))| image_version) - .collect(); - (image_name, versions) - }) - .collect(); - - serde_json::to_writer_pretty(std::io::stdout(), &list).context(SerializeListSnafu) -} diff --git a/rust/boil/src/show/images/cli.rs b/rust/boil/src/show/images/cli.rs new file mode 100644 index 000000000..e4ee4c837 --- /dev/null +++ b/rust/boil/src/show/images/cli.rs @@ -0,0 +1,13 @@ +use clap::Args; + +use crate::build::image::Image; + +#[derive(Debug, Args)] +pub struct ShowImagesArguments { + /// Optionally specify one or more images to display. + pub image: Vec, + + /// Pretty print the structured output. + #[arg(long)] + pub pretty: bool, +} diff --git a/rust/boil/src/show/images/mod.rs b/rust/boil/src/show/images/mod.rs new file mode 100644 index 000000000..b289c222d --- /dev/null +++ b/rust/boil/src/show/images/mod.rs @@ -0,0 +1,78 @@ +use std::collections::BTreeMap; + +use serde::{Serialize, ser::SerializeSeq}; +use snafu::{ResultExt, Snafu}; + +use crate::{ + build::bakefile::{Targets, TargetsError, TargetsOptions}, + show::images::cli::ShowImagesArguments, +}; + +pub mod cli; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("failed to serialize list as JSON"))] + SerializeList { source: serde_json::Error }, + + #[snafu(display("failed to build list of targets"))] + BuildTargets { source: TargetsError }, +} + +// NOTE (@Techassi): I don't know if I like this... but this makes the stdout output very convient +// to consume. +struct OneOrMany(BTreeMap>); + +impl Serialize for OneOrMany { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + if self.0.len() == 1 { + let mut seq = serializer.serialize_seq(Some(1))?; + for entry in &self.0 { + for version in entry.1 { + seq.serialize_element(&version)?; + } + } + + Ok(seq.end()?) + } else { + self.0.serialize(serializer) + } + } +} + +pub fn run_command(arguments: ShowImagesArguments) -> Result<(), Error> { + let list: BTreeMap<_, _> = if arguments.image.is_empty() { + Targets::all(TargetsOptions { only_entry: true }) + .context(BuildTargetsSnafu)? + .into_iter() + } else { + Targets::set(&arguments.image, TargetsOptions { only_entry: true }) + .context(BuildTargetsSnafu)? + .into_iter() + } + .map(|(image_name, image_versions)| { + let versions: Vec<_> = image_versions + .into_iter() + .map(|(image_version, (_, _))| image_version) + .collect(); + (image_name, versions) + }) + .collect(); + + print_to_stdout(list, arguments.pretty) +} + +fn print_to_stdout(list: BTreeMap>, pretty: bool) -> Result<(), Error> { + let stdout = std::io::stdout(); + + let list = OneOrMany(list); + + if pretty { + serde_json::to_writer_pretty(stdout, &list).context(SerializeListSnafu) + } else { + serde_json::to_writer(stdout, &list).context(SerializeListSnafu) + } +} diff --git a/rust/boil/src/show/mod.rs b/rust/boil/src/show/mod.rs index 8f0da9f8e..0be38bffe 100644 --- a/rust/boil/src/show/mod.rs +++ b/rust/boil/src/show/mod.rs @@ -1 +1,17 @@ +use clap::{Args, Subcommand}; + +use crate::show::images::cli::ShowImagesArguments; + pub mod images; + +#[derive(Debug, Args)] +pub struct ShowArguments { + #[command(subcommand)] + pub commands: ShowCommand, +} + +#[derive(Debug, Subcommand)] +pub enum ShowCommand { + Images(ShowImagesArguments), + Tree, +} From d189810c434ea0f10eb49bbadcd64dabe900668f Mon Sep 17 00:00:00 2001 From: Techassi Date: Wed, 20 Aug 2025 14:40:15 +0200 Subject: [PATCH 15/51] chore: Add example to README --- rust/boil/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rust/boil/README.md b/rust/boil/README.md index a53c2f3af..35ae3804b 100644 --- a/rust/boil/README.md +++ b/rust/boil/README.md @@ -28,6 +28,9 @@ boil build airflow opa # output boil show images +# Display a list of versions of the image located in the 'airflow' folder +boil show images airflow + # Soon (hopefully) implemented boil show graph ``` From 8ea62b48a3516888c1dcd5463ba7fa22c4f97af7 Mon Sep 17 00:00:00 2001 From: Techassi Date: Sun, 24 Aug 2025 21:43:38 +0200 Subject: [PATCH 16/51] feat: Move shared data into common target which targets inherit from --- rust/boil/src/build/bakefile.rs | 157 +++++++++++++++++++++----------- rust/boil/src/build/docker.rs | 4 + 2 files changed, 110 insertions(+), 51 deletions(-) diff --git a/rust/boil/src/build/bakefile.rs b/rust/boil/src/build/bakefile.rs index 67cef7c0f..936c1e177 100644 --- a/rust/boil/src/build/bakefile.rs +++ b/rust/boil/src/build/bakefile.rs @@ -28,6 +28,7 @@ use crate::{ utils::{format_image_manifest_uri, format_image_repository_uri}, }; +pub const COMMON_TARGET_NAME: &str = "common--target"; pub const ENTRY_TARGET_NAME_PREFIX: &str = "entry--"; #[derive(Debug, Snafu)] @@ -232,6 +233,22 @@ impl Bakefile { .collect() } + /// Creates the common target, containing shared data, which will be inherited by other targets. + fn common_target(args: &cli::BuildArguments, config: Config) -> Result { + let revision = Self::git_head_revision().context(GetRevisionSnafu)?; + let date_time = Self::now()?; + + let target = BakefileTarget::common( + date_time, + revision, + config, + args.docker_build_arguments.clone(), + args.image_version.base_prerelease(), + ); + + Ok(target) + } + fn from_targets( targets: Targets, args: &cli::BuildArguments, @@ -240,23 +257,15 @@ impl Bakefile { let mut bakefile_targets = BTreeMap::new(); let mut groups: BTreeMap = BTreeMap::new(); - let revision = Self::git_head_revision().context(GetRevisionSnafu)?; - let date_time = Self::now()?; - - // TODO (@Techassi): Can we somehow optimize this to come by with minimal amount of - // cloning, because we also need to clone on every loop iteration below. - let mut docker_build_arguments = config.build_arguments; - docker_build_arguments.extend(args.docker_build_arguments.clone()); - docker_build_arguments.insert(BuildArgument::new( - "RELEASE_VERSION".to_owned(), - args.image_version.base_prerelease(), - )); + // Create a common target, which contains shared data, like annotations, arguments, labels, etc... + let common_target = Self::common_target(args, config)?; + bakefile_targets.insert(COMMON_TARGET_NAME.to_owned(), common_target); - for (image_name, image_versions) in targets.0.into_iter() { + for (image_name, image_versions) in targets.into_iter() { for (image_version, (image_options, is_entry)) in image_versions { // TODO (@Techassi): Clean this up // TODO (@Techassi): Move the arg formatting into functions - let mut docker_build_arguments = docker_build_arguments.clone(); + let mut build_arguments = BuildArguments::new(); let local_version_docker_args: Vec<_> = image_options .local_images @@ -272,9 +281,10 @@ impl Bakefile { }) .collect(); - docker_build_arguments.extend(image_options.build_arguments.clone()); - docker_build_arguments.extend(local_version_docker_args); - docker_build_arguments.insert(BuildArgument::new( + build_arguments.extend(image_options.build_arguments); + build_arguments.extend(local_version_docker_args); + // TODO (@Techassi): Rename this to IMAGE_VERSION + build_arguments.insert(BuildArgument::new( "PRODUCT_VERSION".to_owned(), image_version.to_string(), )); @@ -318,24 +328,20 @@ impl Bakefile { }) .collect(); - let annotations = BakefileTarget::annotations( - &date_time, - &revision, - &image_version, - &args.image_version, - &config.metadata, - ); - let labels = BakefileTarget::labels(date_time.clone(), revision.clone()); + let annotations = + BakefileTarget::image_version_annotation(&image_version, &args.image_version); let target = BakefileTarget { tags: vec![image_manifest_uri], - arguments: docker_build_arguments, + arguments: build_arguments, platforms: vec![args.target_platform.clone()], - context: PathBuf::from("."), + // NOTE (@Techassi): Should this instead be scoped to the folder of the image we build + context: Some(PathBuf::from(".")), + dockerfile: Some(dockerfile), + inherits: vec![COMMON_TARGET_NAME.to_owned()], annotations, - dockerfile, contexts, - labels, + ..Default::default() }; bakefile_targets.insert(target_name, target); @@ -404,63 +410,112 @@ impl Bakefile { // TODO (@Techassi): Figure out of we can use borrowed data in here. This would avoid a whole bunch // of cloning. -#[derive(Debug, Serialize)] +#[derive(Debug, Default, Serialize)] pub struct BakefileTarget { + /// Defines build arguments for the target. + #[serde(rename = "args", skip_serializing_if = "BuildArguments::is_empty")] + pub arguments: BuildArguments, + + /// Adds annotations to images built with bake. + #[serde(skip_serializing_if = "Vec::is_empty")] pub annotations: Vec, - pub context: PathBuf, + /// Specifies the location of the build context to use for this target. + /// + /// Accepts a URL or a directory path. + #[serde(skip_serializing_if = "Option::is_none")] + pub context: Option, + + /// Additional build contexts. + /// + /// This attribute takes a map, where keys result in named contexts that you can reference in + /// your builds. #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub contexts: BTreeMap, - pub dockerfile: PathBuf, - #[serde(rename = "args", skip_serializing_if = "BuildArguments::is_empty")] - pub arguments: BuildArguments, + /// Name of the Dockerfile to use for the build. + #[serde(skip_serializing_if = "Option::is_none")] + pub dockerfile: Option, + + /// A target can inherit attributes from other targets. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub inherits: Vec, + /// Assigns image labels to the build. + #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub labels: BTreeMap, - pub tags: Vec, + + // TODO (@Techassi): Explore how we can build multiple platforms at once + /// Set target platforms for the build target. + /// + /// Technically, multiple architectures can be listed in here, but boil chooses to build only + /// one architecture at a time. + #[serde(skip_serializing_if = "Vec::is_empty")] pub platforms: Vec, + + /// Image names and tags to use for the build target. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub tags: Vec, } impl BakefileTarget { - fn annotations( - date_time: &str, - revision: &str, - image_version: &str, - sdp_image_version: &Version, - global_metadata: &config::Metadata, - ) -> Vec { + fn common( + date_time: String, + revision: String, + config: Config, + docker_build_arguments: Vec, + release_version: String, + ) -> Self { let config::Metadata { documentation, licenses, authors, source, vendor, - } = global_metadata; + } = config.metadata; // Annotations describe OCI image components. - vec![ + let annotations = vec![ format!("{ANNOTATION_CREATED}={date_time}"), format!("{ANNOTATION_AUTHORS}={authors}"), format!("{ANNOTATION_DOCUMENTATION}={documentation}"), format!("{ANNOTATION_SOURCE}={source}"), - // TODO (@Techassi): Move this version formatting into a function - // TODO (@Techassi): Make this vendor agnostic, don't hard-code stackable here - format!("{ANNOTATION_VERSION}={image_version}-stackable{sdp_image_version}"), format!("{ANNOTATION_REVISION}={revision}"), format!("{ANNOTATION_VENDOR}={vendor}"), format!("{ANNOTATION_LICENSES}={licenses}"), - ] - } + ]; + + let mut arguments = config.build_arguments; + arguments.extend(docker_build_arguments); + arguments.insert(BuildArgument::new( + "RELEASE_VERSION".to_owned(), + release_version, + )); - fn labels(date_time: String, revision: String) -> BTreeMap { // Labels describe Docker resources, and con be considered legacy. We // should use annotations instead. These labels are only added to be // consistent with `bake`. - BTreeMap::from([ + let labels = BTreeMap::from([ (ANNOTATION_CREATED.to_owned(), date_time.clone()), (ANNOTATION_REVISION.to_owned(), revision), ("build-date".to_owned(), date_time), - ]) + ]); + + Self { + annotations, + arguments, + labels, + ..Default::default() + } + } + + fn image_version_annotation(image_version: &str, sdp_image_version: &Version) -> Vec { + // Annotations describe OCI image components. + vec![ + // TODO (@Techassi): Move this version formatting into a function + // TODO (@Techassi): Make this vendor agnostic, don't hard-code stackable here + format!("{ANNOTATION_VERSION}={image_version}-stackable{sdp_image_version}"), + ] } } diff --git a/rust/boil/src/build/docker.rs b/rust/boil/src/build/docker.rs index 70fa77657..575fb931d 100644 --- a/rust/boil/src/build/docker.rs +++ b/rust/boil/src/build/docker.rs @@ -154,6 +154,10 @@ impl Serialize for BuildArguments { } impl BuildArguments { + pub fn new() -> Self { + Self(BTreeSet::new()) + } + pub fn is_empty(&self) -> bool { self.0.is_empty() } From 38d1a2fa81f1c03c2885bc5c60c9ac7c285a2337 Mon Sep 17 00:00:00 2001 From: Techassi Date: Mon, 25 Aug 2025 15:43:27 +0200 Subject: [PATCH 17/51] chore: Add rust-toolchain file --- rust-toolchain.toml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 rust-toolchain.toml diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 000000000..291696d0e --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.87.0" +profile = "default" From 45eec1d0b0954c1b3ead741b1a5e8afae5c88e1a Mon Sep 17 00:00:00 2001 From: Techassi Date: Mon, 1 Sep 2025 14:21:11 +0200 Subject: [PATCH 18/51] chore: Make config arg global --- rust/boil/src/cli.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/boil/src/cli.rs b/rust/boil/src/cli.rs index 04ea44efb..c78fcf8b9 100644 --- a/rust/boil/src/cli.rs +++ b/rust/boil/src/cli.rs @@ -8,7 +8,7 @@ use crate::{build::cli::BuildArguments, completions::CompletionsArguments, show: #[command(author, version, about)] pub struct Cli { /// Path to the configuration file. - #[arg(short = 'c', long = "configuration", default_value_os_t = Self::default_config_path())] + #[arg(short, long = "configuration", global = true, default_value_os_t = Self::default_config_path())] pub config_path: PathBuf, /// Path to the OpenShift configuration file. From 43e7745161ce16d27f17c2c2bf54feece4273ada Mon Sep 17 00:00:00 2001 From: Techassi Date: Mon, 1 Sep 2025 14:25:10 +0200 Subject: [PATCH 19/51] feat: Add --strip-architecture arg --- rust/boil/src/build/bakefile.rs | 1 + rust/boil/src/build/cli.rs | 4 ++++ rust/boil/src/utils.rs | 9 ++++++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/rust/boil/src/build/bakefile.rs b/rust/boil/src/build/bakefile.rs index 936c1e177..287654dbe 100644 --- a/rust/boil/src/build/bakefile.rs +++ b/rust/boil/src/build/bakefile.rs @@ -307,6 +307,7 @@ impl Bakefile { &image_version, &args.image_version, args.target_platform.architecture(), + args.strip_architecture, ); let dockerfile = PathBuf::new().join(&image_name).join("Dockerfile"); diff --git a/rust/boil/src/build/cli.rs b/rust/boil/src/build/cli.rs index 5465c9e1a..9681c91a9 100644 --- a/rust/boil/src/build/cli.rs +++ b/rust/boil/src/build/cli.rs @@ -96,6 +96,10 @@ pub struct BuildArguments { )] pub export_image_manifest_uris: Option, + /// Strips the architecture from the image (index) manifest tag. + #[arg(long, help_heading = "Build Options")] + pub strip_architecture: bool, + /// Loads the image into the local image store. #[arg(long, help_heading = "Build Options")] pub load: bool, diff --git a/rust/boil/src/utils.rs b/rust/boil/src/utils.rs index def0f0320..f3b4e4ea7 100644 --- a/rust/boil/src/utils.rs +++ b/rust/boil/src/utils.rs @@ -20,8 +20,15 @@ pub fn format_image_manifest_uri( image_version: &str, sdp_image_version: &Version, architecture: &Architecture, + strip_architecture: bool, ) -> String { - format!("{image_repository_uri}:{image_version}-stackable{sdp_image_version}-{architecture}") + if strip_architecture { + format!("{image_repository_uri}:{image_version}-stackable{sdp_image_version}") + } else { + format!( + "{image_repository_uri}:{image_version}-stackable{sdp_image_version}-{architecture}" + ) + } } pub trait CommandExt { From 8d991469dda271d5d2d022f18b0c0adafc1bb600 Mon Sep 17 00:00:00 2001 From: Techassi Date: Mon, 1 Sep 2025 16:01:15 +0200 Subject: [PATCH 20/51] ci: Add workflow to release boil --- .github/workflows/boil_build.yaml | 49 +++++++++++++++++++++++++++++ .github/workflows/boil_release.yaml | 37 ++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 .github/workflows/boil_build.yaml create mode 100644 .github/workflows/boil_release.yaml diff --git a/.github/workflows/boil_build.yaml b/.github/workflows/boil_build.yaml new file mode 100644 index 000000000..bbe6f6c36 --- /dev/null +++ b/.github/workflows/boil_build.yaml @@ -0,0 +1,49 @@ +--- +name: Build boil + +on: + workflow_call: + inputs: + os: + required: true + type: string + target: + required: true + type: string + upload: + default: false + type: boolean + +env: + RUST_VERSION: 1.87.0 + +jobs: + build: + name: Build boil-${{ inputs.target }} + runs-on: ${{ inputs.os }} + steps: + - name: Checkout + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + + - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 + with: + toolchain: ${{ env.RUST_VERSION }} + targets: ${{ inputs.target }} + + - name: Build Binary + env: + TARGET: ${{ inputs.target }} + run: cargo build --target "$TARGET" --release --package boil + + - name: Rename Binary + env: + TARGET: ${{ inputs.target }} + run: mv "target/$TARGET/release/boil" "boil-$TARGET" + + - name: Upload Artifact + if: inputs.upload + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + path: boil-${{ inputs.target }} diff --git a/.github/workflows/boil_release.yaml b/.github/workflows/boil_release.yaml new file mode 100644 index 000000000..9a6a01be5 --- /dev/null +++ b/.github/workflows/boil_release.yaml @@ -0,0 +1,37 @@ +--- +name: Release boil + +on: + push: + tags: + - "boil-[0-9]+.[0-9]+.[0-9]+**" + +jobs: + build: + uses: ./.github/workflows/boil_build.yaml + with: + upload: true + target: ${{ matrix.targets.target }} + os: ${{ matrix.targets.os }} + + strategy: + fail-fast: false + matrix: + targets: + - {target: aarch64-unknown-linux-gnu, os: ubuntu-24.04-arm} + - {target: x86_64-unknown-linux-gnu, os: ubuntu-latest} + - {target: aarch64-apple-darwin, os: macos-latest} + - {target: x86_64-apple-darwin, os: macos-latest} + release: + runs-on: ubuntu-latest + needs: [build] + steps: + - name: Download Artifacts + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + with: + path: artifacts + + - name: Upload Release Binary + uses: softprops/action-gh-release@72f2c25fcb47643c292f7107632f7a47c1df5cd8 # v2.3.2 + with: + files: artifacts/artifact/* From 2565cb86439670dae50d1531a74a4ad939005084 Mon Sep 17 00:00:00 2001 From: Techassi Date: Mon, 1 Sep 2025 16:19:05 +0200 Subject: [PATCH 21/51] ci: Also build boil on PR (without releasing it) --- .../{boil_release.yaml => boil_build_release.yaml} | 12 ++++++++++-- .../{boil_build.yaml => boil_reusable_build.yaml} | 2 -- 2 files changed, 10 insertions(+), 4 deletions(-) rename .github/workflows/{boil_release.yaml => boil_build_release.yaml} (78%) rename .github/workflows/{boil_build.yaml => boil_reusable_build.yaml} (98%) diff --git a/.github/workflows/boil_release.yaml b/.github/workflows/boil_build_release.yaml similarity index 78% rename from .github/workflows/boil_release.yaml rename to .github/workflows/boil_build_release.yaml index 9a6a01be5..c727bc16c 100644 --- a/.github/workflows/boil_release.yaml +++ b/.github/workflows/boil_build_release.yaml @@ -1,14 +1,20 @@ --- -name: Release boil +name: Build/Release boil on: + pull_request: + paths: + - '.github/workflows/boil_build.yaml' + - 'rust-toolchain.toml' + - 'Cargo.*' + - '**.rs' push: tags: - "boil-[0-9]+.[0-9]+.[0-9]+**" jobs: build: - uses: ./.github/workflows/boil_build.yaml + uses: ./.github/workflows/boil_reusable_build.yaml with: upload: true target: ${{ matrix.targets.target }} @@ -22,7 +28,9 @@ jobs: - {target: x86_64-unknown-linux-gnu, os: ubuntu-latest} - {target: aarch64-apple-darwin, os: macos-latest} - {target: x86_64-apple-darwin, os: macos-latest} + release: + if: github.event_name == 'push' runs-on: ubuntu-latest needs: [build] steps: diff --git a/.github/workflows/boil_build.yaml b/.github/workflows/boil_reusable_build.yaml similarity index 98% rename from .github/workflows/boil_build.yaml rename to .github/workflows/boil_reusable_build.yaml index bbe6f6c36..7dd4b7a13 100644 --- a/.github/workflows/boil_build.yaml +++ b/.github/workflows/boil_reusable_build.yaml @@ -1,6 +1,4 @@ --- -name: Build boil - on: workflow_call: inputs: From 02c6a576e7a5e1cf265d741cc2dcdff2a22ac74c Mon Sep 17 00:00:00 2001 From: Techassi Date: Mon, 1 Sep 2025 16:44:48 +0200 Subject: [PATCH 22/51] ci: Remove reusable workflow --- .github/workflows/boil_build_release.yaml | 54 +++++++++++++++------- .github/workflows/boil_reusable_build.yaml | 47 ------------------- 2 files changed, 37 insertions(+), 64 deletions(-) delete mode 100644 .github/workflows/boil_reusable_build.yaml diff --git a/.github/workflows/boil_build_release.yaml b/.github/workflows/boil_build_release.yaml index c727bc16c..19b05d732 100644 --- a/.github/workflows/boil_build_release.yaml +++ b/.github/workflows/boil_build_release.yaml @@ -4,7 +4,7 @@ name: Build/Release boil on: pull_request: paths: - - '.github/workflows/boil_build.yaml' + - '.github/workflows/boil_build_release.yaml' - 'rust-toolchain.toml' - 'Cargo.*' - '**.rs' @@ -12,14 +12,21 @@ on: tags: - "boil-[0-9]+.[0-9]+.[0-9]+**" +env: + RUST_VERSION: 1.87.0 + jobs: - build: - uses: ./.github/workflows/boil_reusable_build.yaml - with: - upload: true - target: ${{ matrix.targets.target }} - os: ${{ matrix.targets.os }} + create-release: + name: Create Draft Release + if: github.event_name == 'push' + runs-on: ubuntu-latest + steps: + - name: Create Draft Release + uses: softprops/action-gh-release@72f2c25fcb47643c292f7107632f7a47c1df5cd8 # v2.3.2 + with: + draft: true + build: strategy: fail-fast: false matrix: @@ -28,18 +35,31 @@ jobs: - {target: x86_64-unknown-linux-gnu, os: ubuntu-latest} - {target: aarch64-apple-darwin, os: macos-latest} - {target: x86_64-apple-darwin, os: macos-latest} - - release: - if: github.event_name == 'push' - runs-on: ubuntu-latest - needs: [build] + runs-on: ${{ matrix.targets.os }} steps: - - name: Download Artifacts - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + - name: Checkout + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: - path: artifacts + persist-credentials: false + + - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 + with: + toolchain: ${{ env.RUST_VERSION }} + targets: ${{ matrix.targets.target }} + + - name: Build Binary + env: + TARGET: ${{ matrix.targets.target }} + run: cargo build --target "$TARGET" --release --package boil + + - name: Rename Binary + env: + TARGET: ${{ matrix.targets.target }} + run: mv "target/$TARGET/release/boil" "boil-$TARGET" - - name: Upload Release Binary + - name: Upload Artifact to Release + if: github.event_name == 'push' uses: softprops/action-gh-release@72f2c25fcb47643c292f7107632f7a47c1df5cd8 # v2.3.2 with: - files: artifacts/artifact/* + draft: false + files: boil-${{ matrix.targets.target }} diff --git a/.github/workflows/boil_reusable_build.yaml b/.github/workflows/boil_reusable_build.yaml deleted file mode 100644 index 7dd4b7a13..000000000 --- a/.github/workflows/boil_reusable_build.yaml +++ /dev/null @@ -1,47 +0,0 @@ ---- -on: - workflow_call: - inputs: - os: - required: true - type: string - target: - required: true - type: string - upload: - default: false - type: boolean - -env: - RUST_VERSION: 1.87.0 - -jobs: - build: - name: Build boil-${{ inputs.target }} - runs-on: ${{ inputs.os }} - steps: - - name: Checkout - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - persist-credentials: false - - - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 - with: - toolchain: ${{ env.RUST_VERSION }} - targets: ${{ inputs.target }} - - - name: Build Binary - env: - TARGET: ${{ inputs.target }} - run: cargo build --target "$TARGET" --release --package boil - - - name: Rename Binary - env: - TARGET: ${{ inputs.target }} - run: mv "target/$TARGET/release/boil" "boil-$TARGET" - - - name: Upload Artifact - if: inputs.upload - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - path: boil-${{ inputs.target }} From c10d03a8f33b826f9a17bd7e0271d30a81eda764 Mon Sep 17 00:00:00 2001 From: Techassi Date: Mon, 1 Sep 2025 16:53:45 +0200 Subject: [PATCH 23/51] ci: Only build boil for aarch64 on macOS --- .github/workflows/boil_build_release.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/boil_build_release.yaml b/.github/workflows/boil_build_release.yaml index 19b05d732..982effc5b 100644 --- a/.github/workflows/boil_build_release.yaml +++ b/.github/workflows/boil_build_release.yaml @@ -34,7 +34,6 @@ jobs: - {target: aarch64-unknown-linux-gnu, os: ubuntu-24.04-arm} - {target: x86_64-unknown-linux-gnu, os: ubuntu-latest} - {target: aarch64-apple-darwin, os: macos-latest} - - {target: x86_64-apple-darwin, os: macos-latest} runs-on: ${{ matrix.targets.os }} steps: - name: Checkout From 288f80199a3aa9ab80b744682317ea549cf417e3 Mon Sep 17 00:00:00 2001 From: Techassi Date: Wed, 10 Sep 2025 16:51:07 +0200 Subject: [PATCH 24/51] chore: Remove the term "product" from the Rust source code --- rust/boil/src/build/bakefile.rs | 32 ++++++++++++++++---------------- rust/boil/src/build/cli.rs | 2 +- rust/boil/src/build/mod.rs | 2 -- rust/boil/src/cli.rs | 2 +- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/rust/boil/src/build/bakefile.rs b/rust/boil/src/build/bakefile.rs index 287654dbe..612a40cbd 100644 --- a/rust/boil/src/build/bakefile.rs +++ b/rust/boil/src/build/bakefile.rs @@ -57,8 +57,8 @@ pub enum Error { #[derive(Debug, Snafu)] pub enum TargetsError { - #[snafu(display("encountered invalid product version"))] - InvalidProductVersion { source: ImageConfigError }, + #[snafu(display("encountered invalid image version"))] + InvalidImageVersion { source: ImageConfigError }, #[snafu(display("failed to read image config"))] ReadImageConfig { source: ImageConfigError }, @@ -136,15 +136,15 @@ impl Targets { // files are not a source a truth, but just provide data needed during the build. let image_config_path = PathBuf::new().join(&image.name).join("boil-config.toml"); - // Read the product config which defines supported product versions and their dependencies as + // Read the image config which defines supported image versions and their dependencies as // well as other values. let image_config = ImageConfig::from_file(image_config_path).context(ReadImageConfigSnafu)?; - // Create a list of product versions we need to generate targets for in the bakefile. + // Create a list of image versions we need to generate targets for in the bakefile. let pairs = image_config .filter_by_version(&image.versions) - .context(InvalidProductVersionSnafu)?; + .context(InvalidImageVersionSnafu)?; targets.insert_targets(image.name.clone(), pairs, &options, true)?; } @@ -174,15 +174,15 @@ impl Targets { continue; } - let product_config_path = + let image_config_path = PathBuf::new().join(image_name).join("boil-config.toml"); - let product_config = ImageConfig::from_file(product_config_path) - .context(ReadImageConfigSnafu)?; + let image_config = + ImageConfig::from_file(image_config_path).context(ReadImageConfigSnafu)?; - let pairs = product_config + let pairs = image_config .filter_by_version(&[image_version]) - .context(InvalidProductVersionSnafu)?; + .context(InvalidImageVersionSnafu)?; // Wowzers, recursion! self.insert_targets(image_name.clone(), pairs, options, false)?; @@ -372,15 +372,15 @@ impl Bakefile { /// Formats and returns the target name, eg. `stackable-base-1_0_0`. fn format_target_name(image_name: &str, image_version: &str) -> String { - // Replace any slashes from nested product names, eg. shared/protobuf, because docker buildx + // Replace any slashes from nested image names, eg. shared/protobuf, because docker buildx // has this weird restriction (because it also supports push, which we do on our own). We // are therefore artificially limited what target names we can use: [a-zA-Z0-9_-]+ - let product_name = image_name.replace('/', "__"); + let image_name = image_name.replace('/', "__"); // The dots in the semantic version also need to be replaced. - let product_version = image_version.to_string().replace('.', "_"); + let image_version = image_version.to_string().replace('.', "_"); - format!("{product_name}-{product_version}") + format!("{image_name}-{image_version}") } /// Formats and return the context name, eg. `stackable/image/stackable-base-1_0_0`. @@ -464,7 +464,7 @@ impl BakefileTarget { date_time: String, revision: String, config: Config, - docker_build_arguments: Vec, + build_arguments: Vec, release_version: String, ) -> Self { let config::Metadata { @@ -487,7 +487,7 @@ impl BakefileTarget { ]; let mut arguments = config.build_arguments; - arguments.extend(docker_build_arguments); + arguments.extend(build_arguments); arguments.insert(BuildArgument::new( "RELEASE_VERSION".to_owned(), release_version, diff --git a/rust/boil/src/build/cli.rs b/rust/boil/src/build/cli.rs index 9681c91a9..214f136d4 100644 --- a/rust/boil/src/build/cli.rs +++ b/rust/boil/src/build/cli.rs @@ -65,7 +65,7 @@ pub struct BuildArguments { #[arg(long, help_heading = "Registry Options")] pub use_localhost_registry: bool, - /// Override the target containerfile used, points to /. + /// Override the target containerfile used, points to /. #[arg( long, default_value_os_t = Self::default_target_containerfile(), diff --git a/rust/boil/src/build/mod.rs b/rust/boil/src/build/mod.rs index 10a2db7fc..1929e9356 100644 --- a/rust/boil/src/build/mod.rs +++ b/rust/boil/src/build/mod.rs @@ -43,8 +43,6 @@ pub enum Error { pub fn run_command(args: BuildArguments, config: Config) -> Result<(), Error> { // TODO (@Techassi): Parse Dockerfile instead to build the target graph - // let pattern = format!("**/{}/boil-config.toml", arguments.product.name); - // Validation ensure!( args.image_version.build.is_empty(), diff --git a/rust/boil/src/cli.rs b/rust/boil/src/cli.rs index c78fcf8b9..726642b1e 100644 --- a/rust/boil/src/cli.rs +++ b/rust/boil/src/cli.rs @@ -38,7 +38,7 @@ impl Cli { #[derive(Debug, Subcommand)] pub enum Command { - /// Build one or more product images. + /// Build one or more images. /// /// Requires docker with the buildx extension. #[command(alias = "some-chicken")] From ae70f914a95c2dedd0dafeab94f0236a3f93c17b Mon Sep 17 00:00:00 2001 From: Techassi Date: Wed, 10 Sep 2025 16:53:30 +0200 Subject: [PATCH 25/51] feat: Add CLI argument to load build argumnts from file --- rust/boil/src/build/bakefile.rs | 15 ++++++- rust/boil/src/build/cli.rs | 8 +++- rust/boil/src/build/docker.rs | 75 ++++++++++++++++++++++++--------- 3 files changed, 74 insertions(+), 24 deletions(-) diff --git a/rust/boil/src/build/bakefile.rs b/rust/boil/src/build/bakefile.rs index 612a40cbd..42db73654 100644 --- a/rust/boil/src/build/bakefile.rs +++ b/rust/boil/src/build/bakefile.rs @@ -20,7 +20,7 @@ use crate::{ VersionExt, build::{ cli, - docker::{BuildArgument, BuildArguments}, + docker::{BuildArgument, BuildArguments, ParseBuildArgumentsError}, image::{Image, ImageConfig, ImageConfigError, ImageOptions, VersionOptionsPair}, platform::TargetPlatform, }, @@ -53,6 +53,9 @@ pub enum Error { #[snafu(display("failed to create target graph"))] CreateGraph { source: TargetsError }, + + #[snafu(display("failed to parse build arguments"))] + ParseBuildArguments { source: ParseBuildArgumentsError }, } #[derive(Debug, Snafu)] @@ -238,11 +241,19 @@ impl Bakefile { let revision = Self::git_head_revision().context(GetRevisionSnafu)?; let date_time = Self::now()?; + // Load build arguments from a file if the user requested it + let mut build_arguments = args.build_arguments.clone(); + if let Some(path) = &args.build_arguments_file { + let build_arguments_from_file = + BuildArguments::from_file(path).context(ParseBuildArgumentsSnafu)?; + build_arguments.extend(build_arguments_from_file); + } + let target = BakefileTarget::common( date_time, revision, config, - args.docker_build_arguments.clone(), + build_arguments, args.image_version.base_prerelease(), ); diff --git a/rust/boil/src/build/cli.rs b/rust/boil/src/build/cli.rs index 214f136d4..1aba45dab 100644 --- a/rust/boil/src/build/cli.rs +++ b/rust/boil/src/build/cli.rs @@ -79,9 +79,15 @@ pub struct BuildArguments { #[arg( long = "build-argument", alias = "build-arg", + value_name = "BUILD_ARGUMENT", help_heading = "Build Options" )] - pub docker_build_arguments: Vec, + pub build_arguments: Vec, + + /// Load and override build arguments, in key=value format, each separated by a newline from the + /// specified file. + #[arg(long, alias = "build-args-file", help_heading = "Build Options")] + pub build_arguments_file: Option, /// Write target image tags to . Useful for signing or other follow-up CI steps. #[arg( diff --git a/rust/boil/src/build/docker.rs b/rust/boil/src/build/docker.rs index 575fb931d..b69d369d8 100644 --- a/rust/boil/src/build/docker.rs +++ b/rust/boil/src/build/docker.rs @@ -2,11 +2,12 @@ use std::{ collections::BTreeSet, fmt::Display, ops::{Deref, DerefMut}, + path::{Path, PathBuf}, str::FromStr, }; use serde::{Deserialize, Serialize, de::Visitor, ser::SerializeMap}; -use snafu::{OptionExt, Snafu, ensure}; +use snafu::{OptionExt, ResultExt, Snafu, ensure}; #[derive(Debug, Snafu)] pub enum ParseBuildArgumentError { @@ -17,13 +18,17 @@ pub enum ParseBuildArgumentError { NonAscii, } -// TODO (@Techassi): Unify parsing/casing in one place #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct BuildArgument((String, String)); impl BuildArgument { pub fn new(key: String, value: String) -> Self { - Self((key.replace(['-', '/'], "_").to_uppercase(), value)) + let key = Self::format_key(key); + Self((key, value)) + } + + fn format_key(key: impl AsRef) -> String { + key.as_ref().replace(['-', '/'], "_").to_uppercase() } } @@ -34,7 +39,7 @@ impl FromStr for BuildArgument { ensure!(s.is_ascii(), NonAsciiSnafu); let (key, value) = s.split_once('=').context(InvalidFormatSnafu)?; - let key = key.replace(['-', '/'], "_").to_uppercase(); + let key = Self::format_key(key); Ok(Self((key, value.to_owned()))) } @@ -73,6 +78,18 @@ impl Display for BuildArgument { } } +#[derive(Debug, Snafu)] +pub enum ParseBuildArgumentsError { + #[snafu(display("failed to read file at {path}", path = path.display()))] + ReadFile { + source: std::io::Error, + path: PathBuf, + }, + + #[snafu(display("failed to parse build argument"))] + ParseBuildArgument { source: ParseBuildArgumentError }, +} + #[derive(Clone, Debug, Default)] pub struct BuildArguments(BTreeSet); @@ -111,30 +128,30 @@ impl<'de> Deserialize<'de> for BuildArguments { where D: serde::Deserializer<'de>, { - deserializer.deserialize_map(BuildArgumentsVisitor) - } -} + struct BuildArgumentsVisitor; -struct BuildArgumentsVisitor; + impl<'de> Visitor<'de> for BuildArgumentsVisitor { + type Value = BuildArguments; -impl<'de> Visitor<'de> for BuildArgumentsVisitor { - type Value = BuildArguments; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(formatter, "a map of valid build arguments") + } - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(formatter, "a map of valid build arguments") - } + fn visit_map(self, mut map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + let mut args = BTreeSet::new(); - fn visit_map(self, mut map: A) -> Result - where - A: serde::de::MapAccess<'de>, - { - let mut args = BTreeSet::new(); + while let Some((key, value)) = map.next_entry()? { + args.insert(BuildArgument::new(key, value)); + } - while let Some((key, value)) = map.next_entry()? { - args.insert(BuildArgument::new(key, value)); + Ok(BuildArguments(args)) + } } - Ok(BuildArguments(args)) + deserializer.deserialize_map(BuildArgumentsVisitor) } } @@ -161,4 +178,20 @@ impl BuildArguments { pub fn is_empty(&self) -> bool { self.0.is_empty() } + + pub fn from_file

(path: P) -> Result + where + P: AsRef, + { + let path = path.as_ref(); + let content = std::fs::read_to_string(path).context(ReadFileSnafu { path })?; + let mut args = Self::new(); + + for line in content.lines() { + let arg = BuildArgument::from_str(line).context(ParseBuildArgumentSnafu)?; + args.insert(arg); + } + + Ok(args) + } } From 95352f99f177ea20b13a6eedda2584cb9136c3f3 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 11 Sep 2025 14:00:12 +0200 Subject: [PATCH 26/51] chore: Add metadata to Cargo.toml files --- Cargo.lock | 2 +- Cargo.toml | 2 ++ rust/boil/Cargo.toml | 6 +++++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 92fc3723d..6daec5640 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -105,7 +105,7 @@ checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" [[package]] name = "boil" -version = "0.0.1" +version = "0.1.0" dependencies = [ "clap", "clap_complete", diff --git a/Cargo.toml b/Cargo.toml index 93dcfd2ad..3f9d5999a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,9 @@ members = ["rust/*"] resolver = "2" [workspace.package] +repository = "https://github.com/stackabletech/docker-images" authors = ["Stackable "] +license = "OSL-3.0" [workspace.dependencies] clap = { version = "4.5.41", features = ["derive"] } diff --git a/rust/boil/Cargo.toml b/rust/boil/Cargo.toml index ac2f516ea..4ed5c140a 100644 --- a/rust/boil/Cargo.toml +++ b/rust/boil/Cargo.toml @@ -1,7 +1,11 @@ [package] name = "boil" -version = "0.0.1" +version = "0.1.0" edition = "2024" +authors.workspace = true +license.workspace = true +repository.workspace = true +publish = false [dependencies] clap.workspace = true From 27f30ca1a6748308c843da571eae8e0f0e46f53c Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 11 Sep 2025 14:04:58 +0200 Subject: [PATCH 27/51] chore: Add rustfmt config, apply changes --- .vscode/settings.json | 9 +++++++++ rust/boil/src/build/bakefile.rs | 2 +- rust/boil/src/build/docker.rs | 3 +-- rust/patchable/src/patch.rs | 17 +++++++++++------ rust/patchable/src/patch_mail.rs | 4 ++-- rust/patchable/src/repo.rs | 4 +--- rustfmt.toml | 9 +++++++++ 7 files changed, 34 insertions(+), 14 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 rustfmt.toml diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..a9da7befc --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,9 @@ +{ + "rust-analyzer.rustfmt.overrideCommand": [ + "rustfmt", + "+nightly-2025-05-26", + "--edition", + "2024", + "--" + ], +} diff --git a/rust/boil/src/build/bakefile.rs b/rust/boil/src/build/bakefile.rs index 42db73654..176226091 100644 --- a/rust/boil/src/build/bakefile.rs +++ b/rust/boil/src/build/bakefile.rs @@ -95,9 +95,9 @@ impl DerefMut for Targets { } impl IntoIterator for Targets { - type Item = (String, BTreeMap); type IntoIter = std::collections::btree_map::IntoIter>; + type Item = (String, BTreeMap); fn into_iter(self) -> Self::IntoIter { self.0.into_iter() diff --git a/rust/boil/src/build/docker.rs b/rust/boil/src/build/docker.rs index b69d369d8..ae6dc7fa4 100644 --- a/rust/boil/src/build/docker.rs +++ b/rust/boil/src/build/docker.rs @@ -114,9 +114,8 @@ impl Extend for BuildArguments { } impl IntoIterator for BuildArguments { - type Item = BuildArgument; - type IntoIter = std::collections::btree_set::IntoIter; + type Item = BuildArgument; fn into_iter(self) -> Self::IntoIter { self.0.into_iter() diff --git a/rust/patchable/src/patch.rs b/rust/patchable/src/patch.rs index 00cbd1ae9..5f590dc6e 100644 --- a/rust/patchable/src/patch.rs +++ b/rust/patchable/src/patch.rs @@ -7,15 +7,14 @@ use git2::{Oid, Repository}; use snafu::{OptionExt, ResultExt as _, Snafu}; use tracing_indicatif::suspend_tracing_indicatif; +#[cfg(doc)] +use crate::repo::ensure_worktree_is_at; use crate::{ error::{self, CommitId}, patch_mail::{self, mailinfo, mailsplit}, utils::raw_git_cmd, }; -#[cfg(doc)] -use crate::repo::ensure_worktree_is_at; - #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("failed to open stgit series file {path:?}"))] @@ -61,14 +60,18 @@ pub enum Error { source: git2::Error, parent_commit: error::CommitId, }, - #[snafu(display("failed to apply patch {patch_email_file:?} (from {patch_file:?}) to parent commit {parent_commit}"))] + #[snafu(display( + "failed to apply patch {patch_email_file:?} (from {patch_file:?}) to parent commit {parent_commit}" + ))] ApplyPatch { source: git2::Error, parent_commit: error::CommitId, patch_email_file: PathBuf, patch_file: PathBuf, }, - #[snafu(display("failed to write tree for patch {patch_email_file:?} (from {patch_file:?}) applied to parent commit {parent_commit}"))] + #[snafu(display( + "failed to write tree for patch {patch_email_file:?} (from {patch_file:?}) applied to parent commit {parent_commit}" + ))] WritePatchedTree { source: git2::Error, parent_commit: error::CommitId, @@ -77,7 +80,9 @@ pub enum Error { }, #[snafu(display("failed to read patched tree {tree}"))] ReadPatchedTree { source: git2::Error, tree: Oid }, - #[snafu(display("failed to write commit for patch {patch_email_file:?} (from {patch_file:?}) applied to parent commit {parent_commit}"))] + #[snafu(display( + "failed to write commit for patch {patch_email_file:?} (from {patch_file:?}) applied to parent commit {parent_commit}" + ))] WriteCommit { source: git2::Error, parent_commit: error::CommitId, diff --git a/rust/patchable/src/patch_mail.rs b/rust/patchable/src/patch_mail.rs index 680c32c2b..3a4fa330f 100644 --- a/rust/patchable/src/patch_mail.rs +++ b/rust/patchable/src/patch_mail.rs @@ -9,8 +9,8 @@ use std::{ use git2::{Diff, Repository, Signature}; use snafu::{OptionExt as _, ResultExt, Snafu}; -use tempfile::{tempdir, NamedTempFile}; -use time::{format_description::well_known::Rfc2822, OffsetDateTime}; +use tempfile::{NamedTempFile, tempdir}; +use time::{OffsetDateTime, format_description::well_known::Rfc2822}; use tracing_indicatif::suspend_tracing_indicatif; use crate::utils::raw_git_cmd; diff --git a/rust/patchable/src/repo.rs b/rust/patchable/src/repo.rs index 26e31cdac..0b0335b6a 100644 --- a/rust/patchable/src/repo.rs +++ b/rust/patchable/src/repo.rs @@ -28,9 +28,7 @@ pub enum Error { path: PathBuf, }, - #[snafu(display( - "failed to create worktree branch {branch:?} pointing at {commit} in {repo}" - ))] + #[snafu(display("failed to create worktree branch {branch:?} pointing at {commit} in {repo}"))] CreateWorktreeBranch { source: git2::Error, repo: error::RepoPath, diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 000000000..07217b21f --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,9 @@ +# This file includes unstable features, so you need to run "cargo +nightly fmt" to format your code. +# It's also ok to use the stable toolchain by simple running "cargo fmt", but using the nigthly formatter is prefered. + +# https://doc.rust-lang.org/nightly/edition-guide/rust-2024/rustfmt-style-edition.html +style_edition = "2024" +imports_granularity = "Crate" +group_imports = "StdExternalCrate" +reorder_impl_items = true +use_field_init_shorthand = true From 3d6b482175ecdbc14ef7d07292be85a9e917aee7 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 11 Sep 2025 14:06:55 +0200 Subject: [PATCH 28/51] chore: Adjust pre-commit config file --- .pre-commit-config.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0e71d9318..4af07cfe2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -60,3 +60,27 @@ repos: entry: .scripts/update_readme_badges.sh stages: [pre-commit, pre-merge-commit, manual] pass_filenames: false + + - id: cargo-test + name: cargo-test + language: system + entry: cargo test + stages: [pre-commit, pre-merge-commit] + pass_filenames: false + files: \.rs$|Cargo\.(toml|lock) + + - id: cargo-rustfmt + name: cargo-rustfmt + language: system + entry: cargo +nightly-2025-05-26 fmt --all -- --check + stages: [pre-commit, pre-merge-commit] + pass_filenames: false + files: \.rs$ + + - id: cargo-clippy + name: cargo-clippy + language: system + entry: cargo clippy --all-targets -- -D warnings + stages: [pre-commit, pre-merge-commit] + pass_filenames: false + files: \.rs$ From 2729751847af5246b2cc93abd238c9d373c506f7 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 11 Sep 2025 14:15:33 +0200 Subject: [PATCH 29/51] ci: Add cargo-deny job to workflow --- .github/workflows/boil_build_release.yaml | 28 +++++++++++ deny.toml | 57 +++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 deny.toml diff --git a/.github/workflows/boil_build_release.yaml b/.github/workflows/boil_build_release.yaml index 982effc5b..86ad6ac26 100644 --- a/.github/workflows/boil_build_release.yaml +++ b/.github/workflows/boil_build_release.yaml @@ -16,9 +16,34 @@ env: RUST_VERSION: 1.87.0 jobs: + # This job is always run to ensure we don't miss any new upstream advisories + cargo-deny: + name: Run cargo-deny + runs-on: ubuntu-latest + # Prevent sudden announcement of a new advisory from failing CI + continue-on-error: ${{ matrix.checks == 'advisories' }} + strategy: + matrix: + checks: + - advisories + - bans licenses sources + steps: + - name: Checkout Repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + submodules: recursive + + - name: Run cargo-deny + uses: EmbarkStudios/cargo-deny-action@f2ba7abc2abebaf185c833c3961145a3c275caad # v2.0.13 + with: + command: check ${{ matrix.checks }} + create-release: name: Create Draft Release if: github.event_name == 'push' + needs: + - cargo-deny runs-on: ubuntu-latest steps: - name: Create Draft Release @@ -27,6 +52,9 @@ jobs: draft: true build: + name: Build boil + needs: + - create-release strategy: fail-fast: false matrix: diff --git a/deny.toml b/deny.toml new file mode 100644 index 000000000..e3067ac08 --- /dev/null +++ b/deny.toml @@ -0,0 +1,57 @@ +# This file is the source of truth for all our repos! +# This includes repos not templated by operator-templating, please copy/paste the file for this repos. + +# TIP: Use "cargo deny check" to check if everything is fine + +[graph] +targets = [ + { triple = "x86_64-unknown-linux-gnu" }, + { triple = "aarch64-unknown-linux-gnu" }, + { triple = "x86_64-unknown-linux-musl" }, + { triple = "aarch64-apple-darwin" }, + { triple = "x86_64-apple-darwin" }, +] + +[advisories] +yanked = "deny" + +[bans] +multiple-versions = "allow" + +[licenses] +unused-allowed-license = "allow" +confidence-threshold = 1.0 +allow = [ + "Apache-2.0", + "BSD-2-Clause", + "BSD-3-Clause", + "CC0-1.0", + "ISC", + "LicenseRef-ring", + "LicenseRef-webpki", + "MIT", + "MPL-2.0", + "OpenSSL", # Needed for the ring and/or aws-lc-sys crate. See https://github.com/stackabletech/operator-templating/pull/464 for details + "Unicode-3.0", + "Unicode-DFS-2016", + "Zlib", + "Unlicense", +] +private = { ignore = true } + +[[licenses.clarify]] +name = "ring" +expression = "LicenseRef-ring" +license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }] + +[[licenses.clarify]] +name = "webpki" +expression = "LicenseRef-webpki" +license-files = [{ path = "LICENSE", hash = 0x001c7e6c }] + +[sources] +unknown-registry = "deny" +unknown-git = "deny" + +[sources.allow-org] +github = ["stackabletech"] From 69123a1a11af7136ed5e7eb96af7ffdcbf64db0b Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 11 Sep 2025 14:19:31 +0200 Subject: [PATCH 30/51] chore: Bump tracing-subscriber to 0.3.20 to fix RUSTSEC-2025-0055 --- Cargo.lock | 68 ++++++++++-------------------------------------------- 1 file changed, 12 insertions(+), 56 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6daec5640..bfa8dde12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -715,11 +715,11 @@ checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" [[package]] name = "matchers" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" dependencies = [ - "regex-automata 0.1.10", + "regex-automata", ] [[package]] @@ -750,12 +750,11 @@ dependencies = [ [[package]] name = "nu-ansi-term" -version = "0.46.0" +version = "0.50.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" dependencies = [ - "overload", - "winapi", + "windows-sys 0.52.0", ] [[package]] @@ -820,12 +819,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - [[package]] name = "patchable" version = "0.1.0" @@ -920,17 +913,8 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.9", - "regex-syntax 0.8.5", -] - -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax 0.6.29", + "regex-automata", + "regex-syntax", ] [[package]] @@ -941,15 +925,9 @@ checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.5", + "regex-syntax", ] -[[package]] -name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - [[package]] name = "regex-syntax" version = "0.8.5" @@ -1360,14 +1338,14 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" dependencies = [ "matchers", "nu-ansi-term", "once_cell", - "regex", + "regex-automata", "sharded-slab", "smallvec", "thread_local", @@ -1557,28 +1535,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows-sys" version = "0.52.0" From 0df97624b51236ce5508d6f2f3e0c69ffb744845 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 11 Sep 2025 14:22:28 +0200 Subject: [PATCH 31/51] chore: Add metadata to patchable's Cargo.toml file --- rust/patchable/Cargo.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rust/patchable/Cargo.toml b/rust/patchable/Cargo.toml index cb72ed092..9b8653ee6 100644 --- a/rust/patchable/Cargo.toml +++ b/rust/patchable/Cargo.toml @@ -2,6 +2,10 @@ name = "patchable" version = "0.1.0" edition = "2021" +authors.workspace = true +license.workspace = true +repository.workspace = true +publish = false [dependencies] clap.workspace = true From ec4c14a277439873b162dee00d15b05b2a2bfc0e Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 11 Sep 2025 14:25:33 +0200 Subject: [PATCH 32/51] ci: Still build boil on PR --- .github/workflows/boil_build_release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/boil_build_release.yaml b/.github/workflows/boil_build_release.yaml index 86ad6ac26..45144b31b 100644 --- a/.github/workflows/boil_build_release.yaml +++ b/.github/workflows/boil_build_release.yaml @@ -54,7 +54,7 @@ jobs: build: name: Build boil needs: - - create-release + - cargo-deny strategy: fail-fast: false matrix: From 3d6f2ac2a7125cc6dba952aaddd2c8790362f1d0 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 11 Sep 2025 14:32:06 +0200 Subject: [PATCH 33/51] ci: Install Rust in pre-commit workflow --- .github/workflows/pr_pre-commit.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr_pre-commit.yaml b/.github/workflows/pr_pre-commit.yaml index faa7e5c54..cbb99329d 100644 --- a/.github/workflows/pr_pre-commit.yaml +++ b/.github/workflows/pr_pre-commit.yaml @@ -5,6 +5,7 @@ on: pull_request: env: + RUST_TOOLCHAIN_VERSION: "nightly-2025-05-26" HADOLINT_VERSION: "v2.12.0" PYTHON_VERSION: "3.12" @@ -19,4 +20,5 @@ jobs: - uses: stackabletech/actions/run-pre-commit@497f3e3cbfe9b89b1e570351b97d050eebcad5d0 # 0.8.3 with: python-version: ${{ env.PYTHON_VERSION }} + rust: ${{ env.RUST_TOOLCHAIN_VERSION }} hadolint: ${{ env.HADOLINT_VERSION }} From c89dd37da57a5ff55be42239c1f29614019c43e7 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 11 Sep 2025 14:44:06 +0200 Subject: [PATCH 34/51] chore: Use Apache-2.0 license instead of OSL-3.0 license --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 3f9d5999a..adf6ad015 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ resolver = "2" [workspace.package] repository = "https://github.com/stackabletech/docker-images" authors = ["Stackable "] -license = "OSL-3.0" +license = "Apache-2.0" [workspace.dependencies] clap = { version = "4.5.41", features = ["derive"] } From e58d20e4acaeffca8c48e845f9d9987289d61493 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 11 Sep 2025 15:49:49 +0200 Subject: [PATCH 35/51] chore: Mention nightly Rust toolchain updates --- .github/workflows/pr_pre-commit.yaml | 1 + .vscode/settings.json | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/pr_pre-commit.yaml b/.github/workflows/pr_pre-commit.yaml index cbb99329d..51e470a02 100644 --- a/.github/workflows/pr_pre-commit.yaml +++ b/.github/workflows/pr_pre-commit.yaml @@ -5,6 +5,7 @@ on: pull_request: env: + # Keep in sync with across other repos like operator-rs and operator-templating RUST_TOOLCHAIN_VERSION: "nightly-2025-05-26" HADOLINT_VERSION: "v2.12.0" PYTHON_VERSION: "3.12" diff --git a/.vscode/settings.json b/.vscode/settings.json index a9da7befc..b3ec3613b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,7 @@ { "rust-analyzer.rustfmt.overrideCommand": [ "rustfmt", + // Keep in sync with across other repos like operator-rs and operator-templating "+nightly-2025-05-26", "--edition", "2024", From bccc43bf9cedd44eb2e53910af14b53345f55840 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 12 Sep 2025 09:28:02 +0200 Subject: [PATCH 36/51] chore: Remove OpenShift arg --- rust/boil/src/cli.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/rust/boil/src/cli.rs b/rust/boil/src/cli.rs index 726642b1e..35ffbd2b6 100644 --- a/rust/boil/src/cli.rs +++ b/rust/boil/src/cli.rs @@ -11,10 +11,6 @@ pub struct Cli { #[arg(short, long = "configuration", global = true, default_value_os_t = Self::default_config_path())] pub config_path: PathBuf, - /// Path to the OpenShift configuration file. - #[arg(long, default_value_os_t = Self::default_openshift_config_path())] - pub openshift_config_path: PathBuf, - #[arg(short, long, default_value_os_t = Self::default_base_path())] pub base_path: PathBuf, @@ -27,10 +23,6 @@ impl Cli { PathBuf::from("./boil.toml") } - fn default_openshift_config_path() -> PathBuf { - PathBuf::from("./openshift.toml") - } - fn default_base_path() -> PathBuf { PathBuf::from(".") } From 4b353daac367f7a8127d675c676ceb07882283c0 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 12 Sep 2025 09:28:20 +0200 Subject: [PATCH 37/51] chore: Remove tree subcommand (for now) --- rust/boil/src/main.rs | 1 - rust/boil/src/show/mod.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/rust/boil/src/main.rs b/rust/boil/src/main.rs index 25bf2c3ec..acebf9cf6 100644 --- a/rust/boil/src/main.rs +++ b/rust/boil/src/main.rs @@ -87,7 +87,6 @@ async fn main() -> Result<(), Error> { ShowCommand::Images(arguments) => { show::images::run_command(arguments).context(ShowSnafu) } - ShowCommand::Tree => todo!(), }, Command::Completions(arguments) => { completions::run_command(arguments); diff --git a/rust/boil/src/show/mod.rs b/rust/boil/src/show/mod.rs index 0be38bffe..a6ad01b43 100644 --- a/rust/boil/src/show/mod.rs +++ b/rust/boil/src/show/mod.rs @@ -13,5 +13,4 @@ pub struct ShowArguments { #[derive(Debug, Subcommand)] pub enum ShowCommand { Images(ShowImagesArguments), - Tree, } From 92a9622263ba3f1cea3d2e3efe13ee6a2abd4ea1 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 12 Sep 2025 12:59:01 +0200 Subject: [PATCH 38/51] chore: Move glob pattern into const --- rust/boil/src/build/bakefile.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rust/boil/src/build/bakefile.rs b/rust/boil/src/build/bakefile.rs index 176226091..9a403e775 100644 --- a/rust/boil/src/build/bakefile.rs +++ b/rust/boil/src/build/bakefile.rs @@ -28,6 +28,8 @@ use crate::{ utils::{format_image_manifest_uri, format_image_repository_uri}, }; +/// This glob pattern matches all (deeply nested) image configs. +pub const ALL_CONFIGS_GLOB_PATTERN: &str = "**/boil-config.toml"; pub const COMMON_TARGET_NAME: &str = "common--target"; pub const ENTRY_TARGET_NAME_PREFIX: &str = "entry--"; @@ -106,7 +108,7 @@ impl IntoIterator for Targets { impl Targets { pub fn all(options: TargetsOptions) -> Result { - let image_config_paths = glob("./**/boil-config.toml") + let image_config_paths = glob(ALL_CONFIGS_GLOB_PATTERN) .expect("glob pattern must be valid") .filter_map(Result::ok); From 7e6bede137f61521766b485211f1f1e3275513f2 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 12 Sep 2025 12:59:50 +0200 Subject: [PATCH 39/51] chore: Add more image validation --- rust/boil/src/build/bakefile.rs | 2 +- rust/boil/src/build/image.rs | 17 ++++++++++++++--- rust/boil/src/cli.rs | 7 ------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/rust/boil/src/build/bakefile.rs b/rust/boil/src/build/bakefile.rs index 9a403e775..031d9bc14 100644 --- a/rust/boil/src/build/bakefile.rs +++ b/rust/boil/src/build/bakefile.rs @@ -391,7 +391,7 @@ impl Bakefile { let image_name = image_name.replace('/', "__"); // The dots in the semantic version also need to be replaced. - let image_version = image_version.to_string().replace('.', "_"); + let image_version = image_version.replace('.', "_"); format!("{image_name}-{image_version}") } diff --git a/rust/boil/src/build/image.rs b/rust/boil/src/build/image.rs index cd5a7c6f5..d1e9e3554 100644 --- a/rust/boil/src/build/image.rs +++ b/rust/boil/src/build/image.rs @@ -7,7 +7,7 @@ use std::{ }; use serde::Deserialize; -use snafu::{ResultExt as _, Snafu}; +use snafu::{ResultExt as _, Snafu, ensure}; use crate::{IfContext, build::docker::BuildArguments}; @@ -15,6 +15,12 @@ use crate::{IfContext, build::docker::BuildArguments}; pub enum ParseImageError { #[snafu(display("encountered invalid format, expected name[=version,...]"))] InvalidFormat, + + #[snafu(display("the path contains unsupported characters: '.' or '~'"))] + UnsupportedChars, + + #[snafu(display("absolute paths are not supported"))] + AbsolutePath, } #[derive(Clone, Debug)] @@ -29,11 +35,16 @@ impl FromStr for Image { fn from_str(s: &str) -> Result { let parts: Vec<_> = s.split('=').collect(); + ensure!(!parts[0].contains(['.', '~']), UnsupportedCharsSnafu); + ensure!(!parts[0].starts_with('/'), AbsolutePathSnafu); + + let image_name = parts[0].trim_end_matches('/').to_owned(); + match parts.len() { - 1 => Ok(Self::new_unversioned(parts[0].to_owned())), + 1 => Ok(Self::new_unversioned(image_name)), 2 => { let versions: Vec<_> = parts[1].split(',').map(ToOwned::to_owned).collect(); - Ok(Self::new(parts[0].to_owned(), versions)) + Ok(Self::new(image_name, versions)) } _ => InvalidFormatSnafu.fail(), } diff --git a/rust/boil/src/cli.rs b/rust/boil/src/cli.rs index 35ffbd2b6..85323f289 100644 --- a/rust/boil/src/cli.rs +++ b/rust/boil/src/cli.rs @@ -11,9 +11,6 @@ pub struct Cli { #[arg(short, long = "configuration", global = true, default_value_os_t = Self::default_config_path())] pub config_path: PathBuf, - #[arg(short, long, default_value_os_t = Self::default_base_path())] - pub base_path: PathBuf, - #[command(subcommand)] pub command: Command, } @@ -22,10 +19,6 @@ impl Cli { fn default_config_path() -> PathBuf { PathBuf::from("./boil.toml") } - - fn default_base_path() -> PathBuf { - PathBuf::from(".") - } } #[derive(Debug, Subcommand)] From e1798a287878fa38cbaaf1c21e92efe0142c7741 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 12 Sep 2025 13:40:03 +0200 Subject: [PATCH 40/51] chore: Add image config file name constant --- rust/boil/src/build/bakefile.rs | 13 +++++++------ rust/boil/src/build/image.rs | 5 +++++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/rust/boil/src/build/bakefile.rs b/rust/boil/src/build/bakefile.rs index 031d9bc14..99478c8cd 100644 --- a/rust/boil/src/build/bakefile.rs +++ b/rust/boil/src/build/bakefile.rs @@ -28,8 +28,6 @@ use crate::{ utils::{format_image_manifest_uri, format_image_repository_uri}, }; -/// This glob pattern matches all (deeply nested) image configs. -pub const ALL_CONFIGS_GLOB_PATTERN: &str = "**/boil-config.toml"; pub const COMMON_TARGET_NAME: &str = "common--target"; pub const ENTRY_TARGET_NAME_PREFIX: &str = "entry--"; @@ -108,7 +106,7 @@ impl IntoIterator for Targets { impl Targets { pub fn all(options: TargetsOptions) -> Result { - let image_config_paths = glob(ALL_CONFIGS_GLOB_PATTERN) + let image_config_paths = glob(ImageConfig::ALL_CONFIGS_GLOB_PATTERN) .expect("glob pattern must be valid") .filter_map(Result::ok); @@ -139,7 +137,9 @@ impl Targets { // TODO (@Techassi): We should instead build the graph based on the Dockerfile(s), // because this is the source of truth and what ultimately gets built. The boil config // files are not a source a truth, but just provide data needed during the build. - let image_config_path = PathBuf::new().join(&image.name).join("boil-config.toml"); + let image_config_path = PathBuf::new() + .join(&image.name) + .join(ImageConfig::DEFAULT_FILE_NAME); // Read the image config which defines supported image versions and their dependencies as // well as other values. @@ -179,8 +179,9 @@ impl Targets { continue; } - let image_config_path = - PathBuf::new().join(image_name).join("boil-config.toml"); + let image_config_path = PathBuf::new() + .join(image_name) + .join(ImageConfig::DEFAULT_FILE_NAME); let image_config = ImageConfig::from_file(image_config_path).context(ReadImageConfigSnafu)?; diff --git a/rust/boil/src/build/image.rs b/rust/boil/src/build/image.rs index d1e9e3554..3ef463402 100644 --- a/rust/boil/src/build/image.rs +++ b/rust/boil/src/build/image.rs @@ -103,6 +103,11 @@ pub struct ImageConfig { } impl ImageConfig { + /// This glob pattern matches all (deeply nested) image configs. + pub const ALL_CONFIGS_GLOB_PATTERN: &str = "**/boil-config.toml"; + /// The default image config file name. + pub const DEFAULT_FILE_NAME: &str = "boil-config.toml"; + pub fn filter_by_version( self, versions: &[V], From c54082992ef839b0f541326f4838e09d5d2949ff Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 12 Sep 2025 13:42:41 +0200 Subject: [PATCH 41/51] chore: Move Docker label into a constant --- rust/boil/src/build/bakefile.rs | 4 ++-- rust/boil/src/build/docker.rs | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/rust/boil/src/build/bakefile.rs b/rust/boil/src/build/bakefile.rs index 99478c8cd..f6aa14d47 100644 --- a/rust/boil/src/build/bakefile.rs +++ b/rust/boil/src/build/bakefile.rs @@ -20,7 +20,7 @@ use crate::{ VersionExt, build::{ cli, - docker::{BuildArgument, BuildArguments, ParseBuildArgumentsError}, + docker::{BuildArgument, BuildArguments, LABEL_BUILD_DATE, ParseBuildArgumentsError}, image::{Image, ImageConfig, ImageConfigError, ImageOptions, VersionOptionsPair}, platform::TargetPlatform, }, @@ -513,7 +513,7 @@ impl BakefileTarget { let labels = BTreeMap::from([ (ANNOTATION_CREATED.to_owned(), date_time.clone()), (ANNOTATION_REVISION.to_owned(), revision), - ("build-date".to_owned(), date_time), + (LABEL_BUILD_DATE.to_owned(), date_time), ]); Self { diff --git a/rust/boil/src/build/docker.rs b/rust/boil/src/build/docker.rs index ae6dc7fa4..23e524f1c 100644 --- a/rust/boil/src/build/docker.rs +++ b/rust/boil/src/build/docker.rs @@ -9,6 +9,9 @@ use std::{ use serde::{Deserialize, Serialize, de::Visitor, ser::SerializeMap}; use snafu::{OptionExt, ResultExt, Snafu, ensure}; +/// Label key for the date and time on which the image was built. +pub const LABEL_BUILD_DATE: &str = "build-date"; + #[derive(Debug, Snafu)] pub enum ParseBuildArgumentError { #[snafu(display("invalid format, expected ="))] From 943d7bc46ab8378cbc006b973c96b8f3471310ab Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 12 Sep 2025 13:43:23 +0200 Subject: [PATCH 42/51] fix: Actually use --target-containerfile argument --- rust/boil/src/build/bakefile.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rust/boil/src/build/bakefile.rs b/rust/boil/src/build/bakefile.rs index f6aa14d47..92c55940b 100644 --- a/rust/boil/src/build/bakefile.rs +++ b/rust/boil/src/build/bakefile.rs @@ -324,7 +324,9 @@ impl Bakefile { args.strip_architecture, ); - let dockerfile = PathBuf::new().join(&image_name).join("Dockerfile"); + let dockerfile = PathBuf::new() + .join(&image_name) + .join(&args.target_containerfile); let target_name = if is_entry { Self::format_entry_target_name(&image_name, &image_version) @@ -525,7 +527,6 @@ impl BakefileTarget { } fn image_version_annotation(image_version: &str, sdp_image_version: &Version) -> Vec { - // Annotations describe OCI image components. vec![ // TODO (@Techassi): Move this version formatting into a function // TODO (@Techassi): Make this vendor agnostic, don't hard-code stackable here From 2be036ea1b5afdf6ec03ef0f612e846c3511ec22 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 12 Sep 2025 13:52:42 +0200 Subject: [PATCH 43/51] docs: Add doc comments for clap command handler functions --- rust/boil/src/build/mod.rs | 1 + rust/boil/src/completions/mod.rs | 1 + rust/boil/src/show/images/mod.rs | 1 + 3 files changed, 3 insertions(+) diff --git a/rust/boil/src/build/mod.rs b/rust/boil/src/build/mod.rs index 1929e9356..592736770 100644 --- a/rust/boil/src/build/mod.rs +++ b/rust/boil/src/build/mod.rs @@ -41,6 +41,7 @@ pub enum Error { InvalidImageVersion, } +/// This is the `boil build` command handler function. pub fn run_command(args: BuildArguments, config: Config) -> Result<(), Error> { // TODO (@Techassi): Parse Dockerfile instead to build the target graph // Validation diff --git a/rust/boil/src/completions/mod.rs b/rust/boil/src/completions/mod.rs index 070d3a469..2c70386df 100644 --- a/rust/boil/src/completions/mod.rs +++ b/rust/boil/src/completions/mod.rs @@ -9,6 +9,7 @@ pub struct CompletionsArguments { pub shell: Shell, } +/// This is the `boil completions` command handler function. pub fn run_command(arguments: CompletionsArguments) { let mut cli = Cli::command(); let bin_name = cli.get_bin_name().unwrap_or("boil").to_owned(); diff --git a/rust/boil/src/show/images/mod.rs b/rust/boil/src/show/images/mod.rs index b289c222d..fb39bffcc 100644 --- a/rust/boil/src/show/images/mod.rs +++ b/rust/boil/src/show/images/mod.rs @@ -43,6 +43,7 @@ impl Serialize for OneOrMany { } } +/// This is the `boil show images` command handler function. pub fn run_command(arguments: ShowImagesArguments) -> Result<(), Error> { let list: BTreeMap<_, _> = if arguments.image.is_empty() { Targets::all(TargetsOptions { only_entry: true }) From b651cb15b276a2be36df6c33125f2c5c29cbdc50 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 12 Sep 2025 14:09:10 +0200 Subject: [PATCH 44/51] feat: Auto-detect interactive shell for pretty output --- rust/boil/src/show/images/cli.rs | 15 ++++++++++++--- rust/boil/src/show/images/mod.rs | 19 +++++++++---------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/rust/boil/src/show/images/cli.rs b/rust/boil/src/show/images/cli.rs index e4ee4c837..0c3469ef5 100644 --- a/rust/boil/src/show/images/cli.rs +++ b/rust/boil/src/show/images/cli.rs @@ -1,4 +1,4 @@ -use clap::Args; +use clap::{Args, ValueEnum}; use crate::build::image::Image; @@ -8,6 +8,15 @@ pub struct ShowImagesArguments { pub image: Vec, /// Pretty print the structured output. - #[arg(long)] - pub pretty: bool, + #[arg(long, value_enum, default_value_t = Pretty::default())] + pub pretty: Pretty, +} + +// #[derive(Clone, Debug, Default, strum::Display, strum::EnumString)] +#[derive(Clone, Debug, Default, ValueEnum)] +pub enum Pretty { + #[default] + Auto, + Always, + Never, } diff --git a/rust/boil/src/show/images/mod.rs b/rust/boil/src/show/images/mod.rs index fb39bffcc..c95fe992c 100644 --- a/rust/boil/src/show/images/mod.rs +++ b/rust/boil/src/show/images/mod.rs @@ -1,11 +1,10 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, io::IsTerminal}; -use serde::{Serialize, ser::SerializeSeq}; use snafu::{ResultExt, Snafu}; use crate::{ build::bakefile::{Targets, TargetsError, TargetsOptions}, - show::images::cli::ShowImagesArguments, + show::images::cli::{Pretty, ShowImagesArguments}, }; pub mod cli; @@ -66,14 +65,14 @@ pub fn run_command(arguments: ShowImagesArguments) -> Result<(), Error> { print_to_stdout(list, arguments.pretty) } -fn print_to_stdout(list: BTreeMap>, pretty: bool) -> Result<(), Error> { +fn print_to_stdout(list: BTreeMap>, pretty: Pretty) -> Result<(), Error> { let stdout = std::io::stdout(); - let list = OneOrMany(list); - - if pretty { - serde_json::to_writer_pretty(stdout, &list).context(SerializeListSnafu) - } else { - serde_json::to_writer(stdout, &list).context(SerializeListSnafu) + match pretty { + Pretty::Always | Pretty::Auto if stdout.is_terminal() => { + serde_json::to_writer_pretty(stdout, &list) + } + _ => serde_json::to_writer(stdout, &list), } + .context(SerializeListSnafu) } From 089d2bffb1ace4e63b3332669db360abe6dc2081 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 12 Sep 2025 14:10:22 +0200 Subject: [PATCH 45/51] chore: Only use a single output format --- rust/boil/src/show/images/mod.rs | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/rust/boil/src/show/images/mod.rs b/rust/boil/src/show/images/mod.rs index c95fe992c..f977fd02e 100644 --- a/rust/boil/src/show/images/mod.rs +++ b/rust/boil/src/show/images/mod.rs @@ -18,30 +18,6 @@ pub enum Error { BuildTargets { source: TargetsError }, } -// NOTE (@Techassi): I don't know if I like this... but this makes the stdout output very convient -// to consume. -struct OneOrMany(BTreeMap>); - -impl Serialize for OneOrMany { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - if self.0.len() == 1 { - let mut seq = serializer.serialize_seq(Some(1))?; - for entry in &self.0 { - for version in entry.1 { - seq.serialize_element(&version)?; - } - } - - Ok(seq.end()?) - } else { - self.0.serialize(serializer) - } - } -} - /// This is the `boil show images` command handler function. pub fn run_command(arguments: ShowImagesArguments) -> Result<(), Error> { let list: BTreeMap<_, _> = if arguments.image.is_empty() { From 79e59b3bfcb15f7f900bd02ef40aae0436812a5a Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 12 Sep 2025 14:11:42 +0200 Subject: [PATCH 46/51] chore: Remove unused code --- rust/boil/src/config.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/rust/boil/src/config.rs b/rust/boil/src/config.rs index e65a0360a..19d4ff3c0 100644 --- a/rust/boil/src/config.rs +++ b/rust/boil/src/config.rs @@ -37,6 +37,3 @@ pub struct Metadata { pub vendor: String, pub source: Url, } - -#[derive(Debug, Deserialize)] -pub struct DockerConfig {} From dd5caa77a6b720076121029ba1e5e58ca3da93c1 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 12 Sep 2025 14:13:27 +0200 Subject: [PATCH 47/51] chore: Rename --export-image-manifest-uris CLI argument --- rust/boil/src/build/cli.rs | 4 ++-- rust/boil/src/build/mod.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/boil/src/build/cli.rs b/rust/boil/src/build/cli.rs index 1aba45dab..6a97fcdc0 100644 --- a/rust/boil/src/build/cli.rs +++ b/rust/boil/src/build/cli.rs @@ -94,13 +94,13 @@ pub struct BuildArguments { long, alias = "export-tags-file", help_heading = "Build Options", - value_name = "EXPORT_FILE", + value_name = "FILE", value_hint = ValueHint::FilePath, value_parser = value_parser!(PathBuf), default_missing_value = "boil-target-tags", num_args(0..=1) )] - pub export_image_manifest_uris: Option, + pub write_image_manifest_uris: Option, /// Strips the architecture from the image (index) manifest tag. #[arg(long, help_heading = "Build Options")] diff --git a/rust/boil/src/build/mod.rs b/rust/boil/src/build/mod.rs index 592736770..6373c2c8e 100644 --- a/rust/boil/src/build/mod.rs +++ b/rust/boil/src/build/mod.rs @@ -56,7 +56,7 @@ pub fn run_command(args: BuildArguments, config: Config) -> Result<(), Error> { let count = image_manifest_uris.len(); // Write the image manifest URIs to file if requested - if let Some(path) = args.export_image_manifest_uris { + if let Some(path) = args.write_image_manifest_uris { std::fs::write(path, image_manifest_uris.join("\n")) .context(WriteImageManifestUrisFileSnafu)?; } From a2aba860e4e01657b8224ae1caa6210dba97566f Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 12 Sep 2025 15:36:33 +0200 Subject: [PATCH 48/51] chore: Apply suggestions Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com> --- rust/boil/src/build/cli.rs | 1 + rust/boil/src/main.rs | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/rust/boil/src/build/cli.rs b/rust/boil/src/build/cli.rs index 6a97fcdc0..35756fcc0 100644 --- a/rust/boil/src/build/cli.rs +++ b/rust/boil/src/build/cli.rs @@ -120,6 +120,7 @@ impl BuildArguments { "0.0.0-dev".parse().expect("must be a valid SemVer") } + // TODO: Auto-detect this fn default_architecture() -> TargetPlatform { TargetPlatform::Linux(Architecture::Amd64) } diff --git a/rust/boil/src/main.rs b/rust/boil/src/main.rs index acebf9cf6..0429c439b 100644 --- a/rust/boil/src/main.rs +++ b/rust/boil/src/main.rs @@ -50,7 +50,8 @@ pub trait VersionExt { impl VersionExt for Version { fn base(&self) -> String { - format!("{}.{}.{}", self.major, self.minor, self.patch) + let Self {major, minor, patch, ..} = self; + format!("{major}.{minor}.{patch}") } fn base_prerelease(&self) -> String { From 52eeaed3b74f0f50b09af54b1abcf431008147a2 Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 12 Sep 2025 16:06:44 +0200 Subject: [PATCH 49/51] docs: Add some clarifying doc comments --- rust/boil/src/build/bakefile.rs | 6 ++++++ rust/boil/src/main.rs | 17 ++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/rust/boil/src/build/bakefile.rs b/rust/boil/src/build/bakefile.rs index 92c55940b..aa85382cc 100644 --- a/rust/boil/src/build/bakefile.rs +++ b/rust/boil/src/build/bakefile.rs @@ -105,6 +105,9 @@ impl IntoIterator for Targets { } impl Targets { + /// Returns a map of all targets by globbing for (nested) image config files. + /// + /// The search behaviour can be customized using the provided [`TargetsOptions`]. pub fn all(options: TargetsOptions) -> Result { let image_config_paths = glob(ImageConfig::ALL_CONFIGS_GLOB_PATTERN) .expect("glob pattern must be valid") @@ -130,6 +133,9 @@ impl Targets { Ok(targets) } + /// Returns a filtered set out of all targets by looking up selected image config files. + /// + /// The search behaviour can be customized using the provided [`TargetsOptions`]. pub fn set(images: &[Image], options: TargetsOptions) -> Result { let mut targets = Self::default(); diff --git a/rust/boil/src/main.rs b/rust/boil/src/main.rs index 0429c439b..189616ac7 100644 --- a/rust/boil/src/main.rs +++ b/rust/boil/src/main.rs @@ -18,7 +18,16 @@ mod build; mod completions; mod show; +/// This trait extends functionailty provided by [`snafu`]. +/// +/// [`snafu`] already provides various ways to extend [`Result`]s with additional context-sensitive +/// information. This trait allows calling `if_context` on any type, which runs a predicate to +/// determine if an error with the provided context should be returned. +/// +/// This trait can be thought of as a combination of [`snafu::ensure!`] and returning [`Ok`] +/// afterwards. pub trait IfContext: Sized { + /// Runs `predicate` and returns [`Ok`] if `true` or [`Err`] (with data from `context`) otherwise. fn if_context(self, predicate: P, context: C) -> Result where P: Fn(&Self) -> bool, @@ -50,7 +59,13 @@ pub trait VersionExt { impl VersionExt for Version { fn base(&self) -> String { - let Self {major, minor, patch, ..} = self; + let Self { + major, + minor, + patch, + .. + } = self; + format!("{major}.{minor}.{patch}") } From 4ee352409b605d9d70ba53c91afd2c8dacc9ee7a Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 12 Sep 2025 20:47:54 +0200 Subject: [PATCH 50/51] feat: Improve image parsing and validation --- Cargo.lock | 125 ++++++++++++++++++++++++++++++++++- Cargo.toml | 1 + rust/boil/Cargo.toml | 3 + rust/boil/src/build/image.rs | 62 +++++++++++++++-- 4 files changed, 185 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bfa8dde12..1b9c5fd5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,6 +112,7 @@ dependencies = [ "git2", "glob", "oci-spec", + "rstest", "semver", "serde", "serde_json", @@ -369,6 +370,49 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-macro", + "futures-task", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "getrandom" version = "0.3.1" @@ -847,6 +891,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.31" @@ -865,6 +915,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "proc-macro-crate" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro-error-attr2" version = "2.0.0" @@ -934,12 +993,56 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + +[[package]] +name = "rstest" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros", +] + +[[package]] +name = "rstest_macros" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn", + "unicode-ident", +] + [[package]] name = "rustc-demangle" version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.38.44" @@ -1251,12 +1354,18 @@ dependencies = [ "indexmap", "serde", "serde_spanned", - "toml_datetime", + "toml_datetime 0.7.0", "toml_parser", "toml_writer", "winnow", ] +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" + [[package]] name = "toml_datetime" version = "0.7.0" @@ -1266,6 +1375,17 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "toml_datetime 0.6.11", + "winnow", +] + [[package]] name = "toml_parser" version = "1.0.1" @@ -1622,6 +1742,9 @@ name = "winnow" version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95" +dependencies = [ + "memchr", +] [[package]] name = "wit-bindgen-rt" diff --git a/Cargo.toml b/Cargo.toml index adf6ad015..265a882a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ clap_complete = "4.5.55" git2 = "0.20.1" glob = "0.3.2" oci-spec = "0.8.2" +rstest = "0.26.1" semver = { version = "1.0.26", features = ["serde"] } serde = { version = "1.0.217", features = ["derive"] } serde_json = "1.0.140" diff --git a/rust/boil/Cargo.toml b/rust/boil/Cargo.toml index 4ed5c140a..6ae1c39c7 100644 --- a/rust/boil/Cargo.toml +++ b/rust/boil/Cargo.toml @@ -22,3 +22,6 @@ time.workspace = true tokio.workspace = true toml.workspace = true url.workspace = true + +[dev-dependencies] +rstest.workspace = true diff --git a/rust/boil/src/build/image.rs b/rust/boil/src/build/image.rs index 3ef463402..fda888e65 100644 --- a/rust/boil/src/build/image.rs +++ b/rust/boil/src/build/image.rs @@ -11,8 +11,11 @@ use snafu::{ResultExt as _, Snafu, ensure}; use crate::{IfContext, build::docker::BuildArguments}; -#[derive(Debug, Snafu)] +#[derive(Debug, PartialEq, Snafu)] pub enum ParseImageError { + #[snafu(display("input must not be empty"))] + EmptyInput, + #[snafu(display("encountered invalid format, expected name[=version,...]"))] InvalidFormat, @@ -32,17 +35,32 @@ pub struct Image { impl FromStr for Image { type Err = ParseImageError; - fn from_str(s: &str) -> Result { - let parts: Vec<_> = s.split('=').collect(); + fn from_str(input: &str) -> Result { + // Get rid of any leading and traling whitespace + let input = input.trim(); + ensure!(!input.is_empty(), EmptyInputSnafu); + + let parts: Vec<_> = input.split('=').collect(); - ensure!(!parts[0].contains(['.', '~']), UnsupportedCharsSnafu); + // Ensure that the path/image name is not empty, doesn't contain '~', and is not abolute. + ensure!(!parts[0].is_empty(), InvalidFormatSnafu); + ensure!(!parts[0].contains('~'), UnsupportedCharsSnafu); ensure!(!parts[0].starts_with('/'), AbsolutePathSnafu); - let image_name = parts[0].trim_end_matches('/').to_owned(); + // Get rid of a leading ./ from the image name. This would need to be replaced, because + // Docker doesn't allow dots in various places (like target names). Additionally, it would + // clutter the different names. The same applies for a trailing slash. + let image_name = parts[0] + .trim_start_matches("./") + .trim_end_matches('/') + .to_owned(); match parts.len() { 1 => Ok(Self::new_unversioned(image_name)), 2 => { + // Ensure that the version part is not empty + ensure!(!parts[1].is_empty(), InvalidFormatSnafu); + let versions: Vec<_> = parts[1].split(',').map(ToOwned::to_owned).collect(); Ok(Self::new(image_name, versions)) } @@ -180,3 +198,37 @@ impl From<(String, ImageOptions)> for VersionOptionsPair { } } } + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case("my/image/in.a/folder/with/name=1.2.3-rc.1,4.5.6-rc.2", &["1.2.3-rc.1", "4.5.6-rc.2"])] + #[case("my/image/in.a/folder/with/name=1.2.3-rc.1", &["1.2.3-rc.1"])] + #[case("my.image.in.a.folder.with/name=1.2.3", &["1.2.3"])] + #[case("my/image/in.a/folder/with/name", &[])] + #[case("my.image.in.a.folder.with/name", &[])] + #[case("my/image/with/name=1.2.3", &["1.2.3"])] + #[case("my/image/with/name", &[])] + #[case("name=1.2.3", &["1.2.3"])] + #[case("name", &[])] + fn valid(#[case] input: &str, #[case] expected_versions: &[&str]) { + let Image { versions, .. } = Image::from_str(input).expect("must be a valid image"); + assert_eq!(versions, expected_versions); + } + + #[rstest] + #[case("double/equal/image=1.2.3=4.5.6", ParseImageError::InvalidFormat)] + #[case("~/image/folder/with/tilde", ParseImageError::UnsupportedChars)] + #[case("/absolute/image/folder", ParseImageError::AbsolutePath)] + #[case("empty/version/image=", ParseImageError::InvalidFormat)] + #[case(" ", ParseImageError::EmptyInput)] + #[case("", ParseImageError::EmptyInput)] + fn invalid(#[case] input: &str, #[case] expected_error: ParseImageError) { + let error = Image::from_str(input).expect_err("invalid image must not parse"); + assert_eq!(error, expected_error); + } +} From 71c8f6f519b13af733ea81d0e59ace3aae17dc2c Mon Sep 17 00:00:00 2001 From: Techassi Date: Fri, 12 Sep 2025 20:51:01 +0200 Subject: [PATCH 51/51] chore: Update slab to 0.4.11 to fix RUSTSEC-2025-0047 See https://rustsec.org/advisories/RUSTSEC-2025-0047 --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1b9c5fd5d..0412df929 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1144,9 +1144,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" [[package]] name = "smallvec"