diff --git a/.cargo/config.toml b/.cargo/config.toml index b6cf3ff25..54e07b717 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,2 +1,3 @@ [alias] patchable = ["run", "--bin", "patchable", "--"] +boil = ["run", "--bin", "boil", "--"] diff --git a/.github/workflows/boil_build_release.yaml b/.github/workflows/boil_build_release.yaml new file mode 100644 index 000000000..45144b31b --- /dev/null +++ b/.github/workflows/boil_build_release.yaml @@ -0,0 +1,92 @@ +--- +name: Build/Release boil + +on: + pull_request: + paths: + - '.github/workflows/boil_build_release.yaml' + - 'rust-toolchain.toml' + - 'Cargo.*' + - '**.rs' + push: + tags: + - "boil-[0-9]+.[0-9]+.[0-9]+**" + +env: + RUST_VERSION: 1.87.0 + +jobs: + # This job is always run to ensure we don't miss any new upstream advisories + cargo-deny: + name: Run cargo-deny + runs-on: ubuntu-latest + # Prevent sudden announcement of a new advisory from failing CI + continue-on-error: ${{ matrix.checks == 'advisories' }} + strategy: + matrix: + checks: + - advisories + - bans licenses sources + steps: + - name: Checkout Repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + submodules: recursive + + - name: Run cargo-deny + uses: EmbarkStudios/cargo-deny-action@f2ba7abc2abebaf185c833c3961145a3c275caad # v2.0.13 + with: + command: check ${{ matrix.checks }} + + create-release: + name: Create Draft Release + if: github.event_name == 'push' + needs: + - cargo-deny + runs-on: ubuntu-latest + steps: + - name: Create Draft Release + uses: softprops/action-gh-release@72f2c25fcb47643c292f7107632f7a47c1df5cd8 # v2.3.2 + with: + draft: true + + build: + name: Build boil + needs: + - cargo-deny + strategy: + fail-fast: false + matrix: + targets: + - {target: aarch64-unknown-linux-gnu, os: ubuntu-24.04-arm} + - {target: x86_64-unknown-linux-gnu, os: ubuntu-latest} + - {target: aarch64-apple-darwin, os: macos-latest} + runs-on: ${{ matrix.targets.os }} + steps: + - name: Checkout + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + + - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 + with: + toolchain: ${{ env.RUST_VERSION }} + targets: ${{ matrix.targets.target }} + + - name: Build Binary + env: + TARGET: ${{ matrix.targets.target }} + run: cargo build --target "$TARGET" --release --package boil + + - name: Rename Binary + env: + TARGET: ${{ matrix.targets.target }} + run: mv "target/$TARGET/release/boil" "boil-$TARGET" + + - name: Upload Artifact to Release + if: github.event_name == 'push' + uses: softprops/action-gh-release@72f2c25fcb47643c292f7107632f7a47c1df5cd8 # v2.3.2 + with: + draft: false + files: boil-${{ matrix.targets.target }} diff --git a/.github/workflows/pr_pre-commit.yaml b/.github/workflows/pr_pre-commit.yaml index faa7e5c54..51e470a02 100644 --- a/.github/workflows/pr_pre-commit.yaml +++ b/.github/workflows/pr_pre-commit.yaml @@ -5,6 +5,8 @@ on: pull_request: env: + # Keep in sync with across other repos like operator-rs and operator-templating + RUST_TOOLCHAIN_VERSION: "nightly-2025-05-26" HADOLINT_VERSION: "v2.12.0" PYTHON_VERSION: "3.12" @@ -19,4 +21,5 @@ jobs: - uses: stackabletech/actions/run-pre-commit@497f3e3cbfe9b89b1e570351b97d050eebcad5d0 # 0.8.3 with: python-version: ${{ env.PYTHON_VERSION }} + rust: ${{ env.RUST_TOOLCHAIN_VERSION }} hadolint: ${{ env.HADOLINT_VERSION }} diff --git a/.github/workflows/reusable_build_image.yaml b/.github/workflows/reusable_build_image.yaml index a11cb8c80..5ae205d65 100644 --- a/.github/workflows/reusable_build_image.yaml +++ b/.github/workflows/reusable_build_image.yaml @@ -102,7 +102,7 @@ jobs: name: Failure Notification needs: [generate_matrix, build, publish_manifests] runs-on: ubuntu-latest - if: failure() || github.run_attempt > 1 + if: failure() || (github.run_attempt > 1 && !cancelled()) steps: - name: Send Notification uses: stackabletech/actions/send-slack-notification@55d2f9fcbcd7884ac929ea65fd6f069e7b7a49d2 # v0.8.1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0e71d9318..4af07cfe2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -60,3 +60,27 @@ repos: entry: .scripts/update_readme_badges.sh stages: [pre-commit, pre-merge-commit, manual] pass_filenames: false + + - id: cargo-test + name: cargo-test + language: system + entry: cargo test + stages: [pre-commit, pre-merge-commit] + pass_filenames: false + files: \.rs$|Cargo\.(toml|lock) + + - id: cargo-rustfmt + name: cargo-rustfmt + language: system + entry: cargo +nightly-2025-05-26 fmt --all -- --check + stages: [pre-commit, pre-merge-commit] + pass_filenames: false + files: \.rs$ + + - id: cargo-clippy + name: cargo-clippy + language: system + entry: cargo clippy --all-targets -- -D warnings + stages: [pre-commit, pre-merge-commit] + pass_filenames: false + files: \.rs$ diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..b3ec3613b --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,10 @@ +{ + "rust-analyzer.rustfmt.overrideCommand": [ + "rustfmt", + // Keep in sync with across other repos like operator-rs and operator-templating + "+nightly-2025-05-26", + "--edition", + "2024", + "--" + ], +} diff --git a/Cargo.lock b/Cargo.lock index 2b20caacd..0412df929 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aho-corasick" version = "1.1.3" @@ -47,7 +62,7 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -58,7 +73,7 @@ checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" dependencies = [ "anstyle", "once_cell", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -67,18 +82,60 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "backtrace" +version = "0.3.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets", +] + [[package]] name = "bitflags" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" +[[package]] +name = "boil" +version = "0.1.0" +dependencies = [ + "clap", + "clap_complete", + "git2", + "glob", + "oci-spec", + "rstest", + "semver", + "serde", + "serde_json", + "snafu", + "strum", + "time", + "tokio", + "toml", + "url", +] + [[package]] name = "bumpalo" version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + [[package]] name = "cc" version = "1.2.11" @@ -98,9 +155,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.5.27" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "769b0145982b4b48713e01ec42d61614425f27b7058bda7180a3a41f30104796" +checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9" dependencies = [ "clap_builder", "clap_derive", @@ -108,9 +165,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.27" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7" +checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d" dependencies = [ "anstream", "anstyle", @@ -118,11 +175,20 @@ dependencies = [ "strsim", ] +[[package]] +name = "clap_complete" +version = "4.5.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5abde44486daf70c5be8b8f8f1b66c49f86236edf6fa2abadb4d961c4c6229a" +dependencies = [ + "clap", +] + [[package]] name = "clap_derive" -version = "4.5.24" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" +checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" dependencies = [ "heck", "proc-macro2", @@ -152,18 +218,104 @@ dependencies = [ "libc", "once_cell", "unicode-width 0.2.0", - "windows-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "const_format" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "126f97965c8ad46d6d9163268ff28432e8f6a1196a55578867832e3049df63dd" +dependencies = [ + "const_format_proc_macros", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn", ] [[package]] name = "deranged" -version = "0.3.11" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ "powerfmt", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -194,7 +346,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -203,6 +355,12 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -212,6 +370,49 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-macro", + "futures-task", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "getrandom" version = "0.3.1" @@ -220,10 +421,28 @@ checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.13.3+wasi-0.2.2", "windows-targets", ] +[[package]] +name = "getset" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf0fc11e47561d47397154977bc219f4cf809b2974facc3ccb3b89e2436f912" +dependencies = [ + "proc-macro-error2", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + [[package]] name = "git2" version = "0.20.1" @@ -239,6 +458,12 @@ dependencies = [ "url", ] +[[package]] +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + [[package]] name = "hashbrown" version = "0.15.2" @@ -369,6 +594,12 @@ dependencies = [ "syn", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.0.3" @@ -414,6 +645,17 @@ dependencies = [ "web-time", ] +[[package]] +name = "io-uring" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -517,11 +759,11 @@ checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" [[package]] name = "matchers" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" dependencies = [ - "regex-automata 0.1.10", + "regex-automata", ] [[package]] @@ -530,14 +772,33 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", +] + [[package]] name = "nu-ansi-term" -version = "0.46.0" +version = "0.50.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" dependencies = [ - "overload", - "winapi", + "windows-sys 0.52.0", ] [[package]] @@ -552,6 +813,32 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "oci-spec" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2078e2f6be932a4de9aca90a375a45590809dfb5a08d93ab1ee217107aceeb67" +dependencies = [ + "const_format", + "derive_builder", + "getset", + "regex", + "serde", + "serde_json", + "strum", + "strum_macros", + "thiserror", +] + [[package]] name = "once_cell" version = "1.20.2" @@ -576,12 +863,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - [[package]] name = "patchable" version = "0.1.0" @@ -610,6 +891,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.31" @@ -628,6 +915,37 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "proc-macro-crate" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.93" @@ -654,17 +972,8 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.9", - "regex-syntax 0.8.5", -] - -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax 0.6.29", + "regex-automata", + "regex-syntax", ] [[package]] @@ -675,20 +984,64 @@ checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.5", + "regex-syntax", ] [[package]] name = "regex-syntax" -version = "0.6.29" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] -name = "regex-syntax" -version = "0.8.5" +name = "relative-path" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + +[[package]] +name = "rstest" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros", +] + +[[package]] +name = "rstest_macros" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn", + "unicode-ident", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] [[package]] name = "rustix" @@ -700,7 +1053,28 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "semver" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" +dependencies = [ + "serde", ] [[package]] @@ -723,11 +1097,23 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + [[package]] name = "serde_spanned" -version = "0.6.8" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +checksum = "40734c41988f7306bb04f0ecf60ec0f3f1caa34290e4e8ea471dcd3346483b83" dependencies = [ "serde", ] @@ -747,6 +1133,21 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook-registry" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + [[package]] name = "smallvec" version = "1.13.2" @@ -786,6 +1187,28 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + [[package]] name = "syn" version = "2.0.96" @@ -819,7 +1242,27 @@ dependencies = [ "getrandom", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "thiserror" +version = "2.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d76d3f064b981389ecb4b6b7f45a0bf9fdac1d5b9204c7bd6714fecc302850" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d29feb33e986b6ea906bd9c3559a856983f92371b3eaa5e83782a351623de0" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -834,11 +1277,12 @@ dependencies = [ [[package]] name = "time" -version = "0.3.37" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" dependencies = [ "deranged", + "itoa", "num-conv", "powerfmt", "serde", @@ -848,15 +1292,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" [[package]] name = "time-macros" -version = "0.2.19" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" dependencies = [ "num-conv", "time-core", @@ -872,40 +1316,91 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tokio" +version = "1.46.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" +dependencies = [ + "backtrace", + "bytes", + "io-uring", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "slab", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "toml" -version = "0.8.19" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +checksum = "ed0aee96c12fa71097902e0bb061a5e1ebd766a6636bb605ba401c45c1650eac" dependencies = [ + "indexmap", "serde", "serde_spanned", - "toml_datetime", - "toml_edit", + "toml_datetime 0.7.0", + "toml_parser", + "toml_writer", + "winnow", ] [[package]] name = "toml_datetime" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" + +[[package]] +name = "toml_datetime" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bade1c3e902f58d73d3f294cd7f20391c1cb2fbcb643b73566bc773971df91e3" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.22.23" +version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02a8b472d1a3d7c18e2d61a489aee3453fd9031c33e4f55bd533f4a7adca1bee" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ "indexmap", - "serde", - "serde_spanned", - "toml_datetime", + "toml_datetime 0.6.11", "winnow", ] +[[package]] +name = "toml_parser" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97200572db069e74c512a14117b296ba0a80a30123fbbb5aa1f4a348f639ca30" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b679217f2848de74cabd3e8fc5e6d66f40b7da40f8e1954d92054d9010690fd5" + [[package]] name = "tracing" version = "0.1.41" @@ -963,14 +1458,14 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" dependencies = [ "matchers", "nu-ansi-term", "once_cell", - "regex", + "regex-automata", "sharded-slab", "smallvec", "thread_local", @@ -997,6 +1492,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "url" version = "2.5.4" @@ -1006,6 +1507,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -1071,6 +1573,12 @@ dependencies = [ "quote", ] +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "wasi" version = "0.13.3+wasi-0.2.2" @@ -1148,27 +1656,14 @@ dependencies = [ ] [[package]] -name = "winapi" -version = "0.3.9" +name = "windows-sys" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", + "windows-targets", ] -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows-sys" version = "0.59.0" @@ -1244,9 +1739,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.7.0" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e49d2d35d3fad69b39b94139037ecfb4f359f08958b9c11e7315ce770462419" +checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95" dependencies = [ "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index 805dab66f..265a882a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,14 +2,28 @@ members = ["rust/*"] resolver = "2" +[workspace.package] +repository = "https://github.com/stackabletech/docker-images" +authors = ["Stackable "] +license = "Apache-2.0" + [workspace.dependencies] -clap = { version = "4.5.27", features = ["derive"] } +clap = { version = "4.5.41", features = ["derive"] } +clap_complete = "4.5.55" git2 = "0.20.1" +glob = "0.3.2" +oci-spec = "0.8.2" +rstest = "0.26.1" +semver = { version = "1.0.26", features = ["serde"] } serde = { version = "1.0.217", features = ["derive"] } +serde_json = "1.0.140" snafu = "0.8.5" +strum = { version = "0.27.1", features = ["derive"] } tempfile = "3.16.0" -time = { version = "0.3.37", features = ["parsing"] } -toml = "0.8.19" +time = { version = "0.3.41", features = ["parsing", "formatting"] } +tokio = { version = "1.46.1", features = ["rt", "macros", "process"] } +toml = "0.9.2" tracing = "0.1.41" tracing-indicatif = "0.3.9" tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } +url = { version = "2.5.4", features = ["serde"] } diff --git a/README.md b/README.md index a5f2602ea..efd84522c 100644 --- a/README.md +++ b/README.md @@ -15,45 +15,28 @@ This repository contains Dockerfiles and scripts to build base images for use wi ## Prerequisites -* [Stackable Image Tools](https://github.com/stackabletech/image-tools) (`pip install image-tools-stackabletech`) +* [boil](./rust/boil//README.md) (`cargo boil`) * Docker including the [`buildx` plugin](https://github.com/docker/buildx) * Optional: [OpenShift preflight tool](https://github.com/redhat-openshift-ecosystem/openshift-preflight) to verify an image for OpenShift -## Build Product Images +## Build Images -Product images are published to the `oci.stackable.tech` registry under the `sdp` organization by default. +Images are published to the `oci.stackable.tech` registry under the `sdp` organization by default. -### Build single products locally +### Build images locally -To build and push product images to the default repository use this command: +Consult the [boil README](./rust/boil//README.md) which contains a broad selection of different commands to build images locally. -```sh -bake --product zookeeper --image 0.0.0-dev --push -``` - -This will build images for Apache ZooKeeper versions as defined in the `conf.py` file, tag them with the `image-version` 0.0.0-dev and push them to the registry. - -You can select a specific version of a product to build using the syntax `product=version` e.g. to build Hive 3.1.3 you can use this command: - -```sh -bake --product hive=3.1.3 -i 0.0.0-dev -``` - -> [!NOTE] -> `-i` is the shorthand for `--image` (i.e. the resulting image tag) - -### Build all products locally - -To build all products in all versions locally you can use this command - -```sh -bake --image-version 0.0.0-dev -``` +### Build images via GitHub Actions -### Build everything in GitHub +There are individual GHA workflows (one for each image) which use a +[reusable workflow](.github/workflows/reusable_build_image.yaml) to build all specified versions for +both `amd64` and `arm64`. The workflow is triggered -The GitHub action called `Build (and optionally publish) 0.0.0-dev images` can be triggered manually to do build all images in all versions. -When triggered manually it will _not_ push the images to the registry. +* by pushes to `main` to produce `0.0.0-dev` versions of the images, +* by a regular schedule to rebuild `0.0.0-dev` versions of the images to avoid staleness, +* by tag pushes to produce (release candidate) images for a particular SDP release, +* and by manual workflow dispatches. ## Patches diff --git a/airflow/Dockerfile b/airflow/Dockerfile index 837c50066..164edb912 100644 --- a/airflow/Dockerfile +++ b/airflow/Dockerfile @@ -3,19 +3,19 @@ # - SecretsUsedInArgOrEnv : OPA_AUTH_MANAGER is a false positive and breaks the build. # check=error=true;skip=InvalidDefaultArgInFrom,SecretsUsedInArgOrEnv -ARG GIT_SYNC +ARG GIT_SYNC_VERSION # For updated versions check https://github.com/kubernetes/git-sync/releases # which should contain a image location (e.g. registry.k8s.io/git-sync/git-sync:v3.6.8) -FROM oci.stackable.tech/sdp/git-sync/git-sync:${GIT_SYNC} AS gitsync-image +FROM oci.stackable.tech/sdp/git-sync/git-sync:${GIT_SYNC_VERSION} AS gitsync-image -FROM stackable/image/shared/statsd-exporter AS statsd_exporter-builder +FROM local-image/shared/statsd-exporter AS statsd_exporter-builder -FROM stackable/image/vector AS opa-auth-manager-builder +FROM local-image/vector AS opa-auth-manager-builder ARG OPA_AUTH_MANAGER -ARG PYTHON -ARG UV +ARG PYTHON_VERSION +ARG UV_VERSION COPY airflow/opa-auth-manager/${OPA_AUTH_MANAGER} /tmp/opa-auth-manager @@ -23,10 +23,10 @@ WORKDIR /tmp/opa-auth-manager RUN < /stackable/app/airflow-${PRODUCT}.cdx.json +end)' /tmp/sbom.json > /stackable/app/airflow-${PRODUCT_VERSION}.cdx.json EOF COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter /stackable/statsd_exporter -COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter-${SHARED_STATSD_EXPORTER}.cdx.json /stackable/statsd_exporter-${SHARED_STATSD_EXPORTER}.cdx.json +COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter-${SHARED_STATSD_EXPORTER_VERSION}.cdx.json /stackable/statsd_exporter-${SHARED_STATSD_EXPORTER_VERSION}.cdx.json COPY --from=gitsync-image --chown=${STACKABLE_USER_UID}:0 /git-sync /stackable/git-sync RUN < /tmp/DRUID_SOURCE_DIR +RUN /stackable/patchable --images-repo-root=src checkout druid ${PRODUCT_VERSION} > /tmp/DRUID_SOURCE_DIR -RUN --mount=type=cache,id=maven-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository \ - --mount=type=cache,id=npm-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.npm \ - --mount=type=cache,id=cache-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.cache \ +RUN --mount=type=cache,id=maven-${PRODUCT_VERSION},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository \ + --mount=type=cache,id=npm-${PRODUCT_VERSION},uid=${STACKABLE_USER_UID},target=/stackable/.npm \ + --mount=type=cache,id=cache-${PRODUCT_VERSION},uid=${STACKABLE_USER_UID},target=/stackable/.cache \ < /stackable/package_manifest.txt +rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt chmod g=u /stackable/package_manifest.txt rm -rf /var/cache/yum -ln -sf /stackable/apache-druid-${PRODUCT}-stackable${RELEASE} /stackable/druid +ln -sf /stackable/apache-druid-${PRODUCT_VERSION}-stackable${RELEASE_VERSION} /stackable/druid chown -h ${STACKABLE_USER_UID}:0 stackable/druid # Force to overwrite the existing 'run-druid' @@ -159,7 +159,7 @@ chown -h ${STACKABLE_USER_UID}:0 /stackable/druid/bin/run-druid # fix missing permissions chmod -R g=u /stackable/bin -chmod g=u /stackable/apache-druid-${PRODUCT}-stackable${RELEASE} /stackable/druid-${PRODUCT}-stackable${RELEASE}-src.tar.gz +chmod g=u /stackable/apache-druid-${PRODUCT_VERSION}-stackable${RELEASE_VERSION} /stackable/druid-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-src.tar.gz EOF # ---------------------------------------- diff --git a/druid/boil-config.toml b/druid/boil-config.toml new file mode 100644 index 000000000..8a30bc28e --- /dev/null +++ b/druid/boil-config.toml @@ -0,0 +1,26 @@ +[versions."30.0.1".local-images] +# https://druid.apache.org/docs/30.0.1/operations/java/ +java-base = "17" +java-devel = "17" +"hadoop/hadoop" = "3.3.6" + +[versions."30.0.1".build-arguments] +authorizer-version = "0.7.0" + +[versions."31.0.1".local-images] +# https://druid.apache.org/docs/31.0.1/operations/java/ +java-base = "17" +java-devel = "17" +"hadoop/hadoop" = "3.3.6" + +[versions."31.0.1".build-arguments] +authorizer-version = "0.7.0" + +[versions."33.0.0".local-images] +# https://druid.apache.org/docs/33.0.0/operations/java/ +java-base = "17" +java-devel = "17" +"hadoop/hadoop" = "3.3.6" + +[versions."33.0.0".build-arguments] +authorizer-version = "0.7.0" diff --git a/druid/versions.py b/druid/versions.py deleted file mode 100644 index e380f7c81..000000000 --- a/druid/versions.py +++ /dev/null @@ -1,26 +0,0 @@ -versions = [ - { - "product": "30.0.1", - # https://druid.apache.org/docs/30.0.1/operations/java/ - "java-base": "17", - "java-devel": "17", - "hadoop/hadoop": "3.3.6", - "authorizer": "0.7.0", - }, - { - "product": "31.0.1", - # https://druid.apache.org/docs/31.0.1/operations/java/ - "java-base": "17", - "java-devel": "17", - "hadoop/hadoop": "3.3.6", - "authorizer": "0.7.0", - }, - { - "product": "33.0.0", - # https://druid.apache.org/docs/33.0.0/operations/java/ - "java-base": "17", - "java-devel": "17", - "hadoop/hadoop": "3.3.6", - "authorizer": "0.7.0", - }, -] diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index 5ca3cd6b1..5a8892fc3 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -1,19 +1,19 @@ # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 # check=error=true -FROM stackable/image/hadoop/hadoop AS hadoop-builder +FROM local-image/hadoop/hadoop AS hadoop-builder -FROM stackable/image/java-devel AS hdfs-utils-builder +FROM local-image/java-devel AS hdfs-utils-builder -ARG HDFS_UTILS -ARG PRODUCT -ARG RELEASE +ARG HDFS_UTILS_VERSION +ARG PRODUCT_VERSION +ARG RELEASE_VERSION ARG STACKABLE_USER_UID -ARG HADOOP_HADOOP +ARG HADOOP_HADOOP_VERSION # Reassign the arg to `HADOOP_VERSION` for better readability. -# It is passed as `HADOOP_HADOOP`, because versions.py has to contain `hadoop/hadoop` to establish a dependency on the Hadoop builder. -# The value of `hadoop/hadoop` is transformed by `bake` and automatically passed as `HADOOP_HADOOP` arg. -ENV HADOOP_VERSION=${HADOOP_HADOOP} +# It is passed as `HADOOP_HADOOP_VERSION`, because versions.py has to contain `hadoop/hadoop` to establish a dependency on the Hadoop builder. +# The value of `hadoop/hadoop` is transformed by `bake` and automatically passed as `HADOOP_HADOOP_VERSION` arg. +ENV HADOOP_VERSION=${HADOOP_HADOOP_VERSION} # Starting with hdfs-utils 0.4.0 we need to use Java 17 for compilation. # We can not simply use java-devel with Java 17, as it is also used to compile Hadoop in this @@ -33,7 +33,7 @@ USER ${STACKABLE_USER_UID} WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 hadoop/hdfs-utils/stackable/patches/patchable.toml /stackable/src/hadoop/hdfs-utils/stackable/patches/patchable.toml -COPY --chown=${STACKABLE_USER_UID}:0 hadoop/hdfs-utils/stackable/patches/${HDFS_UTILS} /stackable/src/hadoop/hdfs-utils/stackable/patches/${HDFS_UTILS} +COPY --chown=${STACKABLE_USER_UID}:0 hadoop/hdfs-utils/stackable/patches/${HDFS_UTILS_VERSION} /stackable/src/hadoop/hdfs-utils/stackable/patches/${HDFS_UTILS_VERSION} COPY --from=hadoop-builder --chown=${STACKABLE_USER_UID}:0 /stackable/patched-libs /stackable/patched-libs @@ -42,40 +42,40 @@ COPY --from=hadoop-builder --chown=${STACKABLE_USER_UID}:0 /stackable/patched-li # labels to build a rackID from. # Starting with hdfs-utils version 0.3.0 the topology provider is not a standalone jar anymore and included in hdfs-utils. RUN < /stackable/package_manifest.txt +rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt chmod g=u /stackable/package_manifest.txt rm -rf /var/cache/yum @@ -119,21 +119,21 @@ rm -rf /var/cache/yum # It is so non-root users (as we are) can mount a FUSE device and let other users access it echo "user_allow_other" > /etc/fuse.conf -ln -s "/stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE}" /stackable/hadoop +ln -s "/stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION}" /stackable/hadoop # async-profiler ARCH="${TARGETARCH/amd64/x64}" -curl "https://repo.stackable.tech/repository/packages/async-profiler/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}.tar.gz" | tar -xzC /stackable -ln -s "/stackable/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}" /stackable/async-profiler +curl "https://repo.stackable.tech/repository/packages/async-profiler/async-profiler-${ASYNC_PROFILER_VERSION}-${TARGETOS}-${ARCH}.tar.gz" | tar -xzC /stackable +ln -s "/stackable/async-profiler-${ASYNC_PROFILER_VERSION}-${TARGETOS}-${ARCH}" /stackable/async-profiler # JMX Exporter -curl "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" -o "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" -chmod -x "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" -ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar +curl "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" -o "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" +chmod -x "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" +ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar # Set correct permissions and ownerships -chown --recursive ${STACKABLE_USER_UID}:0 /stackable/hadoop /stackable/jmx /stackable/async-profiler "/stackable/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}" -chmod --recursive g=u /stackable/jmx /stackable/async-profiler "/stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE}" +chown --recursive ${STACKABLE_USER_UID}:0 /stackable/hadoop /stackable/jmx /stackable/async-profiler "/stackable/async-profiler-${ASYNC_PROFILER_VERSION}-${TARGETOS}-${ARCH}" +chmod --recursive g=u /stackable/jmx /stackable/async-profiler "/stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION}" # Workaround for https://issues.apache.org/jira/browse/HADOOP-12845 # The problem is that our stackable-devel image does contain the openssl-devel package diff --git a/hadoop/boil-config.toml b/hadoop/boil-config.toml new file mode 100644 index 000000000..6636ae943 --- /dev/null +++ b/hadoop/boil-config.toml @@ -0,0 +1,20 @@ +# Not part of SDP 25.7.0, but still required for hbase, hive, spark-k8s +[versions."3.3.6".local-images] +"hadoop/hadoop" = "3.3.6" +java-base = "11" +java-devel = "11" + +[versions."3.3.6".build-arguments] +async-profiler-version = "2.9" +jmx-exporter-version = "1.3.0" +hdfs-utils-version = "0.4.0" + +[versions."3.4.1".local-images] +"hadoop/hadoop" = "3.4.1" +java-base = "11" +java-devel = "11" + +[versions."3.4.1".build-arguments] +async-profiler-version = "2.9" +jmx-exporter-version = "1.3.0" +hdfs-utils-version = "0.4.1" diff --git a/hadoop/hadoop/Dockerfile b/hadoop/hadoop/Dockerfile index 342eef556..a17feec73 100644 --- a/hadoop/hadoop/Dockerfile +++ b/hadoop/hadoop/Dockerfile @@ -1,17 +1,17 @@ # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 # check=error=true -FROM stackable/image/java-devel AS hadoop-builder +FROM local-image/java-devel AS hadoop-builder -ARG PRODUCT -ARG RELEASE -ARG PROTOBUF +ARG PRODUCT_VERSION +ARG RELEASE_VERSION +ARG PROTOBUF_VERSION ARG STACKABLE_USER_UID WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/patchable.toml /stackable/src/shared/protobuf/stackable/patches/patchable.toml -COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/${PROTOBUF} /stackable/src/shared/protobuf/stackable/patches/${PROTOBUF} +COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/${PROTOBUF_VERSION} /stackable/src/shared/protobuf/stackable/patches/${PROTOBUF_VERSION} RUN < /stackable/bin/export-snapshot-to-s3 +envsubst '${HBASE_VERSION}:${RELEASE_VERSION}:${LIBS}' < /stackable/bin/export-snapshot-to-s3.env > /stackable/bin/export-snapshot-to-s3 chmod +x /stackable/bin/export-snapshot-to-s3 rm /stackable/bin/export-snapshot-to-s3.env @@ -58,35 +58,35 @@ chmod --recursive g=u /stackable EOF # Final Image -FROM stackable/image/java-base AS final +FROM local-image/java-base AS final -ARG PRODUCT -ARG RELEASE -ARG HADOOP_HADOOP +ARG PRODUCT_VERSION +ARG RELEASE_VERSION +ARG HADOOP_HADOOP_VERSION # Reassign the arg to `HADOOP_VERSION` for better readability. -ENV HADOOP_VERSION=${HADOOP_HADOOP} +ENV HADOOP_VERSION=${HADOOP_HADOOP_VERSION} ARG HBASE_PROFILE -ARG HBASE_HBASE +ARG HBASE_HBASE_VERSION # Reassign the arg to `HBASE_VERSION` for better readability. -ENV HBASE_VERSION=${HBASE_HBASE} -ARG HBASE_HBASE_OPERATOR_TOOLS -ARG HBASE_HBASE_OPA_AUTHORIZER -ARG HBASE_PHOENIX +ENV HBASE_VERSION=${HBASE_HBASE_VERSION} +ARG HBASE_HBASE_OPERATOR_TOOLS_VERSION +ARG HBASE_HBASE_OPA_AUTHORIZER_VERSION +ARG HBASE_PHOENIX_VERSION ARG STACKABLE_USER_UID ARG NAME="Apache HBase" ARG DESCRIPTION="This image is deployed by the Stackable Operator for Apache HBase" LABEL name="${NAME}" -LABEL version="${PRODUCT}" -LABEL release="${RELEASE}" +LABEL version="${PRODUCT_VERSION}" +LABEL release="${RELEASE_VERSION}" LABEL summary="The Stackable image for Apache HBase" LABEL description="${DESCRIPTION}" # https://github.com/opencontainers/image-spec/blob/036563a4a268d7c08b51a08f05a02a0fe74c7268/annotations.md#annotations LABEL org.opencontainers.image.documentation="https://docs.stackable.tech/home/stable/hbase/" -LABEL org.opencontainers.image.version="${PRODUCT}" -LABEL org.opencontainers.image.revision="${RELEASE}" +LABEL org.opencontainers.image.version="${PRODUCT_VERSION}" +LABEL org.opencontainers.image.revision="${RELEASE_VERSION}" LABEL org.opencontainers.image.title="${NAME}" LABEL org.opencontainers.image.description="${DESCRIPTION}" @@ -96,17 +96,17 @@ LABEL io.openshift.tags="ubi9,stackable,hbase,sdp,nosql" LABEL io.k8s.description="${DESCRIPTION}" LABEL io.k8s.display-name="${NAME}" -COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE} /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE}/ -COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE}-src.tar.gz /stackable +COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE_VERSION} /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE_VERSION}/ +COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE_VERSION}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/async-profiler /stackable/async-profiler/ -COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS}-stackable${RELEASE} /stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS}-stackable${RELEASE}/ -COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS}-stackable${RELEASE}-src.tar.gz /stackable +COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS_VERSION}-stackable${RELEASE_VERSION} /stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS_VERSION}-stackable${RELEASE_VERSION}/ +COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS_VERSION}-stackable${RELEASE_VERSION}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/bin/hbck2 /stackable/bin/hbck2 -COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/bin/hbase-entrypoint.sh /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE}/bin/hbase-entrypoint.sh +COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools /stackable/bin/hbase-entrypoint.sh /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE_VERSION}/bin/hbase-entrypoint.sh COPY --chown=${STACKABLE_USER_UID}:0 --from=phoenix /stackable/phoenix /stackable/phoenix/ -COPY --chown=${STACKABLE_USER_UID}:0 --from=phoenix /stackable/phoenix-${HBASE_PHOENIX}-stackable${RELEASE}-src.tar.gz /stackable +COPY --chown=${STACKABLE_USER_UID}:0 --from=phoenix /stackable/phoenix-${HBASE_PHOENIX_VERSION}-stackable${RELEASE_VERSION}-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-s3-builder /stackable/bin/export-snapshot-to-s3 /stackable/bin/export-snapshot-to-s3 COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-s3-builder /stackable/hadoop/share/hadoop/tools/lib/ /stackable/hadoop/share/hadoop/tools/lib/ @@ -116,11 +116,11 @@ COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-s3-builder /stackable/hadoop/ # hadoop-azure-${HADOOP}.jar contains the AzureBlobFileSystem which is required # by hadoop-common-${HADOOP}.jar if the scheme of a file system is "abfs://". COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder \ - /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP_VERSION}-stackable${RELEASE}.jar \ - /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE}/lib/ + /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP_VERSION}-stackable${RELEASE_VERSION}.jar \ + /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE_VERSION}/lib/ -COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-opa-authorizer /stackable/hbase-opa-authorizer-${HBASE_HBASE_OPA_AUTHORIZER}-src.tar.gz /stackable -COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-opa-authorizer /stackable/hbase-opa-authorizer/target/hbase-opa-authorizer*.jar /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE}/lib +COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-opa-authorizer /stackable/hbase-opa-authorizer-${HBASE_HBASE_OPA_AUTHORIZER_VERSION}-src.tar.gz /stackable +COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-opa-authorizer /stackable/hbase-opa-authorizer/target/hbase-opa-authorizer*.jar /stackable/hbase-${HBASE_VERSION}-stackable${RELEASE_VERSION}/lib RUN < /stackable/package_manifest.txt +rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt chmod g=u /stackable/package_manifest.txt rm -rf /var/cache/yum -ln --symbolic --logical --verbose "/stackable/hbase-${HBASE_VERSION}-stackable${RELEASE}" /stackable/hbase +ln --symbolic --logical --verbose "/stackable/hbase-${HBASE_VERSION}-stackable${RELEASE_VERSION}" /stackable/hbase chown --no-dereference ${STACKABLE_USER_UID}:0 /stackable/hbase chmod g=u /stackable/hbase -ln --symbolic --logical --verbose "/stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS}-stackable${RELEASE}" /stackable/hbase-operator-tools +ln --symbolic --logical --verbose "/stackable/hbase-operator-tools-${HBASE_HBASE_OPERATOR_TOOLS_VERSION}-stackable${RELEASE_VERSION}" /stackable/hbase-operator-tools chown --no-dereference ${STACKABLE_USER_UID}:0 /stackable/hbase-operator-tools chmod g=u /stackable/hbase-operator-tools diff --git a/hbase/boil-config.toml b/hbase/boil-config.toml new file mode 100644 index 000000000..2aa8bd793 --- /dev/null +++ b/hbase/boil-config.toml @@ -0,0 +1,25 @@ +[versions."2.6.1".local-images] +"hbase/hbase" = "2.6.1" +"hbase/hbase-operator-tools" = "1.3.0-fd5a5fb-hbase2.6.1" +"hbase/phoenix" = "5.2.1-hbase2.6.1" +"hbase/hbase-opa-authorizer" = "0.1.0" # only for HBase 2.6.1 +"hadoop/hadoop" = "3.3.6" +java-base = "11" +java-devel = "11" + +[versions."2.6.1".build-arguments] +hbase-profile = "2.6" +delete-caches = "true" + +[versions."2.6.2".local-images] +"hbase/hbase" = "2.6.2" +"hbase/hbase-operator-tools" = "1.3.0-fd5a5fb-hbase2.6.2" +"hbase/phoenix" = "5.2.1-hbase2.6.2" +"hbase/hbase-opa-authorizer" = "0.1.0" # only for HBase 2.6.1 +"hadoop/hadoop" = "3.4.1" +java-base = "11" +java-devel = "11" + +[versions."2.6.2".build-arguments] +hbase-profile = "2.6" +delete-caches = "true" diff --git a/hbase/hbase-opa-authorizer/Dockerfile b/hbase/hbase-opa-authorizer/Dockerfile index 78f5a7115..c43e4990a 100644 --- a/hbase/hbase-opa-authorizer/Dockerfile +++ b/hbase/hbase-opa-authorizer/Dockerfile @@ -1,6 +1,6 @@ -FROM stackable/image/java-devel +FROM local-image/java-devel -ARG PRODUCT +ARG PRODUCT_VERSION ARG DELETE_CACHES ARG STACKABLE_USER_UID @@ -8,17 +8,17 @@ USER ${STACKABLE_USER_UID} WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-opa-authorizer/stackable/patches/patchable.toml /stackable/src/hbase/hbase-opa-authorizer/stackable/patches/patchable.toml -COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-opa-authorizer/stackable/patches/${PRODUCT} /stackable/src/hbase/hbase-opa-authorizer/stackable/patches/${PRODUCT} +COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase-opa-authorizer/stackable/patches/${PRODUCT_VERSION} /stackable/src/hbase/hbase-opa-authorizer/stackable/patches/${PRODUCT_VERSION} RUN --mount=type=cache,id=maven-opa,uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository < /stackable/package_manifest.txt +rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt chmod g=u /stackable/package_manifest.txt rm -rf /var/cache/yum -chmod g=u /stackable/apache-hive-metastore-${PRODUCT}-stackable${RELEASE}-bin/bin/start-metastore +chmod g=u /stackable/apache-hive-metastore-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-bin/bin/start-metastore -ln -s /stackable/apache-hive-metastore-${PRODUCT}-stackable${RELEASE}-bin /stackable/hive-metastore +ln -s /stackable/apache-hive-metastore-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-bin /stackable/hive-metastore chown -h ${STACKABLE_USER_UID}:0 /stackable/hive-metastore chmod g=u /stackable/hive-metastore -ln -s /stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE} /stackable/hadoop +ln -s /stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION} /stackable/hadoop chown -h ${STACKABLE_USER_UID}:0 /stackable/hadoop chmod g=u /stackable/hadoop chmod g=u /stackable/*-src.tar.gz diff --git a/hive/boil-config.toml b/hive/boil-config.toml new file mode 100644 index 000000000..7bbcca500 --- /dev/null +++ b/hive/boil-config.toml @@ -0,0 +1,38 @@ +[versions."3.1.3".local-images] +# Hive 3 must be built with Java 8 but will run on Java 11 +java-base = "11" +java-devel = "8" +"hadoop/hadoop" = "3.3.6" + +[versions."3.1.3".build-arguments] +jmx-exporter-version = "1.3.0" +# Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 +aws-java-sdk-bundle-version = "1.12.367" +azure-storage-version = "7.0.1" +azure-keyvault-core-version = "1.0.0" + +[versions."4.0.0".local-images] +# Hive 4 must be built with Java 8 (according to GitHub README) but seems to run on Java 11 +java-base = "11" +java-devel = "8" +"hadoop/hadoop" = "3.3.6" + +[versions."4.0.0".build-arguments] +jmx-exporter-version = "1.3.0" +# Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 +aws-java-sdk-bundle-version = "1.12.367" +azure-storage-version = "7.0.1" +azure-keyvault-core-version = "1.0.0" + +[versions."4.0.1".local-images] +# Hive 4 must be built with Java 8 (according to GitHub README) but seems to run on Java 11 +java-base = "11" +java-devel = "8" +"hadoop/hadoop" = "3.3.6" + +[versions."4.0.1".build-arguments] +jmx-exporter-version = "1.3.0" +# Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 +aws-java-sdk-bundle-version = "1.12.367" +azure-storage-version = "7.0.1" +azure-keyvault-core-version = "1.0.0" diff --git a/hive/versions.py b/hive/versions.py deleted file mode 100644 index d28fa84be..000000000 --- a/hive/versions.py +++ /dev/null @@ -1,38 +0,0 @@ -versions = [ - { - "product": "3.1.3", - "jmx_exporter": "1.3.0", - # Hive 3 must be built with Java 8 but will run on Java 11 - "java-base": "11", - "java-devel": "8", - "hadoop/hadoop": "3.3.6", - # Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 - "aws_java_sdk_bundle": "1.12.367", - "azure_storage": "7.0.1", - "azure_keyvault_core": "1.0.0", - }, - { - "product": "4.0.0", - "jmx_exporter": "1.3.0", - # Hive 4 must be built with Java 8 (according to GitHub README) but seems to run on Java 11 - "java-base": "11", - "java-devel": "8", - "hadoop/hadoop": "3.3.6", - # Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 - "aws_java_sdk_bundle": "1.12.367", - "azure_storage": "7.0.1", - "azure_keyvault_core": "1.0.0", - }, - { - "product": "4.0.1", - "jmx_exporter": "1.3.0", - # Hive 4 must be built with Java 8 (according to GitHub README) but seems to run on Java 11 - "java-base": "11", - "java-devel": "8", - "hadoop/hadoop": "3.3.6", - # Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 - "aws_java_sdk_bundle": "1.12.367", - "azure_storage": "7.0.1", - "azure_keyvault_core": "1.0.0", - }, -] diff --git a/java-base/Dockerfile b/java-base/Dockerfile index 397587d2f..610eb5a46 100644 --- a/java-base/Dockerfile +++ b/java-base/Dockerfile @@ -4,16 +4,16 @@ # # Provides the common Java Runtime for SDP products # -FROM stackable/image/vector +FROM local-image/vector -ARG PRODUCT -ARG RELEASE="1" +ARG PRODUCT_VERSION +ARG RELEASE_VERSION="1" LABEL name="Stackable image for OpenJDK" \ maintainer="info@stackable.tech" \ vendor="Stackable GmbH" \ - version="${PRODUCT}" \ - release="${RELEASE}" \ + version="${PRODUCT_VERSION}" \ + release="${RELEASE_VERSION}" \ summary="The Stackable OpenJDK base image." \ description="This image is the base image for all Stackable Java product images." @@ -32,7 +32,7 @@ EOF RUN microdnf update && \ microdnf install \ # Needed to run Java programs - "temurin-${PRODUCT}-jre" \ + "temurin-${PRODUCT_VERSION}-jre" \ # Needed, because otherwise e.g. Zookeeper fails with # Caused by: java.io.FileNotFoundException: /usr/lib/jvm/java-11-openjdk-11.0.20.0.8-2.el8.x86_64/lib/tzdb.dat (No such file or directory) tzdata-java \ @@ -43,7 +43,7 @@ RUN microdnf update && \ COPY java-base/licenses /licenses -ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT}-jre" +ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT_VERSION}-jre" # This image doesn't include the development packages for Java. # For images that need the devel package (ex. Spark) use this env variable to @@ -51,7 +51,7 @@ ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT}-jre" # # microdnf install java-${JAVA_VERSION}-openjdk-devel # -ENV JAVA_VERSION=$PRODUCT +ENV JAVA_VERSION=$PRODUCT_VERSION # Mitigation for CVE-2021-44228 (Log4Shell) # This variable is supported as of Log4j version 2.10 and diff --git a/java-base/boil-config.toml b/java-base/boil-config.toml new file mode 100644 index 000000000..c7968ebf0 --- /dev/null +++ b/java-base/boil-config.toml @@ -0,0 +1,20 @@ +[versions."8".local-images] +vector = "0.47.0" + +[versions."11".local-images] +vector = "0.47.0" + +[versions."17".local-images] +vector = "0.47.0" + +[versions."21".local-images] +vector = "0.47.0" + +[versions."22".local-images] +vector = "0.47.0" + +[versions."23".local-images] +vector = "0.47.0" + +[versions."24".local-images] +vector = "0.47.0" diff --git a/java-base/versions.py b/java-base/versions.py deleted file mode 100644 index 036d18203..000000000 --- a/java-base/versions.py +++ /dev/null @@ -1,30 +0,0 @@ -versions = [ - { - "product": "8", - "vector": "0.47.0", - }, - { - "product": "11", - "vector": "0.47.0", - }, - { - "product": "17", - "vector": "0.47.0", - }, - { - "product": "21", - "vector": "0.47.0", - }, - { - "product": "22", - "vector": "0.47.0", - }, - { - "product": "23", - "vector": "0.47.0", - }, - { - "product": "24", - "vector": "0.47.0", - }, -] diff --git a/java-devel/Dockerfile b/java-devel/Dockerfile index 8a4343aed..b88e26498 100644 --- a/java-devel/Dockerfile +++ b/java-devel/Dockerfile @@ -5,9 +5,9 @@ # Base image for builder stages in Java based products # -FROM stackable/image/stackable-devel +FROM local-image/stackable-devel -ARG PRODUCT +ARG PRODUCT_VERSION ARG STACKABLE_USER_UID # Find the latest version here: https://github.com/apache/maven @@ -46,7 +46,7 @@ microdnf install \ `# Needed by the maven ant run plugin for the "set-hostname-property" step in zookeeper` \ hostname \ `# Needed for compiling Java projects` \ - "temurin-${PRODUCT}-jdk" \ + "temurin-${PRODUCT_VERSION}-jdk" \ krb5-devel \ libcurl-devel \ make \ @@ -66,7 +66,7 @@ ln -s /opt/maven/bin/mvn /usr/bin/mvn EOF -ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT}-jdk" +ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT_VERSION}-jdk" ENV MAVEN_ARGS="--batch-mode --no-transfer-progress" COPY --chown=${STACKABLE_USER_UID}:0 java-devel/stackable/settings.xml /stackable/.m2/settings.xml diff --git a/java-devel/boil-config.toml b/java-devel/boil-config.toml new file mode 100644 index 000000000..b6d750b42 --- /dev/null +++ b/java-devel/boil-config.toml @@ -0,0 +1,20 @@ +[versions."8".local-images] +stackable-devel = "1.0.0" + +[versions."11".local-images] +stackable-devel = "1.0.0" + +[versions."17".local-images] +stackable-devel = "1.0.0" + +[versions."21".local-images] +stackable-devel = "1.0.0" + +[versions."22".local-images] +stackable-devel = "1.0.0" + +[versions."23".local-images] +stackable-devel = "1.0.0" + +[versions."24".local-images] +stackable-devel = "1.0.0" diff --git a/java-devel/versions.py b/java-devel/versions.py deleted file mode 100644 index cd0838e4f..000000000 --- a/java-devel/versions.py +++ /dev/null @@ -1,30 +0,0 @@ -versions = [ - { - "product": "8", - "stackable-devel": "1.0.0", - }, - { - "product": "11", - "stackable-devel": "1.0.0", - }, - { - "product": "17", - "stackable-devel": "1.0.0", - }, - { - "product": "21", - "stackable-devel": "1.0.0", - }, - { - "product": "22", - "stackable-devel": "1.0.0", - }, - { - "product": "23", - "stackable-devel": "1.0.0", - }, - { - "product": "24", - "stackable-devel": "1.0.0", - }, -] diff --git a/jdk-base/Dockerfile b/jdk-base/Dockerfile index 776804c1d..914ab6cf9 100644 --- a/jdk-base/Dockerfile +++ b/jdk-base/Dockerfile @@ -4,16 +4,16 @@ # # Provides the common Java Development Kit for SDP products # -FROM stackable/image/vector +FROM local-image/vector -ARG PRODUCT -ARG RELEASE="1" +ARG PRODUCT_VERSION +ARG RELEASE_VERSION="1" LABEL name="Stackable image for OpenJDK" \ maintainer="info@stackable.tech" \ vendor="Stackable GmbH" \ - version="${PRODUCT}" \ - release="${RELEASE}" \ + version="${PRODUCT_VERSION}" \ + release="${RELEASE_VERSION}" \ summary="The Stackable OpenJDK base image." \ description="This image is the base image for all Stackable Java product images which require a JDK." @@ -32,7 +32,7 @@ EOF RUN microdnf update && \ microdnf install \ # Needed to run Java programs - "temurin-${PRODUCT}-jdk" \ + "temurin-${PRODUCT_VERSION}-jdk" \ # Needed, because otherwise e.g. Zookeeper fails with # Caused by: java.io.FileNotFoundException: /usr/lib/jvm/java-11-openjdk-11.0.20.0.8-2.el8.x86_64/lib/tzdb.dat (No such file or directory) tzdata-java \ @@ -43,7 +43,7 @@ RUN microdnf update && \ COPY java-base/licenses /licenses -ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT}-jdk" +ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT_VERSION}-jdk" # This image doesn't include the development packages for Java. # For images that need the devel package (ex. Spark) use this env variable to @@ -51,7 +51,7 @@ ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT}-jdk" # # microdnf install java-${JAVA_VERSION}-openjdk-devel # -ENV JAVA_VERSION=$PRODUCT +ENV JAVA_VERSION=$PRODUCT_VERSION # Mitigation for CVE-2021-44228 (Log4Shell) # This variable is supported as of Log4j version 2.10 and diff --git a/jdk-base/boil-config.toml b/jdk-base/boil-config.toml new file mode 100644 index 000000000..c7968ebf0 --- /dev/null +++ b/jdk-base/boil-config.toml @@ -0,0 +1,20 @@ +[versions."8".local-images] +vector = "0.47.0" + +[versions."11".local-images] +vector = "0.47.0" + +[versions."17".local-images] +vector = "0.47.0" + +[versions."21".local-images] +vector = "0.47.0" + +[versions."22".local-images] +vector = "0.47.0" + +[versions."23".local-images] +vector = "0.47.0" + +[versions."24".local-images] +vector = "0.47.0" diff --git a/jdk-base/versions.py b/jdk-base/versions.py deleted file mode 100644 index 036d18203..000000000 --- a/jdk-base/versions.py +++ /dev/null @@ -1,30 +0,0 @@ -versions = [ - { - "product": "8", - "vector": "0.47.0", - }, - { - "product": "11", - "vector": "0.47.0", - }, - { - "product": "17", - "vector": "0.47.0", - }, - { - "product": "21", - "vector": "0.47.0", - }, - { - "product": "22", - "vector": "0.47.0", - }, - { - "product": "23", - "vector": "0.47.0", - }, - { - "product": "24", - "vector": "0.47.0", - }, -] diff --git a/kafka-testing-tools/Dockerfile b/kafka-testing-tools/Dockerfile index cbfb4299c..7f1e563d7 100644 --- a/kafka-testing-tools/Dockerfile +++ b/kafka-testing-tools/Dockerfile @@ -1,20 +1,20 @@ # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 # check=error=true -FROM stackable/image/kafka/kcat AS kcat +FROM local-image/kafka/kcat AS kcat -FROM stackable/image/stackable-base AS final +FROM local-image/stackable-base AS final -ARG PRODUCT -ARG KAFKA_KCAT -ARG RELEASE +ARG PRODUCT_VERSION +ARG KAFKA_KCAT_VERSION +ARG RELEASE_VERSION ARG STACKABLE_USER_UID LABEL name="Kafka Testing Tools" \ maintainer="info@stackable.tech" \ vendor="Stackable GmbH" \ - version="${PRODUCT}" \ - release="${RELEASE}" \ + version="${PRODUCT_VERSION}" \ + release="${RELEASE_VERSION}" \ summary="The Stackable image for the kcat tool." \ description="Used for integration testing" @@ -30,9 +30,9 @@ RUN microdnf install \ && rm -rf /var/cache/yum # Store kcat version with binary name and add softlink -COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat /stackable/kcat-${KAFKA_KCAT} -COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KAFKA_KCAT}-src.tar.gz /stackable -RUN ln -s /stackable/kcat-${KAFKA_KCAT} /stackable/kcat +COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat /stackable/kcat-${KAFKA_KCAT_VERSION} +COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KAFKA_KCAT_VERSION}-src.tar.gz /stackable +RUN ln -s /stackable/kcat-${KAFKA_KCAT_VERSION} /stackable/kcat COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /licenses /licenses COPY --chown=${STACKABLE_USER_UID}:0 kafka-testing-tools/licenses /licenses diff --git a/kafka-testing-tools/boil-config.toml b/kafka-testing-tools/boil-config.toml new file mode 100644 index 000000000..ca78f8116 --- /dev/null +++ b/kafka-testing-tools/boil-config.toml @@ -0,0 +1,4 @@ +[versions."1.0.0".local-images] +stackable-base = "1.0.0" +"kafka/kcat" = "1.7.0" +java-base = "11" diff --git a/kafka-testing-tools/versions.py b/kafka-testing-tools/versions.py deleted file mode 100644 index b706d446e..000000000 --- a/kafka-testing-tools/versions.py +++ /dev/null @@ -1,8 +0,0 @@ -versions = [ - { - "product": "1.0.0", - "kafka/kcat": "1.7.0", - "java-base": "11", - "stackable-base": "1.0.0", - } -] diff --git a/kafka/Dockerfile b/kafka/Dockerfile index e01f09fda..f9dffb9dd 100644 --- a/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -1,15 +1,15 @@ # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 # check=error=true -FROM stackable/image/kafka/kcat AS kcat -FROM stackable/image/kafka/kafka-opa-plugin AS kafka-opa-plugin +FROM local-image/kafka/kcat AS kcat +FROM local-image/kafka/kafka-opa-plugin AS kafka-opa-plugin -FROM stackable/image/java-devel AS kafka-builder +FROM local-image/java-devel AS kafka-builder -ARG PRODUCT -ARG RELEASE -ARG SCALA -ARG JMX_EXPORTER +ARG PRODUCT_VERSION +ARG RELEASE_VERSION +ARG SCALA_VERSION +ARG JMX_EXPORTER_VERSION ARG STACKABLE_USER_UID USER ${STACKABLE_USER_UID} @@ -17,15 +17,15 @@ WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/jmx/ /stackable/jmx/ COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/patchable.toml /stackable/src/kafka/stackable/patches/patchable.toml -COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/${PRODUCT} /stackable/src/kafka/stackable/patches/${PRODUCT} +COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/patches/${PRODUCT_VERSION} /stackable/src/kafka/stackable/patches/${PRODUCT_VERSION} RUN < /stackable/package_manifest.txt +rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt chmod g=u /stackable/package_manifest.txt rm -rf /var/cache/yum -ln -s /stackable/bin/kcat-${KAFKA_KCAT} /stackable/bin/kcat +ln -s /stackable/bin/kcat-${KAFKA_KCAT_VERSION} /stackable/bin/kcat chown -h ${STACKABLE_USER_UID}:0 /stackable/bin/kcat # kcat was located in /stackable/kcat - legacy ln -s /stackable/bin/kcat /stackable/kcat chown -h ${STACKABLE_USER_UID}:0 /stackable/kcat -ln -s /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE} /stackable/kafka +ln -s /stackable/kafka_${SCALA_VERSION}-${PRODUCT_VERSION}-stackable${RELEASE_VERSION} /stackable/kafka chown -h ${STACKABLE_USER_UID}:0 /stackable/kafka # fix missing permissions chmod g=u /stackable/bin chmod g=u /stackable/jmx -chmod g=u /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE} -chmod g=u /stackable/kafka_${SCALA}-${PRODUCT}-stackable${RELEASE}/libs/opa-authorizer-${KAFKA_KAFKA_OPA_PLUGIN}-all.jar +chmod g=u /stackable/kafka_${SCALA_VERSION}-${PRODUCT_VERSION}-stackable${RELEASE_VERSION} +chmod g=u /stackable/kafka_${SCALA_VERSION}-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}/libs/opa-authorizer-${KAFKA_KAFKA_OPA_PLUGIN_VERSION}-all.jar chmod g=u /stackable/*-src.tar.gz EOF diff --git a/kafka/boil-config.toml b/kafka/boil-config.toml new file mode 100644 index 000000000..9d7fc2ad1 --- /dev/null +++ b/kafka/boil-config.toml @@ -0,0 +1,39 @@ +[versions."3.7.2".local-images] +java-base = "21" +java-devel = "21" +"kafka/kcat" = "1.7.0" +"kafka/kafka-opa-plugin" = "1.5.1" + +[versions."3.7.2".build-arguments] +scala-version = "2.13" +jmx-exporter-version = "1.3.0" + +[versions."3.9.0".local-images] +java-base = "21" +java-devel = "21" +"kafka/kcat" = "1.7.0" +"kafka/kafka-opa-plugin" = "1.5.1" + +[versions."3.9.0".build-arguments] +scala-version = "2.13" +jmx-exporter-version = "1.3.0" + +[versions."3.9.1".local-images] +java-base = "21" +java-devel = "21" +"kafka/kcat" = "1.7.0" +"kafka/kafka-opa-plugin" = "1.5.1" + +[versions."3.9.1".build-arguments] +scala-version = "2.13" +jmx-exporter-version = "1.3.0" + +[versions."4.0.0".local-images] +java-base = "23" +java-devel = "23" +"kafka/kcat" = "1.7.0" +"kafka/kafka-opa-plugin" = "1.5.1" + +[versions."4.0.0".build-arguments] +scala-version = "2.13" +jmx-exporter-version = "1.3.0" diff --git a/kafka/kafka-opa-plugin/Dockerfile b/kafka/kafka-opa-plugin/Dockerfile index 2a5df3166..f963a2cb3 100644 --- a/kafka/kafka-opa-plugin/Dockerfile +++ b/kafka/kafka-opa-plugin/Dockerfile @@ -1,22 +1,22 @@ # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 # check=error=true -FROM stackable/image/java-devel +FROM local-image/java-devel -ARG PRODUCT +ARG PRODUCT_VERSION ARG STACKABLE_USER_UID USER ${STACKABLE_USER_UID} WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 kafka/kafka-opa-plugin/stackable/patches/patchable.toml /stackable/src/kafka/kafka-opa-plugin/stackable/patches/patchable.toml -COPY --chown=${STACKABLE_USER_UID}:0 kafka/kafka-opa-plugin/stackable/patches/${PRODUCT} /stackable/src/kafka/kafka-opa-plugin/stackable/patches/${PRODUCT} +COPY --chown=${STACKABLE_USER_UID}:0 kafka/kafka-opa-plugin/stackable/patches/${PRODUCT_VERSION} /stackable/src/kafka/kafka-opa-plugin/stackable/patches/${PRODUCT_VERSION} RUN <= 1.23.1) go install github.com/CycloneDX/cyclonedx-gomod/cmd/cyclonedx-gomod@v1.7.0 -cd "$(/stackable/patchable --images-repo-root=src checkout opa ${PRODUCT})" +cd "$(/stackable/patchable --images-repo-root=src checkout opa ${PRODUCT_VERSION})" -ORIGINAL_VERSION=${PRODUCT} -NEW_VERSION="${PRODUCT}-stackable${RELEASE}" +ORIGINAL_VERSION=${PRODUCT_VERSION} +NEW_VERSION="${PRODUCT_VERSION}-stackable${RELEASE_VERSION}" sed -i 's/var Version = "'${ORIGINAL_VERSION}'"/var Version = "'${NEW_VERSION}'"/g' v1/version/version.go @@ -102,24 +100,24 @@ mv opa /stackable/opa/ chmod -R g=u /stackable/opa /stackable/opa-${NEW_VERSION}-src.tar.gz EOF -FROM stackable/image/vector +FROM local-image/vector -ARG PRODUCT -ARG RELEASE +ARG PRODUCT_VERSION +ARG RELEASE_VERSION ARG STACKABLE_USER_UID LABEL name="Open Policy Agent" \ maintainer="info@stackable.tech" \ vendor="Stackable GmbH" \ - version="${PRODUCT}" \ - release="${RELEASE}" \ + version="${PRODUCT_VERSION}" \ + release="${RELEASE_VERSION}" \ summary="The Stackable image for Open Policy Agent (OPA)." \ description="This image is deployed by the Stackable Operator for OPA." COPY --chown=${STACKABLE_USER_UID}:0 opa/licenses /licenses COPY --from=opa-builder --chown=${STACKABLE_USER_UID}:0 /stackable/opa /stackable/opa -COPY --from=opa-builder --chown=${STACKABLE_USER_UID}:0 /stackable/opa-${PRODUCT}-stackable${RELEASE}-src.tar.gz /stackable/opa-${PRODUCT}-stackable${RELEASE}-src.tar.gz +COPY --from=opa-builder --chown=${STACKABLE_USER_UID}:0 /stackable/opa-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-src.tar.gz /stackable/opa-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-src.tar.gz COPY --from=multilog-builder --chown=${STACKABLE_USER_UID}:0 /daemontools/admin/daemontools/command/multilog /stackable/multilog RUN <>); + +impl Deref for Targets { + type Target = BTreeMap>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for Targets { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl IntoIterator for Targets { + type IntoIter = + std::collections::btree_map::IntoIter>; + type Item = (String, BTreeMap); + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +impl Targets { + /// Returns a map of all targets by globbing for (nested) image config files. + /// + /// The search behaviour can be customized using the provided [`TargetsOptions`]. + pub fn all(options: TargetsOptions) -> Result { + let image_config_paths = glob(ImageConfig::ALL_CONFIGS_GLOB_PATTERN) + .expect("glob pattern must be valid") + .filter_map(Result::ok); + + let mut targets = Self::default(); + + for image_config_path in image_config_paths { + let image_config = + ImageConfig::from_file(&image_config_path).context(ReadImageConfigSnafu)?; + + let image_name = image_config_path + .parent() + .expect("there must be a parent") + .to_string_lossy() + .into_owned(); + + let pairs = image_config.all(); + + targets.insert_targets(image_name.to_owned(), pairs, &options, true)?; + } + + Ok(targets) + } + + /// Returns a filtered set out of all targets by looking up selected image config files. + /// + /// The search behaviour can be customized using the provided [`TargetsOptions`]. + pub fn set(images: &[Image], options: TargetsOptions) -> Result { + let mut targets = Self::default(); + + for image in images { + // TODO (@Techassi): We should instead build the graph based on the Dockerfile(s), + // because this is the source of truth and what ultimately gets built. The boil config + // files are not a source a truth, but just provide data needed during the build. + let image_config_path = PathBuf::new() + .join(&image.name) + .join(ImageConfig::DEFAULT_FILE_NAME); + + // Read the image config which defines supported image versions and their dependencies as + // well as other values. + let image_config = + ImageConfig::from_file(image_config_path).context(ReadImageConfigSnafu)?; + + // Create a list of image versions we need to generate targets for in the bakefile. + let pairs = image_config + .filter_by_version(&image.versions) + .context(InvalidImageVersionSnafu)?; + + targets.insert_targets(image.name.clone(), pairs, &options, true)?; + } + + Ok(targets) + } + + fn insert_targets( + &mut self, + image_name: String, + pairs: Vec, + options: &TargetsOptions, + is_entry: bool, + ) -> Result<(), TargetsError> { + for VersionOptionsPair { + version: image_version, + options: image_options, + } in pairs + { + if !options.only_entry { + // TODO (@Techassi): Add cycle detection + for (image_name, image_version) in &image_options.local_images { + if self + .get(image_name) + .is_some_and(|image_versions| image_versions.contains_key(image_version)) + { + continue; + } + + let image_config_path = PathBuf::new() + .join(image_name) + .join(ImageConfig::DEFAULT_FILE_NAME); + + let image_config = + ImageConfig::from_file(image_config_path).context(ReadImageConfigSnafu)?; + + let pairs = image_config + .filter_by_version(&[image_version]) + .context(InvalidImageVersionSnafu)?; + + // Wowzers, recursion! + self.insert_targets(image_name.clone(), pairs, options, false)?; + } + } + + self.entry(image_name.clone()) + .or_default() + .insert(image_version, (image_options, is_entry)); + } + + Ok(()) + } +} + +#[derive(Debug, Default, Serialize)] +pub struct Bakefile { + #[serde(rename = "group")] + pub groups: BTreeMap, + + #[serde(rename = "target")] + pub targets: BTreeMap, +} + +impl Bakefile { + /// Create a bakefile from the [`BuildArguments`](cli::BuildArguments) provided via the CLI. + /// + /// This will only create targets for selected entry images and their dependencies. There is no + /// need to filter anything out afterwards. The filtering is done automatically internally. + pub fn from_args(args: &cli::BuildArguments, config: Config) -> Result { + let targets = + Targets::set(&args.images, TargetsOptions::default()).context(CreateGraphSnafu)?; + Self::from_targets(targets, args, config) + } + + /// Returns all image manifest URIs for entry images. + pub fn image_manifest_uris(&self) -> Vec<&str> { + self.targets + .iter() + // We only care about the entry targets, because those are the primary images boil + // builds. + .filter(|(target_name, _)| target_name.starts_with(ENTRY_TARGET_NAME_PREFIX)) + // The image manifest URIs file only contains the image tags + .flat_map(|(_, target)| &target.tags) + // Flatten multiple tags (boil currently only ever writes a single one, but the data + // structure can accept a list). + .map(|s| s.as_str()) + .collect() + } + + /// Creates the common target, containing shared data, which will be inherited by other targets. + fn common_target(args: &cli::BuildArguments, config: Config) -> Result { + let revision = Self::git_head_revision().context(GetRevisionSnafu)?; + let date_time = Self::now()?; + + // Load build arguments from a file if the user requested it + let mut build_arguments = args.build_arguments.clone(); + if let Some(path) = &args.build_arguments_file { + let build_arguments_from_file = + BuildArguments::from_file(path).context(ParseBuildArgumentsSnafu)?; + build_arguments.extend(build_arguments_from_file); + } + + let target = BakefileTarget::common( + date_time, + revision, + config, + build_arguments, + args.image_version.base_prerelease(), + ); + + Ok(target) + } + + fn from_targets( + targets: Targets, + args: &cli::BuildArguments, + config: Config, + ) -> Result { + let mut bakefile_targets = BTreeMap::new(); + let mut groups: BTreeMap = BTreeMap::new(); + + // Create a common target, which contains shared data, like annotations, arguments, labels, etc... + let common_target = Self::common_target(args, config)?; + bakefile_targets.insert(COMMON_TARGET_NAME.to_owned(), common_target); + + for (image_name, image_versions) in targets.into_iter() { + for (image_version, (image_options, is_entry)) in image_versions { + // TODO (@Techassi): Clean this up + // TODO (@Techassi): Move the arg formatting into functions + let mut build_arguments = BuildArguments::new(); + + let local_version_docker_args: Vec<_> = image_options + .local_images + .iter() + .map(|(image_name, image_version)| { + BuildArgument::new( + format!( + "{image_name}_VERSION", + image_name = image_name.to_uppercase().replace('-', "_") + ), + image_version.to_string(), + ) + }) + .collect(); + + build_arguments.extend(image_options.build_arguments); + build_arguments.extend(local_version_docker_args); + // TODO (@Techassi): Rename this to IMAGE_VERSION + build_arguments.insert(BuildArgument::new( + "PRODUCT_VERSION".to_owned(), + image_version.to_string(), + )); + + // The image registry, eg. `oci.stackable.tech` or `localhost` + let image_registry = if args.use_localhost_registry { + &Host::Domain(String::from("localhost")) + } else { + &args.registry + }; + + let image_repository_uri = format_image_repository_uri( + image_registry, + &args.registry_namespace, + &image_name, + ); + + let image_manifest_uri = format_image_manifest_uri( + &image_repository_uri, + &image_version, + &args.image_version, + args.target_platform.architecture(), + args.strip_architecture, + ); + + let dockerfile = PathBuf::new() + .join(&image_name) + .join(&args.target_containerfile); + + let target_name = if is_entry { + Self::format_entry_target_name(&image_name, &image_version) + } else { + Self::format_target_name(&image_name, &image_version) + }; + + let contexts: BTreeMap<_, _> = image_options + .local_images + .iter() + .map(|(image_name, image_version)| { + let context_name = Self::format_context_name(image_name); + let context_target = Self::format_context_target(image_name, image_version); + + (context_name, context_target) + }) + .collect(); + + let annotations = + BakefileTarget::image_version_annotation(&image_version, &args.image_version); + + let target = BakefileTarget { + tags: vec![image_manifest_uri], + arguments: build_arguments, + platforms: vec![args.target_platform.clone()], + // NOTE (@Techassi): Should this instead be scoped to the folder of the image we build + context: Some(PathBuf::from(".")), + dockerfile: Some(dockerfile), + inherits: vec![COMMON_TARGET_NAME.to_owned()], + annotations, + contexts, + ..Default::default() + }; + + bakefile_targets.insert(target_name, target); + + // Add the target to the default group if it is an entry + if is_entry { + groups + .entry("default".to_owned()) + .or_default() + .targets + .push(Self::format_entry_target_name(&image_name, &image_version)); + } + } + } + + Ok(Self { + targets: bakefile_targets, + groups, + }) + } + + /// Formats and returns the entry target name, eg. `entry--opa-1_4_2`. + fn format_entry_target_name(image_name: &str, image_version: &str) -> String { + let target_name = Self::format_target_name(image_name, image_version); + format!("{ENTRY_TARGET_NAME_PREFIX}{target_name}") + } + + /// Formats and returns the target name, eg. `stackable-base-1_0_0`. + fn format_target_name(image_name: &str, image_version: &str) -> String { + // Replace any slashes from nested image names, eg. shared/protobuf, because docker buildx + // has this weird restriction (because it also supports push, which we do on our own). We + // are therefore artificially limited what target names we can use: [a-zA-Z0-9_-]+ + let image_name = image_name.replace('/', "__"); + + // The dots in the semantic version also need to be replaced. + let image_version = image_version.replace('.', "_"); + + format!("{image_name}-{image_version}") + } + + /// Formats and return the context name, eg. `stackable/image/stackable-base-1_0_0`. + fn format_context_name(name: &str) -> String { + format!("local-image/{name}") + } + + /// Formats and returns the context target name, eg. `target:stackable-base-1_0_0`. + fn format_context_target(name: &str, version: &str) -> String { + let target_name = Self::format_target_name(name, version); + format!("target:{target_name}") + } + + fn now() -> Result { + time::UtcDateTime::now() + .format(&Rfc3339) + .context(FormatTimeSnafu) + } + + fn git_head_revision() -> Result { + let repo = git2::Repository::open(".").context(OpenRepositorySnafu)?; + let rev = repo.revparse("HEAD").context(ParseHeadRevisionSnafu)?; + let rev = rev.from().context(InvalidRangeSnafu)?.id().to_string(); + + Ok(rev) + } +} + +// TODO (@Techassi): Figure out of we can use borrowed data in here. This would avoid a whole bunch +// of cloning. +#[derive(Debug, Default, Serialize)] +pub struct BakefileTarget { + /// Defines build arguments for the target. + #[serde(rename = "args", skip_serializing_if = "BuildArguments::is_empty")] + pub arguments: BuildArguments, + + /// Adds annotations to images built with bake. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub annotations: Vec, + + /// Specifies the location of the build context to use for this target. + /// + /// Accepts a URL or a directory path. + #[serde(skip_serializing_if = "Option::is_none")] + pub context: Option, + + /// Additional build contexts. + /// + /// This attribute takes a map, where keys result in named contexts that you can reference in + /// your builds. + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub contexts: BTreeMap, + + /// Name of the Dockerfile to use for the build. + #[serde(skip_serializing_if = "Option::is_none")] + pub dockerfile: Option, + + /// A target can inherit attributes from other targets. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub inherits: Vec, + + /// Assigns image labels to the build. + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub labels: BTreeMap, + + // TODO (@Techassi): Explore how we can build multiple platforms at once + /// Set target platforms for the build target. + /// + /// Technically, multiple architectures can be listed in here, but boil chooses to build only + /// one architecture at a time. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub platforms: Vec, + + /// Image names and tags to use for the build target. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub tags: Vec, +} + +impl BakefileTarget { + fn common( + date_time: String, + revision: String, + config: Config, + build_arguments: Vec, + release_version: String, + ) -> Self { + let config::Metadata { + documentation, + licenses, + authors, + source, + vendor, + } = config.metadata; + + // Annotations describe OCI image components. + let annotations = vec![ + format!("{ANNOTATION_CREATED}={date_time}"), + format!("{ANNOTATION_AUTHORS}={authors}"), + format!("{ANNOTATION_DOCUMENTATION}={documentation}"), + format!("{ANNOTATION_SOURCE}={source}"), + format!("{ANNOTATION_REVISION}={revision}"), + format!("{ANNOTATION_VENDOR}={vendor}"), + format!("{ANNOTATION_LICENSES}={licenses}"), + ]; + + let mut arguments = config.build_arguments; + arguments.extend(build_arguments); + arguments.insert(BuildArgument::new( + "RELEASE_VERSION".to_owned(), + release_version, + )); + + // Labels describe Docker resources, and con be considered legacy. We + // should use annotations instead. These labels are only added to be + // consistent with `bake`. + let labels = BTreeMap::from([ + (ANNOTATION_CREATED.to_owned(), date_time.clone()), + (ANNOTATION_REVISION.to_owned(), revision), + (LABEL_BUILD_DATE.to_owned(), date_time), + ]); + + Self { + annotations, + arguments, + labels, + ..Default::default() + } + } + + fn image_version_annotation(image_version: &str, sdp_image_version: &Version) -> Vec { + vec![ + // TODO (@Techassi): Move this version formatting into a function + // TODO (@Techassi): Make this vendor agnostic, don't hard-code stackable here + format!("{ANNOTATION_VERSION}={image_version}-stackable{sdp_image_version}"), + ] + } +} + +#[derive(Debug, Default, Serialize)] +pub struct BakefileGroup { + targets: Vec, +} diff --git a/rust/boil/src/build/cli.rs b/rust/boil/src/build/cli.rs new file mode 100644 index 000000000..35756fcc0 --- /dev/null +++ b/rust/boil/src/build/cli.rs @@ -0,0 +1,151 @@ +use std::{path::PathBuf, str::FromStr}; + +use clap::{Args, ValueHint, value_parser}; +use semver::Version; +use snafu::{ResultExt, Snafu, ensure}; +use url::Host; + +use crate::build::{ + docker::BuildArgument, + image::Image, + platform::{Architecture, TargetPlatform}, +}; + +#[derive(Debug, Args)] +pub struct BuildArguments { + /// The image(s) which should be build. The format is name[=version,...]. + #[arg(help_heading = "Image Options", required = true)] + pub images: Vec, + + // The action currently does the wrong thing here. It includes the + // architecture even though it should come from the --target-platform arg. + // The release arg is NOT needed, because this version IS the release version. + /// The image version being built. + #[arg( + short, long, + value_parser = parse_image_version, + default_value_t = Self::default_image_version(), + help_heading = "Image Options" + )] + pub image_version: Version, + + /// Target platform of the image. + #[arg( + short, long, + short_alias = 'a', alias = "architecture", + default_value_t = Self::default_architecture(), + help_heading = "Image Options" + )] + pub target_platform: TargetPlatform, + + /// Image registry used in image manifests, URIs, and tags. + #[arg( + short, long, + default_value_t = Self::default_registry(), + value_parser = Host::parse, + value_hint = ValueHint::Hostname, + help_heading = "Registry Options" + )] + pub registry: Host, + + /// The namespace within the given registry. + #[arg( + short = 'n', + long = "registry-namespace", + alias = "organization", + default_value = "sdp", + help_heading = "Registry Options" + )] + pub registry_namespace: String, + + /// Use 'localhost' as the registry instead of to avoid any accidental interactions + /// with remote registries. + /// + /// This is especially useful in CI, which can re-tag the image before pushing it. + #[arg(long, help_heading = "Registry Options")] + pub use_localhost_registry: bool, + + /// Override the target containerfile used, points to /. + #[arg( + long, + default_value_os_t = Self::default_target_containerfile(), + value_hint = ValueHint::FilePath, + help_heading = "Build Options" + )] + pub target_containerfile: PathBuf, + + /// Override build arguments, in key=value format. The key is case insensitive. This argument + /// can be supplied multiple times. + #[arg( + long = "build-argument", + alias = "build-arg", + value_name = "BUILD_ARGUMENT", + help_heading = "Build Options" + )] + pub build_arguments: Vec, + + /// Load and override build arguments, in key=value format, each separated by a newline from the + /// specified file. + #[arg(long, alias = "build-args-file", help_heading = "Build Options")] + pub build_arguments_file: Option, + + /// Write target image tags to . Useful for signing or other follow-up CI steps. + #[arg( + long, + alias = "export-tags-file", + help_heading = "Build Options", + value_name = "FILE", + value_hint = ValueHint::FilePath, + value_parser = value_parser!(PathBuf), + default_missing_value = "boil-target-tags", + num_args(0..=1) + )] + pub write_image_manifest_uris: Option, + + /// Strips the architecture from the image (index) manifest tag. + #[arg(long, help_heading = "Build Options")] + pub strip_architecture: bool, + + /// Loads the image into the local image store. + #[arg(long, help_heading = "Build Options")] + pub load: bool, + + /// Dry run. This does not build the image(s) but instead prints out the bakefile. + #[arg(short, long, alias = "dry")] + pub dry_run: bool, +} + +impl BuildArguments { + fn default_image_version() -> Version { + "0.0.0-dev".parse().expect("must be a valid SemVer") + } + + // TODO: Auto-detect this + fn default_architecture() -> TargetPlatform { + TargetPlatform::Linux(Architecture::Amd64) + } + + fn default_registry() -> Host { + Host::Domain(String::from("oci.stackable.tech")) + } + + fn default_target_containerfile() -> PathBuf { + PathBuf::from("Dockerfile") + } +} + +#[derive(Debug, Snafu)] +pub enum ParseImageVersionError { + #[snafu(display("failed to parse semantic version"))] + ParseVersion { source: semver::Error }, + + #[snafu(display("semantic version must not contain build metadata"))] + ContainsBuildMetadata, +} + +pub fn parse_image_version(input: &str) -> Result { + let version = Version::from_str(input).context(ParseVersionSnafu)?; + ensure!(version.build.is_empty(), ContainsBuildMetadataSnafu); + + Ok(version) +} diff --git a/rust/boil/src/build/docker.rs b/rust/boil/src/build/docker.rs new file mode 100644 index 000000000..23e524f1c --- /dev/null +++ b/rust/boil/src/build/docker.rs @@ -0,0 +1,199 @@ +use std::{ + collections::BTreeSet, + fmt::Display, + ops::{Deref, DerefMut}, + path::{Path, PathBuf}, + str::FromStr, +}; + +use serde::{Deserialize, Serialize, de::Visitor, ser::SerializeMap}; +use snafu::{OptionExt, ResultExt, Snafu, ensure}; + +/// Label key for the date and time on which the image was built. +pub const LABEL_BUILD_DATE: &str = "build-date"; + +#[derive(Debug, Snafu)] +pub enum ParseBuildArgumentError { + #[snafu(display("invalid format, expected ="))] + InvalidFormat, + + #[snafu(display("encountered non ASCII characters"))] + NonAscii, +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct BuildArgument((String, String)); + +impl BuildArgument { + pub fn new(key: String, value: String) -> Self { + let key = Self::format_key(key); + Self((key, value)) + } + + fn format_key(key: impl AsRef) -> String { + key.as_ref().replace(['-', '/'], "_").to_uppercase() + } +} + +impl FromStr for BuildArgument { + type Err = ParseBuildArgumentError; + + fn from_str(s: &str) -> Result { + ensure!(s.is_ascii(), NonAsciiSnafu); + + let (key, value) = s.split_once('=').context(InvalidFormatSnafu)?; + let key = Self::format_key(key); + + Ok(Self((key, value.to_owned()))) + } +} + +impl<'de> Deserialize<'de> for BuildArgument { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct BuildArgumentVisitor; + + impl Visitor<'_> for BuildArgumentVisitor { + type Value = BuildArgument; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(formatter, "a valid build argument") + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + BuildArgument::from_str(v).map_err(serde::de::Error::custom) + } + } + + deserializer.deserialize_str(BuildArgumentVisitor) + } +} + +impl Display for BuildArgument { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let (key, value) = &self.0; + write!(f, "{key}={value}") + } +} + +#[derive(Debug, Snafu)] +pub enum ParseBuildArgumentsError { + #[snafu(display("failed to read file at {path}", path = path.display()))] + ReadFile { + source: std::io::Error, + path: PathBuf, + }, + + #[snafu(display("failed to parse build argument"))] + ParseBuildArgument { source: ParseBuildArgumentError }, +} + +#[derive(Clone, Debug, Default)] +pub struct BuildArguments(BTreeSet); + +impl Deref for BuildArguments { + type Target = BTreeSet; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for BuildArguments { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl Extend for BuildArguments { + fn extend>(&mut self, iter: T) { + self.0.extend(iter); + } +} + +impl IntoIterator for BuildArguments { + type IntoIter = std::collections::btree_set::IntoIter; + type Item = BuildArgument; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +impl<'de> Deserialize<'de> for BuildArguments { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct BuildArgumentsVisitor; + + impl<'de> Visitor<'de> for BuildArgumentsVisitor { + type Value = BuildArguments; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(formatter, "a map of valid build arguments") + } + + fn visit_map(self, mut map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + let mut args = BTreeSet::new(); + + while let Some((key, value)) = map.next_entry()? { + args.insert(BuildArgument::new(key, value)); + } + + Ok(BuildArguments(args)) + } + } + + deserializer.deserialize_map(BuildArgumentsVisitor) + } +} + +impl Serialize for BuildArguments { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut map = serializer.serialize_map(Some(self.len()))?; + + for BuildArgument((key, value)) in &self.0 { + map.serialize_entry(&key, &value)?; + } + + map.end() + } +} + +impl BuildArguments { + pub fn new() -> Self { + Self(BTreeSet::new()) + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn from_file

(path: P) -> Result + where + P: AsRef, + { + let path = path.as_ref(); + let content = std::fs::read_to_string(path).context(ReadFileSnafu { path })?; + let mut args = Self::new(); + + for line in content.lines() { + let arg = BuildArgument::from_str(line).context(ParseBuildArgumentSnafu)?; + args.insert(arg); + } + + Ok(args) + } +} diff --git a/rust/boil/src/build/image.rs b/rust/boil/src/build/image.rs new file mode 100644 index 000000000..fda888e65 --- /dev/null +++ b/rust/boil/src/build/image.rs @@ -0,0 +1,234 @@ +use std::{ + collections::BTreeMap, + fmt::Display, + ops::Deref, + path::{Path, PathBuf}, + str::FromStr, +}; + +use serde::Deserialize; +use snafu::{ResultExt as _, Snafu, ensure}; + +use crate::{IfContext, build::docker::BuildArguments}; + +#[derive(Debug, PartialEq, Snafu)] +pub enum ParseImageError { + #[snafu(display("input must not be empty"))] + EmptyInput, + + #[snafu(display("encountered invalid format, expected name[=version,...]"))] + InvalidFormat, + + #[snafu(display("the path contains unsupported characters: '.' or '~'"))] + UnsupportedChars, + + #[snafu(display("absolute paths are not supported"))] + AbsolutePath, +} + +#[derive(Clone, Debug)] +pub struct Image { + pub name: String, + pub versions: Vec, +} + +impl FromStr for Image { + type Err = ParseImageError; + + fn from_str(input: &str) -> Result { + // Get rid of any leading and traling whitespace + let input = input.trim(); + ensure!(!input.is_empty(), EmptyInputSnafu); + + let parts: Vec<_> = input.split('=').collect(); + + // Ensure that the path/image name is not empty, doesn't contain '~', and is not abolute. + ensure!(!parts[0].is_empty(), InvalidFormatSnafu); + ensure!(!parts[0].contains('~'), UnsupportedCharsSnafu); + ensure!(!parts[0].starts_with('/'), AbsolutePathSnafu); + + // Get rid of a leading ./ from the image name. This would need to be replaced, because + // Docker doesn't allow dots in various places (like target names). Additionally, it would + // clutter the different names. The same applies for a trailing slash. + let image_name = parts[0] + .trim_start_matches("./") + .trim_end_matches('/') + .to_owned(); + + match parts.len() { + 1 => Ok(Self::new_unversioned(image_name)), + 2 => { + // Ensure that the version part is not empty + ensure!(!parts[1].is_empty(), InvalidFormatSnafu); + + let versions: Vec<_> = parts[1].split(',').map(ToOwned::to_owned).collect(); + Ok(Self::new(image_name, versions)) + } + _ => InvalidFormatSnafu.fail(), + } + } +} + +impl Display for Image { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.versions.is_empty() { + f.write_str(&self.name) + } else { + write!( + f, + "{name}={versions}", + name = self.name, + versions = self.versions.join(",") + ) + } + } +} + +impl Image { + fn new(name: String, versions: Vec) -> Self { + Self { name, versions } + } + + fn new_unversioned(name: String) -> Self { + Self { + name, + versions: vec![], + } + } +} + +#[derive(Debug, Snafu)] +pub enum ImageConfigError { + #[snafu(display("failed to read config file at {path}", path = path.display()))] + ReadFile { + source: std::io::Error, + path: PathBuf, + }, + + #[snafu(display("failed to deserialize config file from TOML"))] + Deserialize { source: toml::de::Error }, + + #[snafu(display("provided filter version yielded empty list"))] + EmptyFilter, +} + +#[derive(Debug, Deserialize)] +pub struct ImageConfig { + // TODO (@Techassi): Eventually support this + // #[serde(default)] + // pub metadata: ImageMetadata, + pub versions: ImageVersions, +} + +impl ImageConfig { + /// This glob pattern matches all (deeply nested) image configs. + pub const ALL_CONFIGS_GLOB_PATTERN: &str = "**/boil-config.toml"; + /// The default image config file name. + pub const DEFAULT_FILE_NAME: &str = "boil-config.toml"; + + pub fn filter_by_version( + self, + versions: &[V], + ) -> Result, ImageConfigError> + where + V: AsRef + PartialEq, + { + let versions: Vec<_> = self + .pairs() + .filter(|(image_version, _)| { + versions.is_empty() || versions.iter().any(|v| v.as_ref() == image_version) + }) + .map(Into::into) + .collect(); + + versions.if_context(|v| !v.is_empty(), EmptyFilterSnafu) + } + + pub fn all(self) -> Vec { + self.pairs().map(Into::into).collect() + } + + fn pairs(self) -> impl Iterator { + self.versions.0.into_iter() + } +} + +impl ImageConfig { + pub fn from_file(path: impl AsRef) -> Result { + let path = path.as_ref(); + let contents = std::fs::read_to_string(path).with_context(|_| ReadFileSnafu { path })?; + toml::from_str(&contents).context(DeserializeSnafu) + } +} + +#[derive(Debug, Deserialize)] +pub struct ImageVersions(BTreeMap); + +impl Deref for ImageVersions { + type Target = BTreeMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct ImageOptions { + #[serde(default)] + pub local_images: BTreeMap, + + // NOTE (@Techassi): Potentially add a dependencies field here which will be automatically be + // suffixed with _VERSION. + #[serde(default)] + pub build_arguments: BuildArguments, +} + +#[derive(Debug)] +pub struct VersionOptionsPair { + pub version: String, + pub options: ImageOptions, +} + +impl From<(String, ImageOptions)> for VersionOptionsPair { + fn from(value: (String, ImageOptions)) -> Self { + VersionOptionsPair { + version: value.0, + options: value.1, + } + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case("my/image/in.a/folder/with/name=1.2.3-rc.1,4.5.6-rc.2", &["1.2.3-rc.1", "4.5.6-rc.2"])] + #[case("my/image/in.a/folder/with/name=1.2.3-rc.1", &["1.2.3-rc.1"])] + #[case("my.image.in.a.folder.with/name=1.2.3", &["1.2.3"])] + #[case("my/image/in.a/folder/with/name", &[])] + #[case("my.image.in.a.folder.with/name", &[])] + #[case("my/image/with/name=1.2.3", &["1.2.3"])] + #[case("my/image/with/name", &[])] + #[case("name=1.2.3", &["1.2.3"])] + #[case("name", &[])] + fn valid(#[case] input: &str, #[case] expected_versions: &[&str]) { + let Image { versions, .. } = Image::from_str(input).expect("must be a valid image"); + assert_eq!(versions, expected_versions); + } + + #[rstest] + #[case("double/equal/image=1.2.3=4.5.6", ParseImageError::InvalidFormat)] + #[case("~/image/folder/with/tilde", ParseImageError::UnsupportedChars)] + #[case("/absolute/image/folder", ParseImageError::AbsolutePath)] + #[case("empty/version/image=", ParseImageError::InvalidFormat)] + #[case(" ", ParseImageError::EmptyInput)] + #[case("", ParseImageError::EmptyInput)] + fn invalid(#[case] input: &str, #[case] expected_error: ParseImageError) { + let error = Image::from_str(input).expect_err("invalid image must not parse"); + assert_eq!(error, expected_error); + } +} diff --git a/rust/boil/src/build/mod.rs b/rust/boil/src/build/mod.rs new file mode 100644 index 000000000..6373c2c8e --- /dev/null +++ b/rust/boil/src/build/mod.rs @@ -0,0 +1,112 @@ +use std::{ + fmt::Debug, + process::{Command, Stdio}, +}; + +use snafu::{OptionExt, ResultExt, Snafu, ensure}; + +use crate::{ + build::{bakefile::Bakefile, cli::BuildArguments}, + config::Config, + utils::CommandExt, +}; + +pub mod bakefile; +pub mod cli; +pub mod docker; +pub mod image; +pub mod platform; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("failed to create bakefile"))] + CreateBakefile { source: bakefile::Error }, + + #[snafu(display("failed to write image manifest URIs to file"))] + WriteImageManifestUrisFile { source: std::io::Error }, + + #[snafu(display("failed to serialize bakefile as JSON"))] + SerializeBakefile { source: serde_json::Error }, + + #[snafu(display("failed to acquire stdin handle"))] + AcquireStdinHandle, + + #[snafu(display("failed to run child process"))] + RunChildProcess { source: std::io::Error }, + + #[snafu(display("failed to spawn child process"))] + SpawnChildProcess { source: std::io::Error }, + + #[snafu(display("encountered invalid image version, must not include any build metadata"))] + InvalidImageVersion, +} + +/// This is the `boil build` command handler function. +pub fn run_command(args: BuildArguments, config: Config) -> Result<(), Error> { + // TODO (@Techassi): Parse Dockerfile instead to build the target graph + // Validation + ensure!( + args.image_version.build.is_empty(), + InvalidImageVersionSnafu + ); + + // Create bakefile + let bakefile = Bakefile::from_args(&args, config).context(CreateBakefileSnafu)?; + let image_manifest_uris = bakefile.image_manifest_uris(); + let count = image_manifest_uris.len(); + + // Write the image manifest URIs to file if requested + if let Some(path) = args.write_image_manifest_uris { + std::fs::write(path, image_manifest_uris.join("\n")) + .context(WriteImageManifestUrisFileSnafu)?; + } + + // Output the bakefile contents if in dry-run mode + if args.dry_run { + return serde_json::to_writer_pretty(std::io::stdout(), &bakefile) + .context(SerializeBakefileSnafu); + } + + // TODO (@Techassi): Invoke this directly using the Docker daemon via bollard + // or by building the image ourself. + + // Finally invoke the docker buildx bake command + let mut child = Command::new("docker") + .arg("buildx") + .arg("bake") + .arg_if(args.load, "--load") + .arg("--file") + .arg("-") + .stdin(Stdio::piped()) + .spawn() + .context(SpawnChildProcessSnafu)?; + + // Acquire stdin handle to pipe the bakefile as JSON to it + let stdin_handle = child.stdin.take().with_context(|| { + child + .kill() + .expect("killing the child process must succeed"); + AcquireStdinHandleSnafu + })?; + + serde_json::to_writer(stdin_handle, &bakefile).with_context(|_| { + child + .kill() + .expect("killing the child process must succeed"); + SerializeBakefileSnafu + })?; + + // Wait for successful completion of the child process + let status = child.wait().context(RunChildProcessSnafu)?; + + // TODO (@Techassi): Return an error if the status was not a success + if status.success() { + println!( + "Successfully built {count} image{plural}:\n{images}", + plural = if count > 1 { "s" } else { "" }, + images = image_manifest_uris.join("\n") + ); + } + + Ok(()) +} diff --git a/rust/boil/src/build/platform.rs b/rust/boil/src/build/platform.rs new file mode 100644 index 000000000..857e4b532 --- /dev/null +++ b/rust/boil/src/build/platform.rs @@ -0,0 +1,67 @@ +use std::{fmt::Display, str::FromStr}; + +use serde::Serialize; +use snafu::{OptionExt as _, ResultExt, Snafu}; + +#[derive(Debug, Snafu)] +pub enum ParseArchitecturePairError { + #[snafu(display("encountered invalid format, expected platform/architecture"))] + InvalidFormat, + + #[snafu(display("failed to parse architecture"))] + ParseArchitecture { source: strum::ParseError }, + + #[snafu(display("unsupported, target platform"))] + UnsupportedPlatform, +} + +#[derive(Clone, Debug)] +pub enum TargetPlatform { + Linux(Architecture), +} + +impl Serialize for TargetPlatform { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&self.to_string()) + } +} + +impl FromStr for TargetPlatform { + type Err = ParseArchitecturePairError; + + fn from_str(s: &str) -> Result { + let (platform, architecture) = s.split_once('/').context(InvalidFormatSnafu)?; + let architecture = Architecture::from_str(architecture).context(ParseArchitectureSnafu)?; + + match platform { + "linux" => Ok(Self::Linux(architecture)), + _ => UnsupportedPlatformSnafu.fail(), + } + } +} + +impl Display for TargetPlatform { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self { + TargetPlatform::Linux(architecture) => write!(f, "linux/{architecture}"), + } + } +} + +impl TargetPlatform { + pub fn architecture(&self) -> &Architecture { + match self { + TargetPlatform::Linux(architecture) => architecture, + } + } +} + +#[derive(Copy, Clone, Debug, strum::Display, strum::EnumString, strum::AsRefStr)] +#[strum(serialize_all = "lowercase")] +pub enum Architecture { + Amd64, + Arm64, +} diff --git a/rust/boil/src/cli.rs b/rust/boil/src/cli.rs new file mode 100644 index 000000000..85323f289 --- /dev/null +++ b/rust/boil/src/cli.rs @@ -0,0 +1,37 @@ +use std::path::PathBuf; + +use clap::{Parser, Subcommand}; + +use crate::{build::cli::BuildArguments, completions::CompletionsArguments, show::ShowArguments}; + +#[derive(Debug, Parser)] +#[command(author, version, about)] +pub struct Cli { + /// Path to the configuration file. + #[arg(short, long = "configuration", global = true, default_value_os_t = Self::default_config_path())] + pub config_path: PathBuf, + + #[command(subcommand)] + pub command: Command, +} + +impl Cli { + fn default_config_path() -> PathBuf { + PathBuf::from("./boil.toml") + } +} + +#[derive(Debug, Subcommand)] +pub enum Command { + /// Build one or more images. + /// + /// Requires docker with the buildx extension. + #[command(alias = "some-chicken")] + Build(BuildArguments), + + /// Display various structured outputs in JSON format. + Show(ShowArguments), + + /// Generate shell completions. + Completions(CompletionsArguments), +} diff --git a/rust/boil/src/completions/mod.rs b/rust/boil/src/completions/mod.rs new file mode 100644 index 000000000..2c70386df --- /dev/null +++ b/rust/boil/src/completions/mod.rs @@ -0,0 +1,18 @@ +use clap::{Args, CommandFactory}; +use clap_complete::Shell; + +use crate::cli::Cli; + +#[derive(Debug, Args)] +pub struct CompletionsArguments { + /// Shell to generate completions for. + pub shell: Shell, +} + +/// This is the `boil completions` command handler function. +pub fn run_command(arguments: CompletionsArguments) { + let mut cli = Cli::command(); + let bin_name = cli.get_bin_name().unwrap_or("boil").to_owned(); + + clap_complete::generate(arguments.shell, &mut cli, bin_name, &mut std::io::stdout()); +} diff --git a/rust/boil/src/config.rs b/rust/boil/src/config.rs new file mode 100644 index 000000000..19d4ff3c0 --- /dev/null +++ b/rust/boil/src/config.rs @@ -0,0 +1,39 @@ +use std::path::Path; + +use serde::Deserialize; +use snafu::{ResultExt, Snafu}; +use url::Url; + +use crate::build::docker::BuildArguments; + +#[derive(Debug, Snafu)] +pub enum ConfigError { + ReadFile { source: std::io::Error }, + + Deserialize { source: toml::de::Error }, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct Config { + pub build_arguments: BuildArguments, + pub metadata: Metadata, +} + +impl Config { + pub fn from_file(path: impl AsRef) -> Result { + let contents = std::fs::read_to_string(path).context(ReadFileSnafu)?; + toml::from_str(&contents).context(DeserializeSnafu) + } +} + +// NOTE (@Techassi): Think about if these metadata fields should be required or optional. If they +// are optional, the appropriate annotations are only emitted if set. +#[derive(Debug, Deserialize)] +pub struct Metadata { + pub documentation: Url, + pub licenses: String, + pub authors: String, + pub vendor: String, + pub source: Url, +} diff --git a/rust/boil/src/main.rs b/rust/boil/src/main.rs new file mode 100644 index 000000000..189616ac7 --- /dev/null +++ b/rust/boil/src/main.rs @@ -0,0 +1,112 @@ +use clap::Parser; +use semver::Version; +use snafu::{ResultExt, Snafu}; + +use crate::{ + cli::{Cli, Command}, + config::Config, + show::ShowCommand, +}; + +// Common modules +mod cli; +mod config; +mod utils; + +// Command modules +mod build; +mod completions; +mod show; + +/// This trait extends functionailty provided by [`snafu`]. +/// +/// [`snafu`] already provides various ways to extend [`Result`]s with additional context-sensitive +/// information. This trait allows calling `if_context` on any type, which runs a predicate to +/// determine if an error with the provided context should be returned. +/// +/// This trait can be thought of as a combination of [`snafu::ensure!`] and returning [`Ok`] +/// afterwards. +pub trait IfContext: Sized { + /// Runs `predicate` and returns [`Ok`] if `true` or [`Err`] (with data from `context`) otherwise. + fn if_context(self, predicate: P, context: C) -> Result + where + P: Fn(&Self) -> bool, + C: snafu::IntoError, + E: std::error::Error + snafu::ErrorCompat; +} + +impl IfContext for T { + fn if_context(self, predicate: P, context: C) -> Result + where + P: Fn(&Self) -> bool, + C: snafu::IntoError, + E: std::error::Error + snafu::ErrorCompat, + { + match predicate(&self) { + true => Ok(self), + false => Err(context.into_error(snafu::NoneError)), + } + } +} + +pub trait VersionExt { + /// Returns the base of a [`Version`] as a string, eg. `1.2.3`. + fn base(&self) -> String; + + /// Returns the base and prerelease of a [`Version`] as a string, eg. `1.2.3-rc.1`. + fn base_prerelease(&self) -> String; +} + +impl VersionExt for Version { + fn base(&self) -> String { + let Self { + major, + minor, + patch, + .. + } = self; + + format!("{major}.{minor}.{patch}") + } + + fn base_prerelease(&self) -> String { + let mut base = self.base(); + base.push('-'); + base.push_str(&self.pre); + base + } +} + +#[derive(Debug, Snafu)] +enum Error { + #[snafu(display("failed to run build command"))] + Build { source: build::Error }, + + #[snafu(display("failed to run show command"))] + Show { source: show::images::Error }, + + #[snafu(display("failed to read config"))] + ReadConfig { source: config::ConfigError }, +} + +#[tokio::main(flavor = "current_thread")] +#[snafu::report] +async fn main() -> Result<(), Error> { + let cli = Cli::parse(); + + match cli.command { + Command::Build(arguments) => { + let config = Config::from_file(&cli.config_path).context(ReadConfigSnafu)?; + build::run_command(arguments, config).context(BuildSnafu) + } + Command::Show(arguments) => match arguments.commands { + ShowCommand::Images(arguments) => { + show::images::run_command(arguments).context(ShowSnafu) + } + }, + Command::Completions(arguments) => { + completions::run_command(arguments); + Ok(()) + } + } +} diff --git a/rust/boil/src/show/images/cli.rs b/rust/boil/src/show/images/cli.rs new file mode 100644 index 000000000..0c3469ef5 --- /dev/null +++ b/rust/boil/src/show/images/cli.rs @@ -0,0 +1,22 @@ +use clap::{Args, ValueEnum}; + +use crate::build::image::Image; + +#[derive(Debug, Args)] +pub struct ShowImagesArguments { + /// Optionally specify one or more images to display. + pub image: Vec, + + /// Pretty print the structured output. + #[arg(long, value_enum, default_value_t = Pretty::default())] + pub pretty: Pretty, +} + +// #[derive(Clone, Debug, Default, strum::Display, strum::EnumString)] +#[derive(Clone, Debug, Default, ValueEnum)] +pub enum Pretty { + #[default] + Auto, + Always, + Never, +} diff --git a/rust/boil/src/show/images/mod.rs b/rust/boil/src/show/images/mod.rs new file mode 100644 index 000000000..f977fd02e --- /dev/null +++ b/rust/boil/src/show/images/mod.rs @@ -0,0 +1,54 @@ +use std::{collections::BTreeMap, io::IsTerminal}; + +use snafu::{ResultExt, Snafu}; + +use crate::{ + build::bakefile::{Targets, TargetsError, TargetsOptions}, + show::images::cli::{Pretty, ShowImagesArguments}, +}; + +pub mod cli; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("failed to serialize list as JSON"))] + SerializeList { source: serde_json::Error }, + + #[snafu(display("failed to build list of targets"))] + BuildTargets { source: TargetsError }, +} + +/// This is the `boil show images` command handler function. +pub fn run_command(arguments: ShowImagesArguments) -> Result<(), Error> { + let list: BTreeMap<_, _> = if arguments.image.is_empty() { + Targets::all(TargetsOptions { only_entry: true }) + .context(BuildTargetsSnafu)? + .into_iter() + } else { + Targets::set(&arguments.image, TargetsOptions { only_entry: true }) + .context(BuildTargetsSnafu)? + .into_iter() + } + .map(|(image_name, image_versions)| { + let versions: Vec<_> = image_versions + .into_iter() + .map(|(image_version, (_, _))| image_version) + .collect(); + (image_name, versions) + }) + .collect(); + + print_to_stdout(list, arguments.pretty) +} + +fn print_to_stdout(list: BTreeMap>, pretty: Pretty) -> Result<(), Error> { + let stdout = std::io::stdout(); + + match pretty { + Pretty::Always | Pretty::Auto if stdout.is_terminal() => { + serde_json::to_writer_pretty(stdout, &list) + } + _ => serde_json::to_writer(stdout, &list), + } + .context(SerializeListSnafu) +} diff --git a/rust/boil/src/show/mod.rs b/rust/boil/src/show/mod.rs new file mode 100644 index 000000000..a6ad01b43 --- /dev/null +++ b/rust/boil/src/show/mod.rs @@ -0,0 +1,16 @@ +use clap::{Args, Subcommand}; + +use crate::show::images::cli::ShowImagesArguments; + +pub mod images; + +#[derive(Debug, Args)] +pub struct ShowArguments { + #[command(subcommand)] + pub commands: ShowCommand, +} + +#[derive(Debug, Subcommand)] +pub enum ShowCommand { + Images(ShowImagesArguments), +} diff --git a/rust/boil/src/utils.rs b/rust/boil/src/utils.rs new file mode 100644 index 000000000..f3b4e4ea7 --- /dev/null +++ b/rust/boil/src/utils.rs @@ -0,0 +1,48 @@ +use std::process::Command; + +use semver::Version; +use url::Host; + +use crate::build::platform::Architecture; + +/// Formats and returns the image repository URI, eg. `oci.stackable.tech/sdp/opa`. +pub fn format_image_repository_uri( + image_registry: &Host, + registry_namespace: &str, + image_name: &str, +) -> String { + format!("{image_registry}/{registry_namespace}/{image_name}") +} + +/// Formats and returns the image manifest URI, eg. `oci.stackable.tech/sdp/opa:1.4.2-stackable25.7.0-amd64`. +pub fn format_image_manifest_uri( + image_repository_uri: &str, + image_version: &str, + sdp_image_version: &Version, + architecture: &Architecture, + strip_architecture: bool, +) -> String { + if strip_architecture { + format!("{image_repository_uri}:{image_version}-stackable{sdp_image_version}") + } else { + format!( + "{image_repository_uri}:{image_version}-stackable{sdp_image_version}-{architecture}" + ) + } +} + +pub trait CommandExt { + /// Adds an argument to the command if the `predicate` is `true`. + fn arg_if(&mut self, predicate: bool, arg: S) -> &mut Self + where + S: AsRef; +} + +impl CommandExt for Command { + fn arg_if(&mut self, predicate: bool, arg: S) -> &mut Self + where + S: AsRef, + { + if predicate { self.arg(arg) } else { self } + } +} diff --git a/rust/patchable/Cargo.toml b/rust/patchable/Cargo.toml index cb72ed092..9b8653ee6 100644 --- a/rust/patchable/Cargo.toml +++ b/rust/patchable/Cargo.toml @@ -2,6 +2,10 @@ name = "patchable" version = "0.1.0" edition = "2021" +authors.workspace = true +license.workspace = true +repository.workspace = true +publish = false [dependencies] clap.workspace = true diff --git a/rust/patchable/src/patch.rs b/rust/patchable/src/patch.rs index 00cbd1ae9..5f590dc6e 100644 --- a/rust/patchable/src/patch.rs +++ b/rust/patchable/src/patch.rs @@ -7,15 +7,14 @@ use git2::{Oid, Repository}; use snafu::{OptionExt, ResultExt as _, Snafu}; use tracing_indicatif::suspend_tracing_indicatif; +#[cfg(doc)] +use crate::repo::ensure_worktree_is_at; use crate::{ error::{self, CommitId}, patch_mail::{self, mailinfo, mailsplit}, utils::raw_git_cmd, }; -#[cfg(doc)] -use crate::repo::ensure_worktree_is_at; - #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("failed to open stgit series file {path:?}"))] @@ -61,14 +60,18 @@ pub enum Error { source: git2::Error, parent_commit: error::CommitId, }, - #[snafu(display("failed to apply patch {patch_email_file:?} (from {patch_file:?}) to parent commit {parent_commit}"))] + #[snafu(display( + "failed to apply patch {patch_email_file:?} (from {patch_file:?}) to parent commit {parent_commit}" + ))] ApplyPatch { source: git2::Error, parent_commit: error::CommitId, patch_email_file: PathBuf, patch_file: PathBuf, }, - #[snafu(display("failed to write tree for patch {patch_email_file:?} (from {patch_file:?}) applied to parent commit {parent_commit}"))] + #[snafu(display( + "failed to write tree for patch {patch_email_file:?} (from {patch_file:?}) applied to parent commit {parent_commit}" + ))] WritePatchedTree { source: git2::Error, parent_commit: error::CommitId, @@ -77,7 +80,9 @@ pub enum Error { }, #[snafu(display("failed to read patched tree {tree}"))] ReadPatchedTree { source: git2::Error, tree: Oid }, - #[snafu(display("failed to write commit for patch {patch_email_file:?} (from {patch_file:?}) applied to parent commit {parent_commit}"))] + #[snafu(display( + "failed to write commit for patch {patch_email_file:?} (from {patch_file:?}) applied to parent commit {parent_commit}" + ))] WriteCommit { source: git2::Error, parent_commit: error::CommitId, diff --git a/rust/patchable/src/patch_mail.rs b/rust/patchable/src/patch_mail.rs index 680c32c2b..3a4fa330f 100644 --- a/rust/patchable/src/patch_mail.rs +++ b/rust/patchable/src/patch_mail.rs @@ -9,8 +9,8 @@ use std::{ use git2::{Diff, Repository, Signature}; use snafu::{OptionExt as _, ResultExt, Snafu}; -use tempfile::{tempdir, NamedTempFile}; -use time::{format_description::well_known::Rfc2822, OffsetDateTime}; +use tempfile::{NamedTempFile, tempdir}; +use time::{OffsetDateTime, format_description::well_known::Rfc2822}; use tracing_indicatif::suspend_tracing_indicatif; use crate::utils::raw_git_cmd; diff --git a/rust/patchable/src/repo.rs b/rust/patchable/src/repo.rs index 26e31cdac..0b0335b6a 100644 --- a/rust/patchable/src/repo.rs +++ b/rust/patchable/src/repo.rs @@ -28,9 +28,7 @@ pub enum Error { path: PathBuf, }, - #[snafu(display( - "failed to create worktree branch {branch:?} pointing at {commit} in {repo}" - ))] + #[snafu(display("failed to create worktree branch {branch:?} pointing at {commit} in {repo}"))] CreateWorktreeBranch { source: git2::Error, repo: error::RepoPath, diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 000000000..07217b21f --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,9 @@ +# This file includes unstable features, so you need to run "cargo +nightly fmt" to format your code. +# It's also ok to use the stable toolchain by simple running "cargo fmt", but using the nigthly formatter is prefered. + +# https://doc.rust-lang.org/nightly/edition-guide/rust-2024/rustfmt-style-edition.html +style_edition = "2024" +imports_granularity = "Crate" +group_imports = "StdExternalCrate" +reorder_impl_items = true +use_field_init_shorthand = true diff --git a/shared/statsd-exporter/Dockerfile b/shared/statsd-exporter/Dockerfile index d5b2b4cac..3840fc0f9 100644 --- a/shared/statsd-exporter/Dockerfile +++ b/shared/statsd-exporter/Dockerfile @@ -1,8 +1,8 @@ # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 # check=error=true -FROM stackable/image/stackable-base -ARG PRODUCT +FROM local-image/stackable-base +ARG PRODUCT_VERSION ARG STACKABLE_USER_UID WORKDIR /statsd_exporter @@ -26,18 +26,18 @@ export GOPATH=/go_cache # We use version 1.7.0, since a newer version of cyclonedx-gomod is not compatible with the version of Golang (>= 1.23.1) go install github.com/CycloneDX/cyclonedx-gomod/cmd/cyclonedx-gomod@v1.7.0 -curl "https://repo.stackable.tech/repository/packages/statsd_exporter/statsd_exporter-${PRODUCT}.src.tar.gz" | tar -xzC . +curl "https://repo.stackable.tech/repository/packages/statsd_exporter/statsd_exporter-${PRODUCT_VERSION}.src.tar.gz" | tar -xzC . ( - cd "statsd_exporter-${PRODUCT}" || exit + cd "statsd_exporter-${PRODUCT_VERSION}" || exit # Unfortunately, we need to create a dummy Git repository to allow cyclonedx-gomod to determine the version of statsd_exporter git init git config user.email "fake.commiter@stackable.tech" git config user.name "Fake commiter" git commit --allow-empty --message "Fake commit, so that we can create a tag" - git tag "v${PRODUCT}" + git tag "v${PRODUCT_VERSION}" go build -o ../statsd_exporter - $GOPATH/bin/cyclonedx-gomod app -json -output-version 1.5 -output ../statsd_exporter-${PRODUCT}.cdx.json -packages -files + $GOPATH/bin/cyclonedx-gomod app -json -output-version 1.5 -output ../statsd_exporter-${PRODUCT_VERSION}.cdx.json -packages -files ) -rm -rf "statsd_exporter-${PRODUCT}" +rm -rf "statsd_exporter-${PRODUCT_VERSION}" EOF diff --git a/shared/statsd-exporter/boil-config.toml b/shared/statsd-exporter/boil-config.toml new file mode 100644 index 000000000..c2f9afcb2 --- /dev/null +++ b/shared/statsd-exporter/boil-config.toml @@ -0,0 +1,2 @@ +[versions."0.28.0".local-images] +stackable-base = "1.0.0" diff --git a/shared/statsd-exporter/versions.py b/shared/statsd-exporter/versions.py deleted file mode 100644 index adb26f18a..000000000 --- a/shared/statsd-exporter/versions.py +++ /dev/null @@ -1,6 +0,0 @@ -versions = [ - { - "product": "0.28.0", - "stackable-base": "1.0.0", - } -] diff --git a/spark-connect-client/Dockerfile b/spark-connect-client/Dockerfile index 4750a071e..fd2c9e950 100644 --- a/spark-connect-client/Dockerfile +++ b/spark-connect-client/Dockerfile @@ -1,18 +1,18 @@ # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 # spark-builder: provides client libs for spark-connect -FROM stackable/image/spark-k8s AS spark-builder +FROM local-image/spark-k8s AS spark-builder -ARG PRODUCT -ARG PYTHON -ARG RELEASE +ARG PRODUCT_VERSION +ARG PYTHON_VERSION +ARG RELEASE_VERSION ARG STACKABLE_USER_UID LABEL name="Stackable Spark Connect Examples" \ maintainer="info@stackable.tech" \ vendor="Stackable GmbH" \ - version="${PRODUCT}" \ - release="${RELEASE}" \ + version="${PRODUCT_VERSION}" \ + release="${RELEASE_VERSION}" \ summary="Spark Connect Examples" \ description="Spark Connect client libraries for Python and the JVM, including some examples." @@ -26,7 +26,7 @@ RUN < /stackable/package_manifest.txt +rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt chmod g=u /stackable/package_manifest.txt rm -rf /var/cache/yum -# Add link pointing from /stackable/zookeeper to /stackable/apache-zookeeper-${PRODUCT}-stackable${RELEASE}-bin/ +# Add link pointing from /stackable/zookeeper to /stackable/apache-zookeeper-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-bin/ # to preserve the folder name with the version. -ln -s /stackable/apache-zookeeper-${PRODUCT}-stackable${RELEASE}-bin/ /stackable/zookeeper +ln -s /stackable/apache-zookeeper-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-bin/ /stackable/zookeeper chown -h ${STACKABLE_USER_UID}:0 /stackable/zookeeper # fix missing permissions chmod g=u /stackable/jmx -chmod g=u /stackable/apache-zookeeper-${PRODUCT}-stackable${RELEASE}-bin/ +chmod g=u /stackable/apache-zookeeper-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-bin/ EOF # ---------------------------------------- diff --git a/zookeeper/boil-config.toml b/zookeeper/boil-config.toml new file mode 100644 index 000000000..64ab0416d --- /dev/null +++ b/zookeeper/boil-config.toml @@ -0,0 +1,6 @@ +[versions."3.9.3".local-images] +java-base = "17" +java-devel = "11" + +[versions."3.9.3".build-arguments] +jmx-exporter-version = "1.3.0" diff --git a/zookeeper/versions.py b/zookeeper/versions.py deleted file mode 100644 index d63bfac5a..000000000 --- a/zookeeper/versions.py +++ /dev/null @@ -1,12 +0,0 @@ -versions = [ - { - "product": "3.9.3", - "java-base": "17", - # NOTE (@NickLarsenNZ): Builds fail on Java 17, with the output: - # [ERROR] Failed to execute goal com.github.spotbugs:spotbugs-maven-plugin:4.0.0:spotbugs (spotbugs) on project - # zookeeper: Execution spotbugs of goal com.github.spotbugs:spotbugs-maven-plugin:4.0.0:spotbugs failed: Java - # returned: 1 -> [Help 1] - "java-devel": "11", - "jmx_exporter": "1.3.0", - }, -]