diff --git a/CHANGELOG.md b/CHANGELOG.md index 3aff72840..41fe23238 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ All notable changes to this project will be documented in this file. This uses a git mirror and patchable instead of sourcing from Nexus. - superset: Bump statsd_exporter to `0.30.0` ([#1524]). - hadoop: Refactor cloud library provisioning to downstream images. The jars are now conveniently placed in `/stackable/hadoop-cloud-libraries/` to be easily picked up ([#1511]). +- spark: bump hadoop `3.4.2` to `3.4.3` ([#1533]) ### Fixed @@ -68,8 +69,9 @@ All notable changes to this project will be documented in this file. [#1519]: https://github.com/stackabletech/docker-images/pull/1519 [#1520]: https://github.com/stackabletech/docker-images/pull/1520 [#1521]: https://github.com/stackabletech/docker-images/pull/1521 -[#1525]: https://github.com/stackabletech/docker-images/pull/1525 [#1524]: https://github.com/stackabletech/docker-images/pull/1524 +[#1525]: https://github.com/stackabletech/docker-images/pull/1525 +[#1533]: https://github.com/stackabletech/docker-images/pull/1533 ## [26.3.0] - 2026-03-16 diff --git a/hadoop/hadoop/Dockerfile b/hadoop/hadoop/Dockerfile index fe6429eee..b8387410e 100644 --- a/hadoop/hadoop/Dockerfile +++ b/hadoop/hadoop/Dockerfile @@ -10,6 +10,7 @@ ARG PROTOBUF_VERSION ARG AWS_JAVA_SDK_BUNDLE_VERSION ARG AZURE_STORAGE_VERSION ARG AZURE_KEYVAULT_CORE_VERSION +ARG ANALYTICSACCELERATOR_S3_VERSION ARG STACKABLE_USER_UID WORKDIR /stackable @@ -166,6 +167,8 @@ else cp /stackable/hadoop/share/hadoop/common/lib/hadoop-aws-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/ # Starting with Hadoop 3.4.3 the aws-java-sdk-bundle is not included any more. curl -o /stackable/hadoop-cloud-libraries/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar --fail "https://repo.stackable.tech/repository/packages/aws/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar" + # Starting with Hadoop 3.4.3 the S3A InputStreamType factory references analyticsaccelerator-s3 (software.amazon.s3.analyticsaccelerator.request.ObjectClient) during S3AFileSystem.initialize. + curl -o /stackable/hadoop-cloud-libraries/analyticsaccelerator-s3-${ANALYTICSACCELERATOR_S3_VERSION}.jar --fail "https://repo.stackable.tech/repository/packages/aws/analyticsaccelerator-s3-${ANALYTICSACCELERATOR_S3_VERSION}.jar" cp /stackable/hadoop/share/hadoop/common/lib/hadoop-azure-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/ cp /stackable/hadoop/share/hadoop/common/lib/azure-storage-${AZURE_STORAGE_VERSION}.jar /stackable/hadoop-cloud-libraries/ cp /stackable/hadoop/share/hadoop/common/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE_VERSION}.jar /stackable/hadoop-cloud-libraries/ diff --git a/hadoop/hadoop/boil-config.toml b/hadoop/hadoop/boil-config.toml index b6d0ee3b9..9b2cdea1c 100644 --- a/hadoop/hadoop/boil-config.toml +++ b/hadoop/hadoop/boil-config.toml @@ -30,6 +30,7 @@ protobuf-version = "3.21.12" # https://github.com/apache/hadoop/blob/rel/release aws-java-sdk-bundle-version = "2.35.4" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.3 azure-storage-version = "7.0.1" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.3 azure-keyvault-core-version = "1.0.0" # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 +analyticsaccelerator-s3-version = "1.3.1" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.3 @@ -42,3 +43,4 @@ protobuf-version = "3.25.5" # https://github.com/apache/hadoop/blob/rel/release- aws-java-sdk-bundle-version = "2.35.4" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.5.0 azure-storage-version = "7.0.1" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.5.0 azure-keyvault-core-version = "1.0.0" # Needs to match https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 +analyticsaccelerator-s3-version = "1.3.1" # Needs to match https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.3 diff --git a/spark-k8s/boil-config.toml b/spark-k8s/boil-config.toml index 712c5a28f..5aa25fff4 100644 --- a/spark-k8s/boil-config.toml +++ b/spark-k8s/boil-config.toml @@ -6,7 +6,7 @@ containerfile = "Dockerfile.3" [versions."3.5.8".local-images] -"hadoop/hadoop" = "3.4.2" # TODO: Bump to 3.4.3 +"hadoop/hadoop" = "3.4.3" java-base = "17" java-devel = "17" hbase = "2.6.3" @@ -22,19 +22,20 @@ tini-version = "0.19.0" hbase-connector-version = "1.0.1_3.5.8" + # LTS since 26.7.0 [versions."4.1.1"] containerfile = "Dockerfile.4" [versions."4.1.1".local-images] -"hadoop/hadoop" = "3.4.2" # TODO: Bump to 3.4.3 +"hadoop/hadoop" = "3.4.3" java-base = "21" java-devel = "21" [versions."4.1.1".build-arguments] python-version = "3.12" -jackson-dataformat-xml-version = "2.15.2" # Needs to match https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.12/3.5.7 -stax2-api-version = "4.2.1" # Needs to match https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 -woodstox-core-version = "6.5.1" # Needs to match https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 +jackson-dataformat-xml-version = "2.20.0" # Needs to match https://mvnrepository.com/artifact/org.apache.spark/spark-parent_2.13/4.1.1 +stax2-api-version = "4.2.2" # Needs to match https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.20.0 +woodstox-core-version = "7.1.1" # Needs to match https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.20.0 jmx-exporter-version = "1.3.0" tini-version = "0.19.0"