From d40da985458c67e64921fde1bd5e33612a8bbbcc Mon Sep 17 00:00:00 2001 From: Vitaly Terentyev Date: Mon, 1 Dec 2025 19:03:25 +0400 Subject: [PATCH 1/4] Remove python 3.9 --- .github/workflows/build_release_candidate.yml | 1 - .../workflows/republish_released_docker_containers.yml | 2 +- release/src/main/scripts/run_rc_validation.sh | 8 ++++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml index 88ad95701b6f..88f7e2207880 100644 --- a/.github/workflows/build_release_candidate.yml +++ b/.github/workflows/build_release_candidate.yml @@ -260,7 +260,6 @@ jobs: # tasks as they are added. images_to_publish: [ {"gradle_task": ":pushAllRunnersDockerImages", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"}, - {"gradle_task": ":sdks:python:container:push39", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"}, {"gradle_task": ":sdks:python:container:push310", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"}, {"gradle_task": ":sdks:python:container:push311", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"}, {"gradle_task": ":sdks:python:container:push312", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"}, diff --git a/.github/workflows/republish_released_docker_containers.yml b/.github/workflows/republish_released_docker_containers.yml index 96feb6cd128e..a9d043bec754 100644 --- a/.github/workflows/republish_released_docker_containers.yml +++ b/.github/workflows/republish_released_docker_containers.yml @@ -47,7 +47,6 @@ jobs: # tasks as they are added. images_to_publish: [ {"gradle_task": ":pushAllRunnersDockerImages", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"}, - {"gradle_task": ":sdks:python:container:push39", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"}, {"gradle_task": ":sdks:python:container:push310", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"}, {"gradle_task": ":sdks:python:container:push311", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"}, {"gradle_task": ":sdks:python:container:push312", "include_skip_flags": "-Pinclude-ml -Pinclude-distroless"}, @@ -93,6 +92,7 @@ jobs: run: | ./gradlew ${{ matrix.images_to_publish.gradle_task }} \ -PisRelease \ + -PpythonVersion=3.10 \ -Pdocker-pull-licenses \ -Pprune-images ${{ matrix.images_to_publish.include_skip_flags }} \ -Pdocker-repository-root=gcr.io/apache-beam-testing/updated_released_container_images \ diff --git a/release/src/main/scripts/run_rc_validation.sh b/release/src/main/scripts/run_rc_validation.sh index 9c93ed4ef4d4..62f79c7ee626 100755 --- a/release/src/main/scripts/run_rc_validation.sh +++ b/release/src/main/scripts/run_rc_validation.sh @@ -99,7 +99,7 @@ HUB_VERSION=2.12.0 HUB_ARTIFACTS_NAME=hub-linux-amd64-${HUB_VERSION} BACKUP_BASHRC=.bashrc_backup_$(date +"%Y%m%d%H%M%S") BACKUP_M2=settings_backup_$(date +"%Y%m%d%H%M%S").xml -declare -a PYTHON_VERSIONS_TO_VALIDATE=("python3.9") +declare -a PYTHON_VERSIONS_TO_VALIDATE=("python3.10") echo "" echo "====================Checking Environment & Variables=================" echo "PLEASE update RC_VALIDATE_CONFIGS in file script.config first." @@ -604,7 +604,7 @@ if [[ ("$python_xlang_quickstart" = true) \ PYTHON_MULTILANG_QUICKSTART_OUTPUT_FILE_NAME=${PYTHON_MULTILANG_QUICKSTART_FILE_PREFIX}_output PYTHON_MULTILANG_QUICKSTART_EXPECTED_OUTPUT_FILE_NAME=${PYTHON_MULTILANG_QUICKSTART_FILE_PREFIX}_expected_output PYTHON_MULTILANG_QUICKSTART_SORTED_OUTPUT_FILE_NAME=${PYTHON_MULTILANG_QUICKSTART_FILE_PREFIX}_sorted_output - + # Cleaning up data from any previous runs. rm ${PYTHON_MULTILANG_QUICKSTART_FILE_PREFIX}* rm ./beam-examples-multi-language-${RELEASE_VER}.jar @@ -624,7 +624,7 @@ if [[ ("$python_xlang_quickstart" = true) \ # Downloading the expansion service jar. wget ${REPO_URL}/org/apache/beam/beam-examples-multi-language/${RELEASE_VER}/beam-examples-multi-language-${RELEASE_VER}.jar JAVA_EXPANSION_SERVICE_PORT=33333 - + # Starting up the expansion service in a seperate shell. echo "A new terminal will pop up and start a java expansion service." gnome-terminal -x sh -c \ @@ -746,7 +746,7 @@ if [[ ("$java_xlang_quickstart" = true) \ --expansionService=localhost:${PYTHON_EXPANSION_SERVICE_PORT} \ --output=${JAVA_MULTILANG_QUICKSTART_OUTPUT_FILE_NAME}" - # We cannot validate local output since + # We cannot validate local output since # TODO: Write output to GCS and validate when Python portable runner can forward credentials to GCS appropriately. java_xlang_quickstart_status=$? From d137814862a2b09f81baa37b72518aafd248374b Mon Sep 17 00:00:00 2001 From: Vitaly Terentyev Date: Tue, 2 Dec 2025 00:22:20 +0400 Subject: [PATCH 2/4] Add nvidia license --- .../license_scripts/dep_urls_py.yaml | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/sdks/python/container/license_scripts/dep_urls_py.yaml b/sdks/python/container/license_scripts/dep_urls_py.yaml index 1a47675b4d93..bc6c0745494f 100644 --- a/sdks/python/container/license_scripts/dep_urls_py.yaml +++ b/sdks/python/container/license_scripts/dep_urls_py.yaml @@ -108,6 +108,32 @@ pip_dependencies: license: "https://raw.githubusercontent.com/atdt/monotonic/master/LICENSE" nose: license: "https://raw.githubusercontent.com/nose-devs/nose/master/lgpl.txt" + nvidia-cublas-cu12: + license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" + nvidia-cuda-cupti-cu12: + license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" + nvidia-cuda-nvrtc-cu12: + license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" + nvidia-cuda-runtime-cu12: + license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" + nvidia-cudnn-cu12: + license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" + nvidia-cufft-cu12: + license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" + nvidia-cufile-cu12: + license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" + nvidia-curand-cu12: + license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" + nvidia-cusolver-cu12: + license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" + nvidia-cusparse-cu12: + license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" + nvidia-nccl-cu12: + license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" + nvidia-nvjitlink-cu12: + license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" + nvidia-nvtx-cu12: + license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" numpy: license: "https://raw.githubusercontent.com/numpy/numpy/master/LICENSE.txt" oauth2client: From 065a3aea4281d1a64283be188dcedfbcd263ba95 Mon Sep 17 00:00:00 2001 From: Vitaly Terentyev Date: Tue, 2 Dec 2025 09:21:02 +0400 Subject: [PATCH 3/4] Upgrade pip-licenses --- sdks/python/container/Dockerfile | 2 +- sdks/python/expansion-service-container/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/container/Dockerfile b/sdks/python/container/Dockerfile index 56df970f75c2..9aa9f0518dd9 100644 --- a/sdks/python/container/Dockerfile +++ b/sdks/python/container/Dockerfile @@ -108,7 +108,7 @@ COPY target/go-licenses/* /opt/apache/beam/third_party_licenses/golang/ COPY target/license_scripts /tmp/license_scripts/ RUN if [ "$pull_licenses" = "true" ] ; then \ - pip install 'pip-licenses<5' pyyaml tenacity && \ + pip install 'pip-licenses<6' pyyaml tenacity && \ python /tmp/license_scripts/pull_licenses_py.py ; \ fi diff --git a/sdks/python/expansion-service-container/Dockerfile b/sdks/python/expansion-service-container/Dockerfile index 1f325bb81030..770ac7f9fbef 100644 --- a/sdks/python/expansion-service-container/Dockerfile +++ b/sdks/python/expansion-service-container/Dockerfile @@ -62,7 +62,7 @@ COPY target/go-licenses/* /opt/apache/beam/third_party_licenses/golang/ COPY target/license_scripts /tmp/license_scripts/ RUN if [ "$pull_licenses" = "true" ] ; then \ - pip install 'pip-licenses<5' pyyaml tenacity && \ + pip install 'pip-licenses<6' pyyaml tenacity && \ python /tmp/license_scripts/pull_licenses_py.py ; \ fi From bfea1e36b30063d54710c7a8b5c5d44820b63199 Mon Sep 17 00:00:00 2001 From: Vitaly Terentyev Date: Wed, 3 Dec 2025 00:18:30 +0400 Subject: [PATCH 4/4] Remove nvidia requirements --- .../license_scripts/dep_urls_py.yaml | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/sdks/python/container/license_scripts/dep_urls_py.yaml b/sdks/python/container/license_scripts/dep_urls_py.yaml index bc6c0745494f..1a47675b4d93 100644 --- a/sdks/python/container/license_scripts/dep_urls_py.yaml +++ b/sdks/python/container/license_scripts/dep_urls_py.yaml @@ -108,32 +108,6 @@ pip_dependencies: license: "https://raw.githubusercontent.com/atdt/monotonic/master/LICENSE" nose: license: "https://raw.githubusercontent.com/nose-devs/nose/master/lgpl.txt" - nvidia-cublas-cu12: - license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" - nvidia-cuda-cupti-cu12: - license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" - nvidia-cuda-nvrtc-cu12: - license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" - nvidia-cuda-runtime-cu12: - license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" - nvidia-cudnn-cu12: - license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" - nvidia-cufft-cu12: - license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" - nvidia-cufile-cu12: - license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" - nvidia-curand-cu12: - license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" - nvidia-cusolver-cu12: - license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" - nvidia-cusparse-cu12: - license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" - nvidia-nccl-cu12: - license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" - nvidia-nvjitlink-cu12: - license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" - nvidia-nvtx-cu12: - license: "https://raw.githubusercontent.com/NVIDIA/cccl/refs/heads/main/LICENSE" numpy: license: "https://raw.githubusercontent.com/numpy/numpy/master/LICENSE.txt" oauth2client: