From e9e296865292272fad2e332ec070e26611055a5d Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Wed, 26 Nov 2025 11:32:10 -0500 Subject: [PATCH 1/6] Test docker config to fix workflow. --- .../beam_PostCommit_XVR_GoUsingJava_Dataflow.json | 4 ++-- sdks/java/container/common.gradle | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/trigger_files/beam_PostCommit_XVR_GoUsingJava_Dataflow.json b/.github/trigger_files/beam_PostCommit_XVR_GoUsingJava_Dataflow.json index 920c8d132e4a..b26833333238 100644 --- a/.github/trigger_files/beam_PostCommit_XVR_GoUsingJava_Dataflow.json +++ b/.github/trigger_files/beam_PostCommit_XVR_GoUsingJava_Dataflow.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run", - "modification": 1 -} \ No newline at end of file + "modification": 2 +} diff --git a/sdks/java/container/common.gradle b/sdks/java/container/common.gradle index c81a33827bef..db7ae7ab0c7f 100644 --- a/sdks/java/container/common.gradle +++ b/sdks/java/container/common.gradle @@ -113,9 +113,11 @@ docker { 'base_image': javaBaseImage, 'java_version': imageJavaVersion, ]) - buildx project.useBuildx() + // buildx project.useBuildx() + buildx true platform(*project.containerPlatforms()) - load project.useBuildx() && !pushContainers + // load project.useBuildx() && !pushContainers + load true push pushContainers } From 6d876402b3680f898c4891f22b94e8fd39a548b2 Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Wed, 26 Nov 2025 12:12:55 -0500 Subject: [PATCH 2/6] enable buildx and load in python docker. --- sdks/python/container/common.gradle | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle index ad64dbbb660b..1a999b4c9191 100644 --- a/sdks/python/container/common.gradle +++ b/sdks/python/container/common.gradle @@ -122,9 +122,11 @@ docker { buildArgs(['py_version': "${project.ext.pythonVersion}", 'pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) || project.rootProject.hasProperty(["isRelease"])]) - buildx project.useBuildx() + // buildx project.useBuildx() + buildx true platform(*project.containerPlatforms()) - load project.useBuildx() && !pushContainers + // load project.useBuildx() && !pushContainers + load true push pushContainers } From 7e64ce8be08a2c028d266e25f0755de978b79933 Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Wed, 26 Nov 2025 13:46:10 -0500 Subject: [PATCH 3/6] Another attempt by setting buildx project property. --- runners/google-cloud-dataflow-java/build.gradle | 1 + sdks/java/container/common.gradle | 6 ++---- sdks/python/container/common.gradle | 6 ++---- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index 9f064f2432bc..1543b5d96388 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -55,6 +55,7 @@ ext.dataflowFnapiEnvironmentMajorVersion = '8' ext.dataflowLegacyContainerVersion = 'beam-master-20251107' ext.dataflowFnapiContainerVersion = 'beam-master-20251107' ext.dataflowContainerBaseRepository = 'gcr.io/cloud-dataflow/v1beta3' +ext.useDockerBuildx = true processResources { filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [ diff --git a/sdks/java/container/common.gradle b/sdks/java/container/common.gradle index db7ae7ab0c7f..c81a33827bef 100644 --- a/sdks/java/container/common.gradle +++ b/sdks/java/container/common.gradle @@ -113,11 +113,9 @@ docker { 'base_image': javaBaseImage, 'java_version': imageJavaVersion, ]) - // buildx project.useBuildx() - buildx true + buildx project.useBuildx() platform(*project.containerPlatforms()) - // load project.useBuildx() && !pushContainers - load true + load project.useBuildx() && !pushContainers push pushContainers } diff --git a/sdks/python/container/common.gradle b/sdks/python/container/common.gradle index 1a999b4c9191..ad64dbbb660b 100644 --- a/sdks/python/container/common.gradle +++ b/sdks/python/container/common.gradle @@ -122,11 +122,9 @@ docker { buildArgs(['py_version': "${project.ext.pythonVersion}", 'pull_licenses': project.rootProject.hasProperty(["docker-pull-licenses"]) || project.rootProject.hasProperty(["isRelease"])]) - // buildx project.useBuildx() - buildx true + buildx project.useBuildx() platform(*project.containerPlatforms()) - // load project.useBuildx() && !pushContainers - load true + load project.useBuildx() && !pushContainers push pushContainers } From 96087735c11df86169667cbc7ea46d65870a03d6 Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Wed, 26 Nov 2025 14:16:00 -0500 Subject: [PATCH 4/6] Another attempt --- gradle.properties | 2 ++ runners/google-cloud-dataflow-java/build.gradle | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index f2ea56da8b19..e002163b9f2a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -42,3 +42,5 @@ docker_image_default_repo_prefix=beam_ flink_versions=1.17,1.18,1.19,1.20 # supported python versions python_versions=3.10,3.11,3.12,3.13 + +useDockerBuildx = true diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index 1543b5d96388..9f064f2432bc 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -55,7 +55,6 @@ ext.dataflowFnapiEnvironmentMajorVersion = '8' ext.dataflowLegacyContainerVersion = 'beam-master-20251107' ext.dataflowFnapiContainerVersion = 'beam-master-20251107' ext.dataflowContainerBaseRepository = 'gcr.io/cloud-dataflow/v1beta3' -ext.useDockerBuildx = true processResources { filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [ From 3e7fcf0277d16aef917d2cad519dc305c43e26c2 Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Wed, 26 Nov 2025 18:30:36 -0500 Subject: [PATCH 5/6] Re-enable workflow and sickbay some failed tests for now. --- .../workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml | 6 +++++- gradle.properties | 2 -- sdks/go/test/integration/integration.go | 3 +++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml index cc52476ca303..3ccaaf0e677c 100644 --- a/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml @@ -16,9 +16,11 @@ # TODO(https://github.com/apache/beam/issues/32492): re-enable the suite # on cron and add release/trigger_all_tests.json to trigger path once fixed. -name: PostCommit XVR GoUsingJava Dataflow (disabled) +name: PostCommit XVR GoUsingJava Dataflow on: + schedule: + - cron: '45 5/6 * * *' pull_request_target: paths: ['.github/trigger_files/beam_PostCommit_XVR_GoUsingJava_Dataflow.json'] workflow_dispatch: @@ -83,6 +85,8 @@ jobs: uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerGoUsingJava + arguments: | + -PuseDockerBuildx - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/gradle.properties b/gradle.properties index e002163b9f2a..f2ea56da8b19 100644 --- a/gradle.properties +++ b/gradle.properties @@ -42,5 +42,3 @@ docker_image_default_repo_prefix=beam_ flink_versions=1.17,1.18,1.19,1.20 # supported python versions python_versions=3.10,3.11,3.12,3.13 - -useDockerBuildx = true diff --git a/sdks/go/test/integration/integration.go b/sdks/go/test/integration/integration.go index b6d2c60e0fb9..ea23c5f9ae0e 100644 --- a/sdks/go/test/integration/integration.go +++ b/sdks/go/test/integration/integration.go @@ -301,6 +301,9 @@ var dataflowFilters = []string{ // There is no infrastructure for running KafkaIO tests with Dataflow. "TestKafkaIO.*", "TestSpannerIO.*", + // TODO(36918) These tests are currently failing in Dataflow Runner + "TestBigQueryIO.*", + "TestBigtableIO.*", // Dataflow doesn't support any test that requires loopback. // Eg. For FileIO examples. ".*Loopback.*", From 7696b35f81e32c37f321954b7c88bdccdddbcb24 Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Wed, 26 Nov 2025 18:40:47 -0500 Subject: [PATCH 6/6] Add release trigger --- .github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml index 3ccaaf0e677c..76aebfccb68a 100644 --- a/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml @@ -22,7 +22,7 @@ on: schedule: - cron: '45 5/6 * * *' pull_request_target: - paths: ['.github/trigger_files/beam_PostCommit_XVR_GoUsingJava_Dataflow.json'] + paths: ['.github/trigger_files/beam_PostCommit_XVR_GoUsingJava_Dataflow.json', 'release/trigger_all_tests.json'] workflow_dispatch: #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event