diff --git a/.github/workflows/beam_PreCommit_Website_Stage_GCS.yml b/.github/workflows/beam_PreCommit_Website_Stage_GCS.yml index 6d1d34644696..203d27a6c22a 100644 --- a/.github/workflows/beam_PreCommit_Website_Stage_GCS.yml +++ b/.github/workflows/beam_PreCommit_Website_Stage_GCS.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: PreCommit Website Stage GCS +name: PreCommit Website Stage GCS on: push: @@ -38,7 +38,7 @@ env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} - gcsbucket: apache-beam-website-pull-requests + gcsbucket: apache-beam-website-pull-requests ghprbPullId: #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event @@ -87,11 +87,17 @@ jobs: with: python-version: default java-version: default + - name: Authenticate on GCP + id: auth-gcp + uses: google-github-actions/auth@v3 + with: + service_account: ${{ secrets.GCP_SA_EMAIL }} + credentials_json: ${{ secrets.GCP_SA_KEY }} - name: Run website_stageWebsite script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :website:stageWebsite - arguments: -PwebsiteBucket=$gcsbucket + arguments: -PwebsiteBucket=$gcsbucket -PgcpCredsFile="${{ steps.auth-gcp.outputs.credentials_file_path }}" - name: Add website link to the summary if pr if: env.ghprbPullId != null run: echo "Website published to http://$gcsbucket.storage.googleapis.com/$ghprbPullId/index.html" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/beam_Publish_Website.yml b/.github/workflows/beam_Publish_Website.yml index f25cf20d16e0..c20313b7b040 100644 --- a/.github/workflows/beam_Publish_Website.yml +++ b/.github/workflows/beam_Publish_Website.yml @@ -58,9 +58,15 @@ jobs: uses: ./.github/actions/setup-environment-action with: disable-cache: true + - name: Authenticate on GCP + id: auth-gcp + uses: google-github-actions/auth@v3 + with: + service_account: ${{ secrets.GCP_SA_EMAIL }} + credentials_json: ${{ secrets.GCP_SA_KEY }} - name: run PostCommit Website Publish script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :website:clean :website:publishWebsite - arguments: -PgitPublishRemote="https://github.com/apache/beam.git" + arguments: -PgitPublishRemote="https://github.com/apache/beam.git" -PgcpCredsFile="${{ steps.auth-gcp.outputs.credentials_file_path }}" - uses: actions/checkout@v4 # Extra checkout to make sure we're on master for post steps. diff --git a/website/Dockerfile b/website/Dockerfile index e09d06f22ead..1d18ee762318 100644 --- a/website/Dockerfile +++ b/website/Dockerfile @@ -44,6 +44,23 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +# Install Google Cloud SDK (gsutil) +RUN apt-get update \ + && apt-get install -y apt-transport-https \ + # Download the key and save it directly to the keyring file (skipping apt-key) + && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg \ + | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg \ + # Add the repo source using the signed-by tag + && echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \ + | tee /etc/apt/sources.list.d/google-cloud-sdk.list \ + && apt-get update \ + && apt-get install -y google-cloud-cli \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +ENV CLOUDSDK_CONFIG=/opt/gcloud-config +RUN mkdir -p /opt/gcloud-config && chmod 777 /opt/gcloud-config + # Install node LTS environment RUN curl -sL https://deb.nodesource.com/setup_lts.x | bash - \ && apt-get update \ diff --git a/website/build.gradle b/website/build.gradle index 6aa88a2c7feb..790afe582eb5 100644 --- a/website/build.gradle +++ b/website/build.gradle @@ -128,6 +128,56 @@ task setupDockerContainer(type: Exec) { """echo '[params]\n branch_repo = "${getBranchRepo()}"' > /tmp/_config_branch_repo.toml""" } +def performanceLooksOutDir = "$dockerSourceDir/site/static/images/performance/looks" +def gcpCredsInContainer = "/tmp/gcp-key.json" + +task copyGcpCredsToContainer(type: Exec) { + dependsOn setupDockerContainer + + def gcpCredsHost = providers.gradleProperty("gcpCredsFile").orNull + if (gcpCredsHost) { + commandLine 'docker', 'cp', gcpCredsHost, "${->startDockerContainer.containerId()}:${gcpCredsInContainer}" + } else { + println "[performance_looks] -PgcpCredsFile not provided, skipping copy" + } +} + +task downloadPerformanceLooks(type: Exec) { + dependsOn setupDockerContainer, copyGcpCredsToContainer + + def gcpCredsHost = providers.gradleProperty("gcpCredsFile").orNull + if (gcpCredsHost) { + commandLine 'docker', 'exec', + "${-> setupDockerContainer.containerId()}", '/bin/bash', '-lc', + """ + set -u + + SRC="gs://public_looker_explores_us_a3853f40" + OUT_DIR="${performanceLooksOutDir}" + + export HOME=/tmp + export CLOUDSDK_CONFIG=/tmp/gcloud + export GSUTIL_CONFIG_DIR=/tmp/gsutil + export GOOGLE_APPLICATION_CREDENTIALS="${gcpCredsInContainer}" + + gcloud auth activate-service-account --key-file="\$GOOGLE_APPLICATION_CREDENTIALS" --no-user-output-enabled + + mkdir -p "\$OUT_DIR" + + echo "[performance_looks] Syncing looks..." + + if gsutil -m rsync -r "\$SRC" "\$OUT_DIR"; then + echo "[performance_looks] Download completed successfully" + else + echo "[performance_looks][WARNING] Failed to sync looks from GCS" + echo "[performance_looks][WARNING] Website build will continue without updated images" + fi + """ + } else { + println "[performance_looks] Skipping download of performance looks" + } +} + task stopAndRemoveDockerContainer(type: Exec) { commandLine 'docker', 'rm', '-f', "${->createDockerContainer.containerId()}" } @@ -147,6 +197,7 @@ def createBuildTask = { BuildTaskConfiguration config = it as BuildTaskConfiguration task "build${config.name}Website" (type:Exec) { dependsOn setupDockerContainer + dependsOn downloadPerformanceLooks finalizedBy stopAndRemoveDockerContainer def configs = "$dockerSourceDir/site/config.toml" diff --git a/website/www/site/data/performance.yaml b/website/www/site/data/performance.yaml index 17a6612160c6..c388b501ea4b 100644 --- a/website/www/site/data/performance.yaml +++ b/website/www/site/data/performance.yaml @@ -12,6 +12,7 @@ host: https://storage.googleapis.com path: public_looker_explores_us_a3853f40 +static_base: /images/performance/looks looks: bigquery: read: diff --git a/website/www/site/layouts/shortcodes/performance_looks.html b/website/www/site/layouts/shortcodes/performance_looks.html index 1a4f1d0a578f..fc807fecf7f1 100644 --- a/website/www/site/layouts/shortcodes/performance_looks.html +++ b/website/www/site/layouts/shortcodes/performance_looks.html @@ -11,20 +11,21 @@ */}}
- {{ $host := .Site.Data.performance.host }} - {{ $path := .Site.Data.performance.path }} + {{ $staticBase := .Site.Data.performance.static_base | default "/images/performance/looks" }} {{ $looks := .Site.Data.performance.looks }} {{ $io := index $looks (.Get "io") }} {{ $rw := index $io (.Get "read_or_write") }} {{ $section := index $rw (.Get "section") }} - {{ range $section }} {{ $folder := index $rw "folder" }} -

{{.title}}

-
- {{.title}} -
+ + {{ range $section }} +

{{ .title }}

+
+ {{ .title }} +
{{ end }}