Download performance looks to static while building website

.github/workflows/beam_PreCommit_Website_Stage_GCS.yml

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: PreCommit Website Stage GCS  
+name: PreCommit Website Stage GCS
 
 on:
   push:
@@ -38,7 +38,7 @@ env:
   DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
   GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
   GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
-  gcsbucket: apache-beam-website-pull-requests  
+  gcsbucket: apache-beam-website-pull-requests
   ghprbPullId:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -87,11 +87,17 @@ jobs:
         with:
           python-version: default
           java-version: default
+      - name: Authenticate on GCP
+        id: auth-gcp
+        uses: google-github-actions/auth@v3
+        with:
+          service_account: ${{ secrets.GCP_SA_EMAIL }}
+          credentials_json: ${{ secrets.GCP_SA_KEY }}
       - name: Run website_stageWebsite script
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
           gradle-command: :website:stageWebsite
-          arguments: -PwebsiteBucket=$gcsbucket
+          arguments: -PwebsiteBucket=$gcsbucket -PgcpCredsFile="${{ steps.auth-gcp.outputs.credentials_file_path }}"
       - name: Add website link to the summary if pr
         if: env.ghprbPullId != null
         run: echo "Website published to http://$gcsbucket.storage.googleapis.com/$ghprbPullId/index.html" >> $GITHUB_STEP_SUMMARY

.github/workflows/beam_Publish_Website.yml

@@ -58,9 +58,15 @@ jobs:
         uses: ./.github/actions/setup-environment-action
         with:
           disable-cache: true
+      - name: Authenticate on GCP
+        id: auth-gcp
+        uses: google-github-actions/auth@v3
+        with:
+          service_account: ${{ secrets.GCP_SA_EMAIL }}
+          credentials_json: ${{ secrets.GCP_SA_KEY }}
       - name: run PostCommit Website Publish script
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
           gradle-command: :website:clean :website:publishWebsite
-          arguments: -PgitPublishRemote="https://github.com/apache/beam.git"
+          arguments: -PgitPublishRemote="https://github.com/apache/beam.git" -PgcpCredsFile="${{ steps.auth-gcp.outputs.credentials_file_path }}"
       - uses: actions/checkout@v4 # Extra checkout to make sure we're on master for post steps.

website/Dockerfile

@@ -44,6 +44,23 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
+# Install Google Cloud SDK (gsutil)
+RUN apt-get update \
+    && apt-get install -y apt-transport-https \
+    # Download the key and save it directly to the keyring file (skipping apt-key)
+    && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg \
+       | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg \
+    # Add the repo source using the signed-by tag
+    && echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \
+       | tee /etc/apt/sources.list.d/google-cloud-sdk.list \
+    && apt-get update \
+    && apt-get install -y google-cloud-cli \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV CLOUDSDK_CONFIG=/opt/gcloud-config
+RUN mkdir -p /opt/gcloud-config && chmod 777 /opt/gcloud-config
+
 # Install node LTS environment
 RUN curl -sL https://deb.nodesource.com/setup_lts.x | bash - \
     && apt-get update \

website/build.gradle

@@ -128,6 +128,56 @@ task setupDockerContainer(type: Exec) {
     """echo '[params]\n  branch_repo = "${getBranchRepo()}"' > /tmp/_config_branch_repo.toml"""
 }
 
+def performanceLooksOutDir = "$dockerSourceDir/site/static/images/performance/looks"
+def gcpCredsInContainer = "/tmp/gcp-key.json"
+
+task copyGcpCredsToContainer(type: Exec) {
+  dependsOn setupDockerContainer
+
+  def gcpCredsHost = providers.gradleProperty("gcpCredsFile").orNull
+  if (gcpCredsHost) {
+    commandLine 'docker', 'cp', gcpCredsHost, "${->startDockerContainer.containerId()}:${gcpCredsInContainer}"
+  } else {
+    println "[performance_looks] -PgcpCredsFile not provided, skipping copy"
+  }
+}
+
+task downloadPerformanceLooks(type: Exec) {
+  dependsOn setupDockerContainer, copyGcpCredsToContainer
+
+  def gcpCredsHost = providers.gradleProperty("gcpCredsFile").orNull
+  if (gcpCredsHost) {
+    commandLine 'docker', 'exec',
+      "${-> setupDockerContainer.containerId()}", '/bin/bash', '-lc',
+      """
+      set -u
+
+      SRC="gs://public_looker_explores_us_a3853f40"
+      OUT_DIR="${performanceLooksOutDir}"
+
+      export HOME=/tmp
+      export CLOUDSDK_CONFIG=/tmp/gcloud
+      export GSUTIL_CONFIG_DIR=/tmp/gsutil
+      export GOOGLE_APPLICATION_CREDENTIALS="${gcpCredsInContainer}"
+
+      gcloud auth activate-service-account --key-file="\$GOOGLE_APPLICATION_CREDENTIALS" --no-user-output-enabled
+
+      mkdir -p "\$OUT_DIR"
+
+      echo "[performance_looks] Syncing looks..."
+
+      if gsutil -m rsync -r "\$SRC" "\$OUT_DIR"; then
+        echo "[performance_looks] Download completed successfully"
+      else
+        echo "[performance_looks][WARNING] Failed to sync looks from GCS"
+        echo "[performance_looks][WARNING] Website build will continue without updated images"
+      fi
+    """
+  } else {
+    println "[performance_looks] Skipping download of performance looks"
+  }
+}
+
 task stopAndRemoveDockerContainer(type: Exec) {
   commandLine 'docker', 'rm', '-f', "${->createDockerContainer.containerId()}"
 }
@@ -147,6 +197,7 @@ def createBuildTask = {
   BuildTaskConfiguration config = it as BuildTaskConfiguration
   task "build${config.name}Website" (type:Exec) {
     dependsOn setupDockerContainer
+    dependsOn downloadPerformanceLooks
     finalizedBy stopAndRemoveDockerContainer
 
     def configs = "$dockerSourceDir/site/config.toml"

website/www/site/data/performance.yaml

@@ -12,6 +12,7 @@
 
 host: https://storage.googleapis.com
 path: public_looker_explores_us_a3853f40
+static_base: /images/performance/looks
 looks:
   bigquery:
     read:

website/www/site/layouts/shortcodes/performance_looks.html

@@ -11,20 +11,21 @@
 */}}
 
 <div>
-  {{ $host := .Site.Data.performance.host }}
-  {{ $path := .Site.Data.performance.path }}
+  {{ $staticBase := .Site.Data.performance.static_base | default "/images/performance/looks" }}
   {{ $looks := .Site.Data.performance.looks }}
   {{ $io := index $looks (.Get "io") }}
   {{ $rw := index $io (.Get "read_or_write") }}
   {{ $section := index $rw (.Get "section") }}
-  {{ range $section }}
   {{ $folder := index $rw "folder" }}
-  <h4>{{.title}}</h4>
-    <div class="performance">
-      <img 
-         src="{{$host}}/{{$path}}/{{$folder}}/{{.id}}.png"
-         alt="{{.title}}"
-      >
-    </div>
+
+  {{ range $section }}
+  <h4>{{ .title }}</h4>
+  <div class="performance">
+    <img
+      src="{{ $staticBase }}/{{ $folder }}/{{ .id }}.png"
+      alt="{{ .title }}"
+      loading="lazy"
+    >
+  </div>
   {{ end }}
 </div>