Harden Docker image and improve WebPageTest agent diagnostics

Dockerfile

@@ -1,74 +1,92 @@
 ### IMPORTANT DOCKER COMMANDS ###
-
 ###     docker images                               - List images available
-###     docker build <GITHUB-REPO-LINK> -t TAGNAME  - Builds the Dockerfile from the github repo
-###     docker ps                                   - List running images
-###     docker stop <IMAGE ID || IMAGE NAME>        - Stops running image with either --name <IMAGE NAME> || IMAGE ID>
-###     docker run -it -d TAGNAME /bin/bash         - Runs bash
-###     docker exec -it <IMAGE ID> /bin/bash        - Connects to bash for terminal execution (Needs to be running first)
+###     docker build <PATH> -t TAGNAME              - Builds the Dockerfile
+###     docker ps                                   - List running containers
+###     docker stop <CONTAINER ID || NAME>          - Stops running container
+###     docker run -it -d TAGNAME /bin/bash         - Runs bash in detached mode
+###     docker exec -it <CONTAINER ID> /bin/bash    - Connects to running container
 
 ### INSTALLING METHOD ###
+###     Recommended to install with:
+###     docker build . -t wpt-agent-debug:latest
+###     Average build time: ~10 minutes
 
-###     Recommend to install with "docker build <GITHUB-REPO-LINK> -t TAGNAME",
-###     grabs the latest copy of WPT and build time on average takes 10 minutes. 
-
+# ------------------------
+# Base production image
+# ------------------------
 FROM ubuntu:22.04 as production
 
-### TIMEZONE INSIDE THE CONTAINER ###
 ARG TIMEZONE=UTC
 
-### UPDATE ###
-RUN apt update 
-
-### INSTALL APT-GET LIBS ###
-# DEBIAN_FRONTEND prevents interactive prompts while installing
-# set default timezone beforehand to avoid user interaction for tzdata package
-RUN ln -fs /usr/share/zoneinfo/$TIMEZONE /etc/localtime && DEBIAN_FRONTEND=noninteractive apt install -y \
+# --- Update & install dependencies ---
+RUN apt update && \
+    ln -fs /usr/share/zoneinfo/$TIMEZONE /etc/localtime && \
+    DEBIAN_FRONTEND=noninteractive apt install -y \
     python3 python3-pip python3-ujson \
     imagemagick dbus-x11 traceroute software-properties-common psmisc libnss3-tools iproute2 net-tools openvpn \
     libtiff5-dev libjpeg-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python3-tk \
-    python3-dev libavutil-dev libmp3lame-dev libx264-dev yasm autoconf automake build-essential libass-dev libfreetype6-dev libtheora-dev \
-    libtool libvorbis-dev pkg-config texi2html libtext-unidecode-perl python3-numpy python3-scipy perl \
+    python3-dev libavutil-dev libmp3lame-dev libx264-dev yasm autoconf automake build-essential libass-dev libfreetype6-dev \
+    libtheora-dev libtool libvorbis-dev pkg-config texi2html libtext-unidecode-perl python3-numpy python3-scipy perl \
     adb ethtool cmake git-core libsdl2-dev libva-dev libvdpau-dev libxcb1-dev libxcb-shm0-dev libxcb-xfixes0-dev texinfo wget \
-    ttf-mscorefonts-installer fonts-noto fonts-roboto fonts-open-sans ffmpeg sudo curl xvfb
-
-### INSTALL NodeJS ###
-RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - && \
-    apt-get install nodejs npm -y
-
-### UPDATE FONT CACHE ###
+    ttf-mscorefonts-installer fonts-noto fonts-roboto fonts-open-sans ffmpeg sudo curl xvfb gnupg ca-certificates \
+    tini
+
+# ------------------------
+# Install Node.js (fixed)
+# ------------------------
+RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
+    apt-get install -y nodejs && \
+    npm install -g npm@latest
+
+# ------------------------
+# Update font cache
+# ------------------------
 RUN fc-cache -f -v
 
-### INSTALLING LIGHTHOUSE FROM NPM ###
+# ------------------------
+# Install Lighthouse globally
+# ------------------------
 RUN npm install -g lighthouse
 
-### INSTALLING CHROME BROWSER ###
-RUN curl -o /tmp/google-chrome-stable_current_amd64.deb  https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
-    apt install -y /tmp/google-chrome-stable_current_amd64.deb && rm /tmp/google-chrome-stable_current_amd64.deb
-
-### UPGRADING PIP AND INSTALLING REQUIRED PACKAGES ###
+# ------------------------
+# Install Google Chrome (stable)
+# ------------------------
+RUN curl -o /tmp/google-chrome-stable_current_amd64.deb \
+    https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
+    apt install -y /tmp/google-chrome-stable_current_amd64.deb && \
+    rm /tmp/google-chrome-stable_current_amd64.deb
+
+# ------------------------
+# Upgrade pip and install requirements
+# ------------------------
 COPY /.github/workflows/requirements.txt /tmp/agent_requirements.txt
 RUN python3 -m pip install --upgrade --user pip && \
     python3 -m pip install --user -r /tmp/agent_requirements.txt && \
     rm /tmp/agent_requirements.txt
 
-### COPYING ENTIRE DIR TO LOCAL DOCKER /wptagent ###
-# see .dockerignore for filterd out folders
-# source copy last so we don't need to rebuild all the other layers 
+# ------------------------
+# Copy WPT Agent source
+# ------------------------
 COPY / /wptagent
 WORKDIR /wptagent
 
-ENTRYPOINT ["/bin/sh", "/wptagent/docker/linux-headless/entrypoint.sh"]
+# ------------------------
+# Entrypoint for production mode (WITH TINI)
+# ------------------------
+ENTRYPOINT ["/usr/bin/tini", "--", "/bin/sh", "/wptagent/docker/linux-headless/entrypoint.sh"]
 
-### DEBUG CONTAINER ###
+# ------------------------
+# Debug build
+# ------------------------
 FROM production as debug
 
-### INSTALLING DEBUG DEPENDENCIES ###
+# Install debug helper
 RUN pip install debugpy
 
-### COPY DEBUG AGENT AND MOVE REAL ONE ###
+# Replace main agent script with debug version
 RUN mv wptagent.py wptagent_starter.py
 COPY wptagent_debug.py wptagent.py
 
-### SETTING PRODUCTION BUILD AS DEFAULT ###
+# Default to production build
 FROM production
+# FROM debug

internal/chrome_desktop.py

@@ -150,7 +150,7 @@ def launch(self, job, task):
             args.append('--no-sandbox')
         if platform.system() == "Linux":
             args.append('--disable-setuid-sandbox')
-            args.append('--disable-dev-shm-usage')
+#            args.append('--disable-dev-shm-usage')
         if len(features):
             args.append('--enable-features=' + ','.join(features))
         if len(ENABLE_BLINK_FEATURES):

internal/webpagetest.py

@@ -201,6 +201,9 @@ def __init__(self, options, workdir):
         # Load any locally-defined custom metrics from {agent root}/custom/metrics/*.js
         self.custom_metrics = {}
         self.load_local_custom_metrics()
+        # Warn if no server is configured
+        if len(self.work_servers) == 0 and len(self.scheduler_nodes) == 0 and not self.options.pubsub:
+            logging.warning("No WebPageTest server configured. Please specify --server option (e.g., --server http://your-server.com/work/) or --scheduler option.")
     # pylint: enable=E0611
 
     def load_local_custom_metrics(self):
@@ -631,6 +634,7 @@ def get_test(self, browsers):
         proxies = {"http": None, "https": None}
         from .os_util import get_free_disk_space
         if len(self.work_servers) == 0 and len(self.scheduler_nodes) == 0:
+            logging.critical("No work servers or scheduler nodes configured. Please specify --server or --scheduler options.")
             return None
         job = None
         self.raw_job = None
@@ -746,7 +750,10 @@ def get_test(self, browsers):
                     count -= 1
                     retry = True
             except requests.exceptions.RequestException as err:
-                logging.critical("Get Work Error: %s", err.strerror)
+                error_msg = str(err)
+                if hasattr(err, 'response') and err.response is not None:
+                    error_msg = "{} (Status: {})".format(error_msg, err.response.status_code)
+                logging.critical("Get Work Error connecting to %s: %s", url, error_msg)
                 now = monotonic()
                 if self.first_failure is None:
                     self.first_failure = now
@@ -755,8 +762,8 @@ def get_test(self, browsers):
                 if elapsed > 1800:
                     self.reboot()
                 time.sleep(0.1)
-            except Exception:
-                pass
+            except Exception as e:
+                logging.exception("Unexpected error in get_test: %s", str(e))
             # Rotate through the list of servers
             if not retry and job is None and len(servers) > 0 and not self.scheduler:
                 self.url = str(servers.pop(0))
@@ -768,14 +775,17 @@ def get_test(self, browsers):
         return job
 
     def notify_test_started(self, job):
-        """Tell the server that we have started the test. Used when the queueing isn't handled directly by the server responsible for a test"""
+        """
+        Tell the server that we have started the test.
+        Used when the queueing isn't handled directly by the server.
+        """
         if 'work_server' in job and 'Test ID' in job:
             try:
                 url = job['work_server'] + 'started.php?id=' + quote_plus(job['Test ID'])
                 proxies = {"http": None, "https": None}
                 self.session.get(url, timeout=30, proxies=proxies)
             except Exception:
-                logging.exception('Error notifying test start')
+                logging.exception("Unexpected error in notify_test_started")
 
     def get_task(self, job):
         """Create a task object for the next test run or return None if the job is done"""