diff --git a/Dockerfile b/Dockerfile index a349fa0d5..f694b1d21 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,74 +1,92 @@ ### IMPORTANT DOCKER COMMANDS ### - ### docker images - List images available -### docker build -t TAGNAME - Builds the Dockerfile from the github repo -### docker ps - List running images -### docker stop - Stops running image with either --name || IMAGE ID> -### docker run -it -d TAGNAME /bin/bash - Runs bash -### docker exec -it /bin/bash - Connects to bash for terminal execution (Needs to be running first) +### docker build -t TAGNAME - Builds the Dockerfile +### docker ps - List running containers +### docker stop - Stops running container +### docker run -it -d TAGNAME /bin/bash - Runs bash in detached mode +### docker exec -it /bin/bash - Connects to running container ### INSTALLING METHOD ### +### Recommended to install with: +### docker build . -t wpt-agent-debug:latest +### Average build time: ~10 minutes -### Recommend to install with "docker build -t TAGNAME", -### grabs the latest copy of WPT and build time on average takes 10 minutes. - +# ------------------------ +# Base production image +# ------------------------ FROM ubuntu:22.04 as production -### TIMEZONE INSIDE THE CONTAINER ### ARG TIMEZONE=UTC -### UPDATE ### -RUN apt update - -### INSTALL APT-GET LIBS ### -# DEBIAN_FRONTEND prevents interactive prompts while installing -# set default timezone beforehand to avoid user interaction for tzdata package -RUN ln -fs /usr/share/zoneinfo/$TIMEZONE /etc/localtime && DEBIAN_FRONTEND=noninteractive apt install -y \ +# --- Update & install dependencies --- +RUN apt update && \ + ln -fs /usr/share/zoneinfo/$TIMEZONE /etc/localtime && \ + DEBIAN_FRONTEND=noninteractive apt install -y \ python3 python3-pip python3-ujson \ imagemagick dbus-x11 traceroute software-properties-common psmisc libnss3-tools iproute2 net-tools openvpn \ libtiff5-dev libjpeg-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python3-tk \ - python3-dev libavutil-dev libmp3lame-dev libx264-dev yasm autoconf automake build-essential libass-dev libfreetype6-dev libtheora-dev \ - libtool libvorbis-dev pkg-config texi2html libtext-unidecode-perl python3-numpy python3-scipy perl \ + python3-dev libavutil-dev libmp3lame-dev libx264-dev yasm autoconf automake build-essential libass-dev libfreetype6-dev \ + libtheora-dev libtool libvorbis-dev pkg-config texi2html libtext-unidecode-perl python3-numpy python3-scipy perl \ adb ethtool cmake git-core libsdl2-dev libva-dev libvdpau-dev libxcb1-dev libxcb-shm0-dev libxcb-xfixes0-dev texinfo wget \ - ttf-mscorefonts-installer fonts-noto fonts-roboto fonts-open-sans ffmpeg sudo curl xvfb - -### INSTALL NodeJS ### -RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - && \ - apt-get install nodejs npm -y - -### UPDATE FONT CACHE ### + ttf-mscorefonts-installer fonts-noto fonts-roboto fonts-open-sans ffmpeg sudo curl xvfb gnupg ca-certificates \ + tini + +# ------------------------ +# Install Node.js (fixed) +# ------------------------ +RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \ + apt-get install -y nodejs && \ + npm install -g npm@latest + +# ------------------------ +# Update font cache +# ------------------------ RUN fc-cache -f -v -### INSTALLING LIGHTHOUSE FROM NPM ### +# ------------------------ +# Install Lighthouse globally +# ------------------------ RUN npm install -g lighthouse -### INSTALLING CHROME BROWSER ### -RUN curl -o /tmp/google-chrome-stable_current_amd64.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ - apt install -y /tmp/google-chrome-stable_current_amd64.deb && rm /tmp/google-chrome-stable_current_amd64.deb - -### UPGRADING PIP AND INSTALLING REQUIRED PACKAGES ### +# ------------------------ +# Install Google Chrome (stable) +# ------------------------ +RUN curl -o /tmp/google-chrome-stable_current_amd64.deb \ + https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ + apt install -y /tmp/google-chrome-stable_current_amd64.deb && \ + rm /tmp/google-chrome-stable_current_amd64.deb + +# ------------------------ +# Upgrade pip and install requirements +# ------------------------ COPY /.github/workflows/requirements.txt /tmp/agent_requirements.txt RUN python3 -m pip install --upgrade --user pip && \ python3 -m pip install --user -r /tmp/agent_requirements.txt && \ rm /tmp/agent_requirements.txt -### COPYING ENTIRE DIR TO LOCAL DOCKER /wptagent ### -# see .dockerignore for filterd out folders -# source copy last so we don't need to rebuild all the other layers +# ------------------------ +# Copy WPT Agent source +# ------------------------ COPY / /wptagent WORKDIR /wptagent -ENTRYPOINT ["/bin/sh", "/wptagent/docker/linux-headless/entrypoint.sh"] +# ------------------------ +# Entrypoint for production mode (WITH TINI) +# ------------------------ +ENTRYPOINT ["/usr/bin/tini", "--", "/bin/sh", "/wptagent/docker/linux-headless/entrypoint.sh"] -### DEBUG CONTAINER ### +# ------------------------ +# Debug build +# ------------------------ FROM production as debug -### INSTALLING DEBUG DEPENDENCIES ### +# Install debug helper RUN pip install debugpy -### COPY DEBUG AGENT AND MOVE REAL ONE ### +# Replace main agent script with debug version RUN mv wptagent.py wptagent_starter.py COPY wptagent_debug.py wptagent.py -### SETTING PRODUCTION BUILD AS DEFAULT ### +# Default to production build FROM production +# FROM debug diff --git a/internal/chrome_desktop.py b/internal/chrome_desktop.py index 3d7f903fd..8bc57b9d4 100644 --- a/internal/chrome_desktop.py +++ b/internal/chrome_desktop.py @@ -150,7 +150,7 @@ def launch(self, job, task): args.append('--no-sandbox') if platform.system() == "Linux": args.append('--disable-setuid-sandbox') - args.append('--disable-dev-shm-usage') +# args.append('--disable-dev-shm-usage') if len(features): args.append('--enable-features=' + ','.join(features)) if len(ENABLE_BLINK_FEATURES): diff --git a/internal/webpagetest.py b/internal/webpagetest.py index fdc7342f8..014b099a5 100644 --- a/internal/webpagetest.py +++ b/internal/webpagetest.py @@ -201,6 +201,9 @@ def __init__(self, options, workdir): # Load any locally-defined custom metrics from {agent root}/custom/metrics/*.js self.custom_metrics = {} self.load_local_custom_metrics() + # Warn if no server is configured + if len(self.work_servers) == 0 and len(self.scheduler_nodes) == 0 and not self.options.pubsub: + logging.warning("No WebPageTest server configured. Please specify --server option (e.g., --server http://your-server.com/work/) or --scheduler option.") # pylint: enable=E0611 def load_local_custom_metrics(self): @@ -631,6 +634,7 @@ def get_test(self, browsers): proxies = {"http": None, "https": None} from .os_util import get_free_disk_space if len(self.work_servers) == 0 and len(self.scheduler_nodes) == 0: + logging.critical("No work servers or scheduler nodes configured. Please specify --server or --scheduler options.") return None job = None self.raw_job = None @@ -746,7 +750,10 @@ def get_test(self, browsers): count -= 1 retry = True except requests.exceptions.RequestException as err: - logging.critical("Get Work Error: %s", err.strerror) + error_msg = str(err) + if hasattr(err, 'response') and err.response is not None: + error_msg = "{} (Status: {})".format(error_msg, err.response.status_code) + logging.critical("Get Work Error connecting to %s: %s", url, error_msg) now = monotonic() if self.first_failure is None: self.first_failure = now @@ -755,8 +762,8 @@ def get_test(self, browsers): if elapsed > 1800: self.reboot() time.sleep(0.1) - except Exception: - pass + except Exception as e: + logging.exception("Unexpected error in get_test: %s", str(e)) # Rotate through the list of servers if not retry and job is None and len(servers) > 0 and not self.scheduler: self.url = str(servers.pop(0)) @@ -768,14 +775,17 @@ def get_test(self, browsers): return job def notify_test_started(self, job): - """Tell the server that we have started the test. Used when the queueing isn't handled directly by the server responsible for a test""" + """ + Tell the server that we have started the test. + Used when the queueing isn't handled directly by the server. + """ if 'work_server' in job and 'Test ID' in job: try: url = job['work_server'] + 'started.php?id=' + quote_plus(job['Test ID']) proxies = {"http": None, "https": None} self.session.get(url, timeout=30, proxies=proxies) except Exception: - logging.exception('Error notifying test start') + logging.exception("Unexpected error in notify_test_started") def get_task(self, job): """Create a task object for the next test run or return None if the job is done"""