bug 1597598 - use taskgraph (#226)

* bug 1597598 - use taskgraph

* add pr kind

* address review comments

* add tox to pr kind-dependencies

* dict comprehension

* don't pre-hash digest-extra; don't set empty `cached_task` attribute

* digest-paths, using hash_paths

* digest-paths -> resources

* script_name transform

* pushsnapscript - shared setup script

* env.update

* pushflatpakscript - shared setup script

* pushapkscript - shared setup script

* _resolve_replace_string

* add parameters. still need to add target tasks method

* cache type

* label

* remove tox artifacts dir for now

* explicit docker-image dep for tox

* target tasks

* parameters['docker_tag']

* docker-image py3{7,8} -> python3{7,8}

* push_docker_image param

* Update taskcluster/docker/k8s-image/Dockerfile

Co-authored-by: Johan Lorenzo <jlorenzo@mozilla.com>

* add comment to pushsnapscript/Dockerfile

* rm chmod from pushsnapscript tox Dockerfile

* Update taskcluster/scriptworker_taskgraph/parameters.py

Co-authored-by: Johan Lorenzo <jlorenzo@mozilla.com>

* remove unused repo_name

* address resources review comments

* fix

* adjust checks

* remove skip_on_project_specific_branches

Co-authored-by: Johan Lorenzo <jlorenzo@mozilla.com>

Author: escapewindow

.taskcluster.yml

@@ -1,7 +1,7 @@
 #/bin/sh
 set -e
 
-test $GIT_HEAD_REV
+test $HEAD_REV
 test $TASK_ID
 test $TASKCLUSTER_ROOT_URL
 test $REPO_URL

docker.d/generate_version_json.sh

@@ -5,7 +5,8 @@ test $DOCKERHUB_EMAIL
 test $DOCKERHUB_USER
 test $DOCKER_REPO
 test $DOCKER_TAG
-test $GIT_HEAD_REV
+test $HEAD_REV
+test $HOME
 test $PUSH_DOCKER_IMAGE
 test $SECRET_URL
 test $PROJECT_NAME
@@ -18,12 +19,12 @@ docker tag $DOCKER_REPO:$DOCKER_TAG $DOCKER_REPO:$DOCKER_ARCHIVE_TAG
 
 echo "=== Generating dockercfg ==="
 # docker login stopped working in Taskcluster for some reason
-wget -qO- $SECRET_URL | jq '.secret.docker.dockercfg' > /root/.dockercfg
-chmod 600 /root/.dockercfg
+wget -qO- $SECRET_URL | jq '.secret.docker.dockercfg' > $HOME/.dockercfg
+chmod 600 $HOME/.dockercfg
 
 echo "=== Pushing to docker hub ==="
 docker push $DOCKER_REPO:$DOCKER_TAG
 docker push $DOCKER_REPO:$DOCKER_ARCHIVE_TAG
 
 echo "=== Clean up ==="
-rm -f /root/.dockercfg
+rm -f $HOME/.dockercfg

docker.d/push_image.sh

@@ -3,16 +3,16 @@ FROM python:3.8
 RUN groupadd --gid 10001 app && \
     useradd -g app --uid 10001 --shell /usr/sbin/nologin --create-home --home-dir /app app
 
-RUN apt-get update \
- && apt-get install -y default-jdk \
- && apt-get clean \
- && ln -s /app/docker.d/healthcheck /bin/healthcheck
+RUN ln -s /app/docker.d/healthcheck /bin/healthcheck
 
-USER app
 WORKDIR /app
 
 COPY . /app
 
+RUN /app/docker.d/image_setup.sh
+
+USER app
+
 RUN python -m venv /app \
  && cd pushapkscript \
  && /app/bin/pip install -r requirements/base.txt \

pushapkscript/Dockerfile

@@ -2,12 +2,12 @@ ARG PYTHON_VERSION
 
 FROM python:${PYTHON_VERSION}
 
-RUN apt-get update && apt-get install -y default-jdk && apt-get clean
-
 WORKDIR /app
 
 COPY MANIFEST.in setup.py tox.ini /app/
 COPY requirements/ /app/requirements/
+COPY docker.d/image_setup.sh /app/
+RUN /app/image_setup.sh
 
 RUN pip install -r requirements/local.txt
 

pushapkscript/Dockerfile.test

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -o errexit -o pipefail
+
+apt-get update
+apt-get install -y default-jdk
+apt-get clean

pushapkscript/docker.d/image_setup.sh

@@ -3,16 +3,15 @@ FROM python:3.8
 RUN groupadd --gid 10001 app && \
     useradd -g app --uid 10001 --shell /usr/sbin/nologin --create-home --home-dir /app app
 
-RUN apt-get update \
- && apt-get install -y gir1.2-ostree-1.0 libgirepository1.0-dev \
- && apt-get clean \
- && ln -s /app/docker.d/healthcheck /bin/healthcheck
+RUN ln -s /app/docker.d/healthcheck /bin/healthcheck
 
-USER app
 WORKDIR /app
-
 COPY . /app
 
+RUN /app/docker.d/image_setup.sh
+
+USER app
+
 RUN python -m venv /app \
  && cd pushflatpakscript \
  && /app/bin/pip install -r requirements/base.txt \

pushflatpakscript/Dockerfile

@@ -2,11 +2,12 @@ ARG PYTHON_VERSION
 
 FROM python:${PYTHON_VERSION}
 
-RUN apt-get update \
- && apt-get install -y gir1.2-ostree-1.0 libgirepository1.0-dev
-
 WORKDIR /app
 
+COPY docker.d/image_setup.sh /app/
+
+RUN /app/image_setup.sh
+
 COPY MANIFEST.in setup.py tox.ini /app/
 COPY requirements/ /app/requirements/
 

pushflatpakscript/Dockerfile.test

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -o errexit -o pipefail
+
+apt-get update
+apt-get install -y gir1.2-ostree-1.0 libgirepository1.0-dev
+apt-get clean

pushflatpakscript/docker.d/image_setup.sh

@@ -3,20 +3,16 @@ FROM python:3.8
 RUN groupadd --gid 10001 app && \
     useradd -g app --uid 10001 --shell /usr/sbin/nologin --create-home --home-dir /app app
 
-RUN apt-get update \
- && apt-get install -y libsodium-dev \
- && apt-get install -y squashfs-tools \
- && apt-get install -y xdelta3 \
- && apt-get clean \
- # XXX Avoid snapcraft from loading useless libs when running on Ubuntu
- && truncate -s 0 /etc/os-release \
- && ln -s /app/docker.d/healthcheck /bin/healthcheck
+RUN ln -s /app/docker.d/healthcheck /bin/healthcheck
 
-USER app
 WORKDIR /app
-
 COPY . /app
 
+# Production image requires extra packages
+RUN /app/docker.d/image_setup.sh libsodium-dev squashfs-tools xdelta3
+
+USER app
+
 RUN python -m venv /app \
  && cd pushsnapscript \
  && /app/bin/pip install -r requirements/base.txt \