From 45a0522ded609e44d6ac2ac8cb63f2aac95e4b98 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 16 Feb 2025 12:18:47 -0500 Subject: [PATCH 01/10] Fix CI --- .../{build-docker-images.yaml => docker.yaml} | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) rename .github/workflows/{build-docker-images.yaml => docker.yaml} (79%) diff --git a/.github/workflows/build-docker-images.yaml b/.github/workflows/docker.yaml similarity index 79% rename from .github/workflows/build-docker-images.yaml rename to .github/workflows/docker.yaml index 447b0c0..e8c5450 100644 --- a/.github/workflows/build-docker-images.yaml +++ b/.github/workflows/docker.yaml @@ -1,16 +1,14 @@ -name: docker-images-build +name: Docker on: push: branches: "main" pull_request: - schedule: - - cron: "0 0 * * *" # Daily “At 00:00” workflow_dispatch: # allows you to trigger manually -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true +# concurrency: +# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} +# cancel-in-progress: true jobs: build: @@ -18,16 +16,20 @@ jobs: strategy: fail-fast: false matrix: - jobqueue: ["pbs", "sge", "slurm", "htcondor"] - + jobqueue: + - pbs + - sge + - slurm + - htcondor steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Check versions + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - name: docker version run: | docker version docker compose version - - name: Building Image + - name: docker compose build shell: bash -l {0} run: | cd ./ci/${{ matrix.jobqueue }} From 49ea4a80cce1deaffd2393fb95578c3713a4e023 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 16 Feb 2025 12:25:03 -0500 Subject: [PATCH 02/10] Update docker.yaml --- .github/workflows/docker.yaml | 43 +++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index e8c5450..80c3d9a 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -5,18 +5,19 @@ on: branches: "main" pull_request: workflow_dispatch: # allows you to trigger manually - # concurrency: # group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} # cancel-in-progress: true - +permissions: + contents: read + packages: write jobs: build: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - jobqueue: + clustermanager: - pbs - sge - slurm @@ -25,37 +26,45 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false + - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: docker version run: | docker version docker compose version - name: docker compose build - shell: bash -l {0} + # shell: bash -l {0} run: | - cd ./ci/${{ matrix.jobqueue }} + cd "./ci/${MATRIX_CLUSTERMANAGER:?}" cp ../environment.yml environment.yml docker compose build + env: + MATRIX_CLUSTERMANAGER: ${{ matrix.clustermanager }} + - name: List images run: | docker ps -a docker images - - - name: Publish main images to DockerHub Registry - if: github.event_name != 'pull_request' && matrix.jobqueue != 'htcondor' + # TODO: Docker login here + - name: docker push to GitHub Container Registry + if: github.event_name != 'pull_request' && matrix.clustermanager != 'htcondor' run: | echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin - docker push daskdev/dask-jobqueue:${{ matrix.jobqueue }} + docker push daskdev/dask-clustermanager:${{ matrix.clustermanager }} - - name: Publish secondary SGE images to DockerHub Registry - if: github.event_name != 'pull_request' && matrix.jobqueue == 'sge' + - name: Publish secondary SGE images to GitHub Container Registry + if: github.event_name != 'pull_request' && matrix.clustermanager == 'sge' run: | echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin - docker push daskdev/dask-jobqueue:${{ matrix.jobqueue }}-slave + docker push daskdev/dask-clustermanager:${{ matrix.clustermanager }}-slave - #HTCondor images have composed tags. - - name: Publish HTCondor images to DockerHub Registry - if: github.event_name != 'pull_request' && matrix.jobqueue == 'htcondor' + # HTCondor images have composed tags. + - name: Publish HTCondor images to GitHub Container Registry + if: github.event_name != 'pull_request' && matrix.clustermanager == 'htcondor' run: | echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin - docker push daskdev/dask-jobqueue:${{ matrix.jobqueue }}-submit - docker push daskdev/dask-jobqueue:${{ matrix.jobqueue }}-execute + docker push daskdev/dask-clustermanager:${{ matrix.clustermanager }}-submit + docker push daskdev/dask-clustermanager:${{ matrix.clustermanager }}-execute From 20df782a23d3fa51186ff19b46c387210e94b135 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 16 Feb 2025 12:25:29 -0500 Subject: [PATCH 03/10] Update docker.yaml --- .github/workflows/docker.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 80c3d9a..c0f51cd 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -18,10 +18,10 @@ jobs: fail-fast: false matrix: clustermanager: - - pbs - - sge - - slurm - htcondor + # - pbs + # - sge + # - slurm steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: From a3e36b0260b85cc4c9c6cbb9fb2c2c8bf4c34af7 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 16 Feb 2025 12:32:17 -0500 Subject: [PATCH 04/10] Update docker.yaml --- .github/workflows/docker.yaml | 41 ++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index c0f51cd..a4f2316 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -8,6 +8,9 @@ on: # concurrency: # group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} # cancel-in-progress: true +env: + MY_GCR_NAMESPACE: 'JuliaParallel' + MY_GCR_IMAGENAME: 'dask-jobqueue-ci-images' permissions: contents: read packages: write @@ -43,28 +46,30 @@ jobs: docker compose build env: MATRIX_CLUSTERMANAGER: ${{ matrix.clustermanager }} - - name: List images run: | docker ps -a docker images # TODO: Docker login here - - name: docker push to GitHub Container Registry - if: github.event_name != 'pull_request' && matrix.clustermanager != 'htcondor' - run: | - echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin - docker push daskdev/dask-clustermanager:${{ matrix.clustermanager }} - - - name: Publish secondary SGE images to GitHub Container Registry - if: github.event_name != 'pull_request' && matrix.clustermanager == 'sge' - run: | - echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin - docker push daskdev/dask-clustermanager:${{ matrix.clustermanager }}-slave - - # HTCondor images have composed tags. + # echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin + # - name: docker push to GitHub Container Registry + # if: github.event_name != 'pull_request' && matrix.clustermanager != 'htcondor' + # run: | + # docker push "daskdev/dask-clustermanager:${MATRIX_CLUSTERMANAGER:?}" + # docker push ghcr.io//store:latest + # docker build . --tag ghcr.io//store:latest + # env: + # MATRIX_CLUSTERMANAGER: ${{ matrix.clustermanager }} + # - name: Publish secondary SGE images to GitHub Container Registry + # if: github.event_name != 'pull_request' && matrix.clustermanager == 'sge' + # run: | + # docker push daskdev/dask-clustermanager:${{ matrix.clustermanager }}-slave - name: Publish HTCondor images to GitHub Container Registry - if: github.event_name != 'pull_request' && matrix.clustermanager == 'htcondor' + # HTCondor images have composed tags. + # if: github.event_name != 'pull_request' && matrix.clustermanager == 'htcondor' run: | - echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin - docker push daskdev/dask-clustermanager:${{ matrix.clustermanager }}-submit - docker push daskdev/dask-clustermanager:${{ matrix.clustermanager }}-execute + docker push "ghcr.io/${MY_GCR_NAMESPACE:?}/${MY_GCR_IMAGENAME:?}:${MATRIX_CLUSTERMANAGER:?}-submit" + docker push "ghcr.io/${MY_GCR_NAMESPACE:?}/${MY_GCR_IMAGENAME:?}:${MATRIX_CLUSTERMANAGER:?}-execute" + env: + MATRIX_CLUSTERMANAGER: ${{ matrix.clustermanager }} + From c5f252c6cb73b35d12f07da125ecfe9b6c2cb8ec Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 16 Feb 2025 12:32:53 -0500 Subject: [PATCH 05/10] Update docker.yaml --- .github/workflows/docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index a4f2316..a85864a 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -15,7 +15,7 @@ permissions: contents: read packages: write jobs: - build: + build-and-push: runs-on: ubuntu-latest strategy: fail-fast: false From db43a04dbd3fa0c4fd1040d109929781453f6c0c Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 16 Feb 2025 12:34:04 -0500 Subject: [PATCH 06/10] Update docker.yaml --- .github/workflows/docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index a85864a..378791b 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -1,5 +1,4 @@ name: Docker - on: push: branches: "main" @@ -36,6 +35,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: docker version run: | + which -a docker docker version docker compose version - name: docker compose build From fd1c58166dbc9f427026b6fe9112bbb73a7ec8a6 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 16 Feb 2025 12:34:59 -0500 Subject: [PATCH 07/10] Update docker.yaml --- .github/workflows/docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 378791b..faf05e7 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -8,7 +8,7 @@ on: # group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} # cancel-in-progress: true env: - MY_GCR_NAMESPACE: 'JuliaParallel' + MY_GCR_NAMESPACE: 'juliaparallel' MY_GCR_IMAGENAME: 'dask-jobqueue-ci-images' permissions: contents: read From 84b9e09ddf1cdfc78a043950cf852d248129ff73 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 16 Feb 2025 12:38:33 -0500 Subject: [PATCH 08/10] Update docker-compose.yml --- ci/htcondor/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/htcondor/docker-compose.yml b/ci/htcondor/docker-compose.yml index 358aafb..3fdce8e 100644 --- a/ci/htcondor/docker-compose.yml +++ b/ci/htcondor/docker-compose.yml @@ -12,7 +12,7 @@ services: command: bash -c 'condor_store_cred -p password -f /root/secrets/pool_password ; exec bash -x /start.sh' submit: - image: daskdev/dask-jobqueue:htcondor-submit + image: ghcr.io/juliaparallel/dask-jobqueue-ci-images-submit build: context: . target: submit @@ -30,7 +30,7 @@ services: - shared_space:/shared_space execute1: - image: daskdev/dask-jobqueue:htcondor-execute + image: ghcr.io/juliaparallel/dask-jobqueue-ci-images-execute build: context: . target: execute @@ -46,7 +46,7 @@ services: - shared_space:/shared_space execute2: - image: daskdev/dask-jobqueue:htcondor-execute + image: ghcr.io/juliaparallel/dask-jobqueue-ci-images-execute build: context: . target: execute From 91ccdc927aea52fb2a168e9e4f24c4cfd2783a23 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 16 Feb 2025 12:42:00 -0500 Subject: [PATCH 09/10] Update docker-compose.yml --- ci/htcondor/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/htcondor/docker-compose.yml b/ci/htcondor/docker-compose.yml index 3fdce8e..029ac85 100644 --- a/ci/htcondor/docker-compose.yml +++ b/ci/htcondor/docker-compose.yml @@ -12,7 +12,7 @@ services: command: bash -c 'condor_store_cred -p password -f /root/secrets/pool_password ; exec bash -x /start.sh' submit: - image: ghcr.io/juliaparallel/dask-jobqueue-ci-images-submit + image: ghcr.io/juliaparallel/dask-jobqueue-ci-images:htcondor-submit build: context: . target: submit @@ -30,7 +30,7 @@ services: - shared_space:/shared_space execute1: - image: ghcr.io/juliaparallel/dask-jobqueue-ci-images-execute + image: ghcr.io/juliaparallel/dask-jobqueue-ci-images:htcondor-execute build: context: . target: execute @@ -46,7 +46,7 @@ services: - shared_space:/shared_space execute2: - image: ghcr.io/juliaparallel/dask-jobqueue-ci-images-execute + image: ghcr.io/juliaparallel/dask-jobqueue-ci-images:htcondor-execute build: context: . target: execute From 854c2cb07830d4225c82b4786c5bae49e4ba50d3 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 16 Feb 2025 12:47:25 -0500 Subject: [PATCH 10/10] Update docker.yaml --- .github/workflows/docker.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index faf05e7..2a5ab6c 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -16,6 +16,7 @@ permissions: jobs: build-and-push: runs-on: ubuntu-latest + timeout-minutes: 60 strategy: fail-fast: false matrix: