From 53cfcdb66b799da04803ee8bc9fecbc5f32abc72 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 16 Feb 2025 12:57:53 -0500 Subject: [PATCH 1/6] More CI fixes --- .github/workflows/docker.yaml | 49 ++++++++++++++++------------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 2a5ab6c..6c3de18 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -22,25 +22,19 @@ jobs: matrix: clustermanager: - htcondor - # - pbs - # - sge - # - slurm + - pbs + - sge + - slurm steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - name: docker version run: | which -a docker docker version docker compose version - name: docker compose build - # shell: bash -l {0} run: | cd "./ci/${MATRIX_CLUSTERMANAGER:?}" cp ../environment.yml environment.yml @@ -51,26 +45,27 @@ jobs: run: | docker ps -a docker images - # TODO: Docker login here - # echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin - # - name: docker push to GitHub Container Registry - # if: github.event_name != 'pull_request' && matrix.clustermanager != 'htcondor' - # run: | - # docker push "daskdev/dask-clustermanager:${MATRIX_CLUSTERMANAGER:?}" - # docker push ghcr.io//store:latest - # docker build . --tag ghcr.io//store:latest - # env: - # MATRIX_CLUSTERMANAGER: ${{ matrix.clustermanager }} - # - name: Publish secondary SGE images to GitHub Container Registry - # if: github.event_name != 'pull_request' && matrix.clustermanager == 'sge' - # run: | - # docker push daskdev/dask-clustermanager:${{ matrix.clustermanager }}-slave - - name: Publish HTCondor images to GitHub Container Registry - # HTCondor images have composed tags. - # if: github.event_name != 'pull_request' && matrix.clustermanager == 'htcondor' + - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Publish HTCondor images + if: matrix.clustermanager == 'htcondor' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') run: | docker push "ghcr.io/${MY_GCR_NAMESPACE:?}/${MY_GCR_IMAGENAME:?}:${MATRIX_CLUSTERMANAGER:?}-submit" docker push "ghcr.io/${MY_GCR_NAMESPACE:?}/${MY_GCR_IMAGENAME:?}:${MATRIX_CLUSTERMANAGER:?}-execute" env: MATRIX_CLUSTERMANAGER: ${{ matrix.clustermanager }} - + - name: Publish most images (but not HTCondor) + if: matrix.clustermanager != 'htcondor' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + run: | + docker push "ghcr.io/${MY_GCR_NAMESPACE:?}/${MY_GCR_IMAGENAME:?}:${MATRIX_CLUSTERMANAGER:?}" + env: + MATRIX_CLUSTERMANAGER: ${{ matrix.clustermanager }} + - name: Publish secondary SGE images + if: matrix.clustermanager == 'sge' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + run: | + docker push "ghcr.io/${MY_GCR_NAMESPACE:?}/${MY_GCR_IMAGENAME:?}:${MATRIX_CLUSTERMANAGER:?}-slave" + env: + MATRIX_CLUSTERMANAGER: ${{ matrix.clustermanager }} From 89b0ae55772eba94fff1143fb956e31c20c24846 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 16 Feb 2025 12:58:35 -0500 Subject: [PATCH 2/6] Update docker.yaml --- .github/workflows/docker.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 6c3de18..4f84b12 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -4,6 +4,7 @@ on: branches: "main" pull_request: workflow_dispatch: # allows you to trigger manually + merge_group: # GitHub Merge Queue # concurrency: # group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} # cancel-in-progress: true From 7d3a52bff5161a83996ac6f0a982a87192941c14 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 16 Feb 2025 13:00:02 -0500 Subject: [PATCH 3/6] Update docker-compose.yml --- ci/pbs/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/pbs/docker-compose.yml b/ci/pbs/docker-compose.yml index 550c7ae..6bfb62a 100644 --- a/ci/pbs/docker-compose.yml +++ b/ci/pbs/docker-compose.yml @@ -3,7 +3,7 @@ version: "2" services: master: - image: daskdev/dask-jobqueue:pbs + image: ghcr.io/juliaparallel/dask-jobqueue-ci-images:pbs build: . container_name: pbs_master hostname: pbs_master @@ -16,7 +16,7 @@ services: command: bash /run-master.sh slave_one: - image: daskdev/dask-jobqueue:pbs + image: ghcr.io/juliaparallel/dask-jobqueue-ci-images:pbs build: . container_name: pbs_slave_1 hostname: pbs_slave_1 @@ -33,7 +33,7 @@ services: - master slave_two: - image: daskdev/dask-jobqueue:pbs + image: ghcr.io/juliaparallel/dask-jobqueue-ci-images:pbs build: . container_name: pbs_slave_2 hostname: pbs_slave_2 From d8a45ab14cde02f26590e1debbb2b91aab99149a Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 16 Feb 2025 13:00:27 -0500 Subject: [PATCH 4/6] Update docker-compose.yml --- ci/sge/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/sge/docker-compose.yml b/ci/sge/docker-compose.yml index 65d1c6a..02e0c2e 100644 --- a/ci/sge/docker-compose.yml +++ b/ci/sge/docker-compose.yml @@ -3,7 +3,7 @@ version: "3.4" services: master: - image: daskdev/dask-jobqueue:sge + image: ghcr.io/juliaparallel/dask-jobqueue-ci-images:sge build: context: . target: master @@ -19,7 +19,7 @@ services: command: bash /dask-jobqueue/ci/sge/run-master.sh slave-one: - image: daskdev/dask-jobqueue:sge-slave + image: ghcr.io/juliaparallel/dask-jobqueue-ci-images:sge-slave build: context: . target: slave @@ -37,7 +37,7 @@ services: - master slave-two: - image: daskdev/dask-jobqueue:sge-slave + image: ghcr.io/juliaparallel/dask-jobqueue-ci-images:sge-slave build: context: . target: slave From 152c4c748dc852550307653b1fc7ecb01f80424e Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 16 Feb 2025 13:01:05 -0500 Subject: [PATCH 5/6] Update docker-compose.yml --- ci/slurm/docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/slurm/docker-compose.yml b/ci/slurm/docker-compose.yml index ac06352..b0afec1 100644 --- a/ci/slurm/docker-compose.yml +++ b/ci/slurm/docker-compose.yml @@ -16,7 +16,7 @@ services: common-network: slurmdbd: - image: daskdev/dask-jobqueue:slurm + image: ghcr.io/juliaparallel/dask-jobqueue-ci-images:slurm build: . command: ["slurmdbd"] container_name: slurmdbd @@ -33,7 +33,7 @@ services: common-network: slurmctld: - image: daskdev/dask-jobqueue:slurm + image: ghcr.io/juliaparallel/dask-jobqueue-ci-images:slurm build: . command: ["slurmctld"] container_name: slurmctld @@ -58,7 +58,7 @@ services: - NET_ADMIN c1: - image: daskdev/dask-jobqueue:slurm + image: ghcr.io/juliaparallel/dask-jobqueue-ci-images:slurm build: . command: ["slurmd"] hostname: c1 @@ -81,7 +81,7 @@ services: - NET_ADMIN c2: - image: daskdev/dask-jobqueue:slurm + image: ghcr.io/juliaparallel/dask-jobqueue-ci-images:slurm build: . command: ["slurmd"] hostname: c2 From 8823d76220d36d41f21504657f15f06554a24b24 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 16 Feb 2025 13:02:58 -0500 Subject: [PATCH 6/6] Update docker.yaml --- .github/workflows/docker.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 4f84b12..79d0773 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -47,6 +47,7 @@ jobs: docker ps -a docker images - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') with: registry: ghcr.io username: ${{ github.actor }}