From 9c5455e4484b6a00b46d803c5148a24dbfb75dc3 Mon Sep 17 00:00:00 2001 From: aldbr Date: Thu, 8 May 2025 17:11:55 +0200 Subject: [PATCH 1/2] feat: deploy gubbins images fix: fffff --- .github/workflows/deployment.yml | 106 +++++++++++++++++++++++++++++++ extensions/gubbins_values.yaml | 8 +-- 2 files changed, 109 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 4ab223060..2ff51e708 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -300,3 +300,109 @@ jobs: else echo "No changes to commit" fi + +# ------------------------------------------------------------------------------------------------- +# Gubbins deployment +# ------------------------------------------------------------------------------------------------- + + build-gubbins-wheels: + name: Build wheels + runs-on: "ubuntu-latest" + needs: deploy-pypi + if: ${{ needs.deploy-pypi.outputs.create-release == 'true' }} + defaults: + run: + # We need extglob for REFERENCE_BRANCH substitution + shell: bash -l -O extglob {0} + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Installing dependencies + run: | + python -m pip install \ + build \ + python-dateutil \ + pytz \ + readme_renderer[md] \ + requests \ + setuptools_scm + - name: Build distributions + run: | + # build the gubbins package + for pkg_dir in $PWD/extensions/gubbins/gubbins-*; do + # Skip the testing package + if [[ "${pkg_dir}" =~ .*testing.* ]]; + then + echo "Do not build ${pkg_dir}"; + continue; + fi + echo "Building $pkg_dir" + python -m build --outdir $PWD/dist $pkg_dir + done + - name: 'Upload Artifact' + uses: actions/upload-artifact@v4 + with: + name: gubbins-whl + path: dist/*.whl + retention-days: 5 + + # Build to docker image with the code in it + docker-gubbins: + needs: + - deploy-pypi + - build-gubbins-wheels + if: ${{ needs.deploy-pypi.outputs.create-release == 'true' }} + timeout-minutes: 30 + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GitHub container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Download diracx wheels + uses: actions/download-artifact@v4 + with: + name: diracx-whl + - name: Download gubbins wheels + uses: actions/download-artifact@v4 + with: + name: gubbins-whl + - name: "Find wheels" + id: find_wheel + run: | + # We need to copy them there to be able to access them in the RUN --mount + cp diracx*.whl gubbins*.whl extensions/containers/client/ + cp diracx*.whl gubbins*.whl extensions/containers/services/ + for wheel_fn in *.whl; do + pkg_name=$(basename "${wheel_fn}" | cut -d '-' -f 1) + echo "${pkg_name}-wheel-name=$(ls "${pkg_name}"-*.whl)" >> $GITHUB_OUTPUT + done + + - name: Build and push client (dev) + uses: docker/build-push-action@v6 + with: + context: extensions/containers/client + push: ${{ needs.deploy-pypi.outputs.create-release == 'true' }} + tags: ghcr.io/diracgrid/gubbins/client:dev + platforms: linux/amd64,linux/arm64 + - name: Build and push service (dev) + uses: docker/build-push-action@v6 + with: + context: extensions/containers/services + push: ${{ needs.deploy-pypi.outputs.create-release == 'true' }} + tags: ghcr.io/diracgrid/gubbins/services:dev + platforms: linux/amd64,linux/arm64 + build-args: | + EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration + EXTENSION_CUSTOM_SOURCES_TO_INSTALL=/bindmount/gubbins_db*.whl,/bindmount/gubbins_logic*.whl,/bindmount/gubbins_routers*.whl,/bindmount/gubbins_client*.whl diff --git a/extensions/gubbins_values.yaml b/extensions/gubbins_values.yaml index 4bc01de1a..958750ff8 100644 --- a/extensions/gubbins_values.yaml +++ b/extensions/gubbins_values.yaml @@ -1,11 +1,9 @@ # values specific to testing gubbins global: - # Needed to be able to force pre-load the image in kind - # see https://kind.sigs.k8s.io/docs/user/quick-start/#loading-an-image-into-your-cluster - # and https://iximiuz.com/en/posts/kubernetes-kind-load-docker-image/ - imagePullPolicy: IfNotPresent + imagePullPolicy: Always images: - services: gubbins/services + tag: dev + services: ghcr.io/diracgrid/gubbins/services tag: dev developer: From 1f81e2e72a9ca9deb15f3ccd0f3b3b0938e6a902 Mon Sep 17 00:00:00 2001 From: aldbr Date: Thu, 22 May 2025 09:58:24 +0200 Subject: [PATCH 2/2] fix: adapt to new github workflows --- .github/workflows/deployment.yml | 8 +++---- .github/workflows/main.yml | 39 +++++++++++++++++++++----------- diracx_values.yaml | 11 +++++++++ extensions/gubbins_values.yaml | 6 +++-- 4 files changed, 45 insertions(+), 19 deletions(-) create mode 100644 diracx_values.yaml diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 2ff51e708..93856ab2c 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -309,7 +309,6 @@ jobs: name: Build wheels runs-on: "ubuntu-latest" needs: deploy-pypi - if: ${{ needs.deploy-pypi.outputs.create-release == 'true' }} defaults: run: # We need extglob for REFERENCE_BRANCH substitution @@ -354,7 +353,6 @@ jobs: needs: - deploy-pypi - build-gubbins-wheels - if: ${{ needs.deploy-pypi.outputs.create-release == 'true' }} timeout-minutes: 30 runs-on: ubuntu-latest steps: @@ -393,14 +391,16 @@ jobs: uses: docker/build-push-action@v6 with: context: extensions/containers/client - push: ${{ needs.deploy-pypi.outputs.create-release == 'true' }} + push: ${{ github.event_name != 'pull_request' && github.repository == 'DIRACGrid/diracx' && github.ref_name == 'main' }} tags: ghcr.io/diracgrid/gubbins/client:dev platforms: linux/amd64,linux/arm64 + build-args: | + EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration - name: Build and push service (dev) uses: docker/build-push-action@v6 with: context: extensions/containers/services - push: ${{ needs.deploy-pypi.outputs.create-release == 'true' }} + push: ${{ github.event_name != 'pull_request' && github.repository == 'DIRACGrid/diracx' && github.ref_name == 'main' }} tags: ghcr.io/diracgrid/gubbins/services:dev platforms: linux/amd64,linux/arm64 build-args: | diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1f03728c4..f7b168929 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -99,15 +99,19 @@ jobs: with: cache: false environments: ${{ matrix.extension == 'diracx' && 'default' || 'default-gubbins' }} - - name: Build gubbins wheels - if: ${{ matrix.extension == 'gubbins' }} + - name: Build diracx wheels run: | for pkg_dir in $PWD/diracx-*; do echo "Building $pkg_dir" - pixi exec python-build --outdir $PWD/extensions/containers/services/ $pkg_dir + pixi exec python-build --outdir $PWD/containers/services/ $pkg_dir done # Also build the diracx metapackage - pixi exec python-build --outdir $PWD/extensions/containers/services/ . + pixi exec python-build --outdir $PWD/containers/services/ . + - name: Build gubbins wheels + if: ${{ matrix.extension == 'gubbins' }} + run: | + cp $PWD/containers/services/*.whl $PWD/extensions/containers/services/ + cp $PWD/containers/services/*.tar.gz $PWD/extensions/containers/services/ # And build the gubbins package for pkg_dir in $PWD/extensions/gubbins/gubbins-*; do # Skip the testing package @@ -120,25 +124,30 @@ jobs: pixi exec python-build --outdir $PWD/extensions/containers/services/ $pkg_dir done - name: Set up Docker Buildx - if: ${{ matrix.extension == 'gubbins' }} uses: docker/setup-buildx-action@v3 + - name: Build container for diracx + if: ${{ matrix.extension == 'diracx' }} + uses: docker/build-push-action@v6 + with: + context: containers/services + tags: diracx/services:dev + load: true + build-args: | + EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration - name: Build container for gubbins if: ${{ matrix.extension == 'gubbins' }} uses: docker/build-push-action@v6 with: context: extensions/containers/services tags: gubbins/services:dev - outputs: type=docker,dest=/tmp/gubbins_services_image.tar + load: true build-args: | + EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration EXTENSION_CUSTOM_SOURCES_TO_INSTALL=/bindmount/gubbins_db*.whl,/bindmount/gubbins_logic*.whl,/bindmount/gubbins_routers*.whl,/bindmount/gubbins_client*.whl - - name: Load image - if: ${{ matrix.extension == 'gubbins' }} - run: | - docker load --input /tmp/gubbins_services_image.tar - docker image ls -a + - name: Start demo run: | - git clone https://github.com/DIRACGrid/diracx-charts.git ../diracx-charts + git clone --branch main_FEAT_remove-loaded-images https://github.com/aldbr/diracx-charts.git ../diracx-charts declare -a demo_args=() demo_args+=("--enable-open-telemetry") @@ -146,6 +155,7 @@ jobs: demo_args+=("--exit-when-done") demo_args+=("--set-value" "developer.autoReload=false") demo_args+=("--ci-values" "../diracx-charts/demo/ci_values.yaml") + demo_args+=("--prune-loaded-images") declare -a demo_source_dirs=("$PWD") @@ -162,7 +172,10 @@ jobs: demo_args+=("--ci-values" "./extensions/gubbins_values.yaml") demo_args+=("--load-docker-image" "gubbins/services:dev") demo_source_dirs+=("/tmp/gubbins/") - elif [ ${{ matrix.extension }} != 'diracx' ]; then + elif [ ${{ matrix.extension }} == 'diracx' ]; then + demo_args+=("--ci-values" "./diracx_values.yaml") + demo_args+=("--load-docker-image" "diracx/services:dev") + else echo "Unknown extension: ${{ matrix.extension }}" exit 1 fi diff --git a/diracx_values.yaml b/diracx_values.yaml new file mode 100644 index 000000000..4569ddce3 --- /dev/null +++ b/diracx_values.yaml @@ -0,0 +1,11 @@ +# values specific to testing gubbins +global: + # Needed to be able to force pre-load the image in kind + # see https://kind.sigs.k8s.io/docs/user/quick-start/#loading-an-image-into-your-cluster + # and https://iximiuz.com/en/posts/kubernetes-kind-load-docker-image/ + imagePullPolicy: IfNotPresent + images: + services: diracx/services + +developer: + enabled: true diff --git a/extensions/gubbins_values.yaml b/extensions/gubbins_values.yaml index 958750ff8..58b3a4bed 100644 --- a/extensions/gubbins_values.yaml +++ b/extensions/gubbins_values.yaml @@ -1,8 +1,10 @@ # values specific to testing gubbins global: - imagePullPolicy: Always + # Needed to be able to force pre-load the image in kind + # see https://kind.sigs.k8s.io/docs/user/quick-start/#loading-an-image-into-your-cluster + # and https://iximiuz.com/en/posts/kubernetes-kind-load-docker-image/ + imagePullPolicy: IfNotPresent images: - tag: dev services: ghcr.io/diracgrid/gubbins/services tag: dev