From f4ca53ee400400c289bea1d8808b55d5c43e813f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20W=C3=B3jtowicz?= Date: Mon, 15 Dec 2025 10:12:58 +0100 Subject: [PATCH 1/2] Add script to create docker images Add CI work flows Formatting --- .github/workflows/build.yml | 114 ++++++++++++++ .github/workflows/checks.yml | 28 ++++ .github/workflows/jq-install-plan | 1 + .github/workflows/release-ghcr.yaml | 207 +++++++++++++++++++++++++ nix/outputs.nix | 20 ++- nix/project.nix | 18 ++- nix/utils.nix | 30 ++-- scripts/ci/cabal.project.local.Linux | 7 + scripts/ci/cabal.project.local.Windows | 9 ++ 9 files changed, 412 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/checks.yml create mode 100644 .github/workflows/jq-install-plan create mode 100644 .github/workflows/release-ghcr.yaml create mode 100644 scripts/ci/cabal.project.local.Linux create mode 100644 scripts/ci/cabal.project.local.Windows diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..e64d708 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,114 @@ +name: Haskell CI + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +on: + pull_request: + merge_group: + push: + branches: + - main + - "release/*" + +jobs: + build: + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + ghc: ["9.6", "9.12"] + os: [ubuntu-latest] + + env: + # Modify this value to "invalidate" all cabal caches. + CABAL_CACHE_VERSION: "2024-05-22" + # Modify this value to "invalidate" the cabal store cache only. + CACHE_VERSION: "20220919" + # Modify this value to "invalidate" the dist-newstyle cache only. + DIST_CACHE_VERSION: "20221122" + + steps: + + - name: "Install System Dependencies via pacman (msys2)" + if: runner.os == 'Windows' + shell: 'C:/msys64/usr/bin/bash.exe -e {0}' + run: | + /usr/bin/pacman -S --noconfirm mingw-w64-x86_64-pkg-config mingw-w64-x86_64-openssl mingw-w64-x86_64-sed base-devel autoconf-wrapper autoconf automake libtool make + + - name: Install Haskell + uses: input-output-hk/actions/haskell@latest + id: setup-haskell + with: + ghc-version: ${{ matrix.ghc }} + cabal-version: "3.12.1.0" + + - uses: actions/checkout@v6 + + - name: "Configure cabal.project.local" + run: | + cp scripts/ci/cabal.project.local.${{ runner.os }} cabal.project.local + + - name: Update PATH + if: runner.os == 'Windows' + run: | + $env:PATH=("C:\msys64\mingw64\bin;{0}" -f $env:PATH) + echo "PATH=$env:PATH" >> $env:GITHUB_ENV + + - name: Update Hackage and CHaP + run: cabal update + + - name: Record dependencies + id: record-deps + run: | + cabal build all --dry-run + cat dist-newstyle/cache/plan.json | jq -L .github/workflows/jq-install-plan | sort | uniq > dependencies.txt + + - uses: actions/cache/restore@v4 + name: "Restore cache: `cabal store`" + id: cache-dependencies + with: + path: ${{ steps.setup-haskell.outputs.cabal-store }} + key: cache-dependencies-${{ env.CABAL_CACHE_VERSION }}-${{ env.CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('dependencies.txt') }} + restore-keys: cache-dependencies-${{ env.CABAL_CACHE_VERSION }}-${{ env.CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }} + + - uses: actions/cache@v4 + name: "Cache `dist-newstyle`" + with: + path: | + dist-newstyle + !dist-newstyle/**/.git + key: cache-dist-${{ env.CABAL_CACHE_VERSION }}-${{ env.DIST_CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('cabal.project') }} + restore-keys: cache-dist-${{ env.CABAL_CACHE_VERSION }}-${{ env.DIST_CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }} + + - name: Build dependencies + run: cabal build --only-dependencies all -j + + - uses: actions/cache/save@v4 + name: "Save cache: `cabal store`" + if: always() && steps.cache-dependencies.outputs.cache-hit != 'true' + with: + path: ${{ steps.setup-haskell.outputs.cabal-store }} + key: ${{ steps.cache-dependencies.outputs.cache-primary-key }} + + - name: Build all packages + run: cabal build all -j + + # Uncomment the following back in for debugging. Remember to launch a `pwsh` from + # the tmux session to debug `pwsh` issues. And be reminded that the `/msys2` and + # `/msys2/mingw64` paths are not in PATH by default for the workflow, but tmate + # will put them in. + # You may also want to run + # + # $env:PATH=("C:\Program Files\PowerShell\7;{0}" -f $env:ORIGINAL_PATH) + # + # to restore the original path. Do note that some test might need msys2 + # and will silently fail if msys2 is not in path. See the "Run tests" step. + # + # - name: Setup tmate session + # if: ${{ failure() }} + # uses: mxschmitt/action-tmate@v3 + # with: + # limit-access-to-actor: true diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 0000000..5ad7fcd --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,28 @@ +name: Project checks + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +on: + pull_request: + merge_group: + +jobs: + check-changelogs: + name: Check changelogs + runs-on: ubuntu-latest + defaults: + run: + shell: bash + + steps: + - name: Install dependencies + run: sudo apt install -y fd-find + + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Check changelogs + run: ./scripts/ci/check-changelogs.sh \ No newline at end of file diff --git a/.github/workflows/jq-install-plan b/.github/workflows/jq-install-plan new file mode 100644 index 0000000..e4392bf --- /dev/null +++ b/.github/workflows/jq-install-plan @@ -0,0 +1 @@ +.["install-plan"][].id \ No newline at end of file diff --git a/.github/workflows/release-ghcr.yaml b/.github/workflows/release-ghcr.yaml new file mode 100644 index 0000000..92cbb58 --- /dev/null +++ b/.github/workflows/release-ghcr.yaml @@ -0,0 +1,207 @@ +name: Upload to ghcr.io + +on: + push: + tags: + - '**' + # GITHUB_SHA: Last commit in the tagged release + # GITHUB_REF: Tag ref of release refs/tags/ + release: + types: + - published + # GITHUB_SHA: Last commit on the GITHUB_REF branch or tag + # GITHUB_REF: Branch or tag that received dispatch + workflow_dispatch: {} + +permissions: + contents: read + packages: write + +env: + # Only to avoid some repetition + FLAKE_REF: github:${{ github.repository }}/${{ github.ref_name }} + GH_TOKEN: ${{ github.token }} + +jobs: + wait-for-hydra: + name: "Wait for hydra check-runs" + runs-on: ubuntu-latest + + steps: + - name: Waiting for ci/hydra-build:x86_64-linux.required to complete + run: | + while [[ true ]]; do + check_name='ci/hydra-build:x86_64-linux.required' + conclusion=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/check-runs?check_name=$check_name" --paginate --jq '.check_runs[].conclusion') + case "$conclusion" in + success) + echo "$check_name succeeded" + exit 0;; + '') + echo "$check_name pending. Waiting 30s..." + sleep 30;; + *) + echo "$check_name terminated unsuccessfully" + exit 1;; + esac + done + + + prepare: + needs: [wait-for-hydra] + name: "Prepare metadata" + runs-on: ubuntu-latest + outputs: + LATEST_TAG: ${{ steps.latest-tag.outputs.LATEST_TAG }} + LOCKED_URL: ${{ steps.flake-metadata.outputs.LOCKED_URL }} + + steps: + - name: Install Nix + uses: cachix/install-nix-action@v31 + + - name: Display flake metadata + id: flake-metadata + run: | + nix flake metadata ${{ env.FLAKE_REF }} + nix flake metadata ${{ env.FLAKE_REF }} --json | jq -r '"LOCKED_URL=\(.url)"' >> "$GITHUB_OUTPUT" + + - name: Obtaining latest release tag + id: latest-tag + run: | + LATEST_TAG=0.2.0.0 + # LATEST_TAG=$(gh api repos/$GITHUB_REPOSITORY/releases/latest --paginate --jq '.tag_name') + # echo "LATEST_TAG=$LATEST_TAG" >> "$GITHUB_OUTPUT" + # echo "Latest release tag is: $LATEST_TAG" + + + build: + needs: [prepare] + name: "Upload to ghcr.io" + runs-on: ubuntu-latest + strategy: + matrix: + arch: + - name: amd64 + system: x86_64-linux + image: + - name: dmq-node + nix_key: docker-dmq + + steps: + - name: Install Nix + uses: cachix/install-nix-action@v31 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # NOTE We assume that hydra has already built the image, this is + # reasonable since, before applying the tag, we must have already + # pushed the tagged commit somewhere, and Hydra will have had the + # change to build the image. + + - name: Uploading ${{ matrix.image.name }} (${{ matrix.arch.name }}) + run: | + echo "::group::Downloading from cache" + nix build \ + --accept-flake-config \ + --print-out-paths \ + --builders "" \ + --max-jobs 0 \ + --out-link ./result-${{ matrix.image.name }}-${{ matrix.arch.name }} \ + ${{ needs.prepare.outputs.LOCKED_URL }}#packages.${{ matrix.arch.system }}.${{ matrix.image.nix_key }} + echo "::endgroup::" + + echo "::group::Uploading to registry" + skopeo copy \ + docker-archive:./result-${{ matrix.image.name }}-${{ matrix.arch.name }} \ + docker://ghcr.io/intersectmbo/${{ matrix.image.name }}:$GITHUB_REF_NAME-${{ matrix.arch.name }} + echo "::endgroup::" + + + create-manifest: + needs: [prepare, build] + name: "Create Multi-Arch Manifest" + runs-on: ubuntu-latest + + steps: + - name: Install Nix + uses: cachix/install-nix-action@v31 + + # Regctl simplifies obtaining multi-arch digests + - name: Install Nix Profile Commands + run: nix profile install nixpkgs#regctl + + # The docker buildx action has a tight coupling with GH runners + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Show buildx configuration + run: docker buildx ls + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create Manifests + run: | + REPOS=(dmq-node) + ARCHES=(amd64) + + for REPO in "${REPOS[@]}"; do + IMAGE_REPO="ghcr.io/intersectmbo/$REPO" + DIGESTS=() + + echo "::group::Fetching digests for $REPO" + for ARCH in "${ARCHES[@]}"; do + DIGEST=$(skopeo inspect --no-tags "docker://$IMAGE_REPO:$GITHUB_REF_NAME-$ARCH" | jq -r .Digest) + echo "$REPO $ARCH digest: $DIGEST" + DIGESTS+=("$IMAGE_REPO@$DIGEST") + done + echo "::endgroup::" + + echo "::group::Creating manifest for $REPO:$GITHUB_REF_NAME" + docker buildx imagetools create --tag "$IMAGE_REPO:$GITHUB_REF_NAME" "${DIGESTS[@]}" + echo "::endgroup::" + done + + - name: Verify multi-arch manifests + run: | + for REPO in dmq-node; do + IMAGE_REPO="ghcr.io/intersectmbo/$REPO" + echo "::group::Inspecting $REPO:$GITHUB_REF_NAME" + + DIGEST=$(regctl manifest head "$IMAGE_REPO:$GITHUB_REF_NAME") + echo "$REPO multi-arch manifest digest: $DIGEST" + skopeo inspect --raw "docker://$IMAGE_REPO:$GITHUB_REF_NAME" | jq + + echo "::endgroup::" + done + + - name: Tag Containers as :latest + # Github releases are checked for latest tag in the first `or` operand of + # the if statement. However, promoted pre-releases or changed full + # releases do not count as a `published` event and so won't trigger + # this workflow. For those use cases a manual workflow must be run + # from the matching release tag which the second `or` operand checks + # for. + if: | + (github.event_name == 'release' && github.event.release.tag_name == needs.prepare.outputs.LATEST_TAG) || + (github.event_name == 'workflow_dispatch' && github.ref == format('refs/tags/{0}', needs.prepare.outputs.LATEST_TAG)) + run: | + REPOS=(dmq-node) + + for REPO in "${REPOS[@]}"; do + IMAGE_REPO="ghcr.io/intersectmbo/$REPO" + DIGEST=$(regctl manifest head "$IMAGE_REPO:$GITHUB_REF_NAME") + + echo "::group::Creating manifest for $IMAGE_REPO:latest" + docker buildx imagetools create --tag "$IMAGE_REPO:latest" "$IMAGE_REPO@$DIGEST" + echo "::endgroup::" + done \ No newline at end of file diff --git a/nix/outputs.nix b/nix/outputs.nix index 6ce24cb..fe249a6 100644 --- a/nix/outputs.nix +++ b/nix/outputs.nix @@ -11,6 +11,8 @@ let mkShell = ghc: import ./shell.nix { inherit inputs pkgs lib project utils ghc; }; + buildSystem = pkgs.buildPlatform.system; + packages = rec { # TODO: `nix build .\#dmq-node` will have the git revision set in the binary, # `nib build .\#hydraJobs.x86_64-linux.packages.dmq-node:exe:dmq-node` won't @@ -19,11 +21,27 @@ let # pkgs.setGitRev # (inputs.self.rev or inputs.self.dirtyShortRev) project.hsPkgs.dmq-node.components.exes.dmq-node; + default = dmq-node; + } // lib.optionalAttrs (buildSystem == "x86_64-linux") { dmq-node-static = # pkgs.setGitRev # (inputs.self.rev or inputs.self.dirtyShortRev) project.projectCross.musl64.hsPkgs.dmq-node.components.exes.dmq-node; - default = dmq-node; + docker-dmq = pkgs.dockerTools.buildImage { + name = "docker-dmq-node"; + tag = "latest"; + created = "now"; + copyToRoot = pkgs.buildEnv { + name = "dmq-env"; + paths = [ + pkgs.busybox + pkgs.dockerTools.caCertificates + ]; + }; + config = { + Entrypoint = [ "${packages.dmq-node-static}/bin/dmq-node-static" ]; + }; + }; }; app = { diff --git a/nix/project.nix b/nix/project.nix index 0547dc4..6acc18c 100644 --- a/nix/project.nix +++ b/nix/project.nix @@ -21,10 +21,15 @@ let crossPlatforms = p: lib.optionals (pkgs.stdenv.hostPlatform.isLinux && config.compiler-nix-name == "ghc966") - [ p.ucrt64 p.musl64 ]; - - modules = [{ - packages = lib.mkIf pkgs.stdenv.hostPlatform.isMusl { + [ p.ucrt64 p.musl64 ]; + + modules = [ + { + packages.dmq-node.components.tests.dmq-cddl.build-tools = [ pkgs.cddl pkgs.cbor-diag pkgs.cddlc ]; + packages.dmq-node.components.tests.dmq-cddl.preCheck = "export HOME=`pwd`"; + } + { + packages = lib.mkIf pkgs.stdenv.hostPlatform.isMusl { # ruby fails to build with musl, hence we disable cddl tests dmq-node.components.tests.dmq-cddl.build-tools = lib.mkForce [ ]; dmq-node.components.tests.dmq-cddl.doCheck = lib.mkForce false; @@ -34,8 +39,9 @@ let "-L${lib.getLib static-secp256k1}/lib" "-L${lib.getLib static-libblst}/lib" ]; - }; - }]; + }; + } + ]; } ); diff --git a/nix/utils.nix b/nix/utils.nix index 441ac70..fe375d5 100644 --- a/nix/utils.nix +++ b/nix/utils.nix @@ -8,16 +8,16 @@ rec { flatten (if name == "" then name' else "${name}${separator}${name'}"); flatten = name': value: - let - name = builtins.replaceStrings [":"] [separator] name'; - in - if lib.isDerivation value || lib.typeOf value != "set" then - [{ inherit name value; }] - else - lib.concatLists (lib.mapAttrsToList (recurse name) value); + let + name = builtins.replaceStrings [ ":" ] [ separator ] name'; + in + if lib.isDerivation value || lib.typeOf value != "set" then + [{ inherit name value; }] + else + lib.concatLists (lib.mapAttrsToList (recurse name) value); in - assert lib.typeOf set == "set"; - lib.listToAttrs (flatten "" set); + assert lib.typeOf set == "set"; + lib.listToAttrs (flatten "" set); mapAttrsValues = f: lib.mapAttrs (_name: f); @@ -25,13 +25,13 @@ rec { makeHydraRequiredJob = hydraJobs: let - cleanJobs = lib.filterAttrsRecursive + cleanJobs = lib.filterAttrsRecursive (name: _: name != "recurseForDerivations") (removeAttrs hydraJobs [ "required" ]); in - pkgs.releaseTools.aggregate { - name = "required"; - meta.description = "All jobs required to pass CI"; - constituents = lib.collect lib.isDerivation cleanJobs; - }; + pkgs.releaseTools.aggregate { + name = "required"; + meta.description = "All jobs required to pass CI"; + constituents = lib.collect lib.isDerivation cleanJobs; + }; } diff --git a/scripts/ci/cabal.project.local.Linux b/scripts/ci/cabal.project.local.Linux new file mode 100644 index 0000000..091e68d --- /dev/null +++ b/scripts/ci/cabal.project.local.Linux @@ -0,0 +1,7 @@ +max-backjumps: 5000 +reorder-goals: True +tests: True +benchmarks: True + +program-options + ghc-options: -fno-ignore-asserts -Werror diff --git a/scripts/ci/cabal.project.local.Windows b/scripts/ci/cabal.project.local.Windows new file mode 100644 index 0000000..1a91684 --- /dev/null +++ b/scripts/ci/cabal.project.local.Windows @@ -0,0 +1,9 @@ +max-backjumps: 5000 +reorder-goals: True +tests: True +benchmarks: True + +-- IPv6 and nothunks tests are DISABLED on Windows + +program-options + ghc-options: -fno-ignore-asserts -Werror From 91abf6b3a6524ad43b17e934177fbe8fd8cadc97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20W=C3=B3jtowicz?= Date: Thu, 18 Dec 2025 09:35:50 +0100 Subject: [PATCH 2/2] gha: install crypto dependencies --- .github/workflows/build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e64d708..0eaa1be 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,6 +45,11 @@ jobs: ghc-version: ${{ matrix.ghc }} cabal-version: "3.12.1.0" + - name: Install system dependencies + uses: input-output-hk/actions/base@latest + with: + use-sodium-vrf: true # default is true + - uses: actions/checkout@v6 - name: "Configure cabal.project.local"