diff --git a/.github/workflows/.test-bake.yml b/.github/workflows/.test-bake.yml index f0d6d17..7ce57b7 100644 --- a/.github/workflows/.test-bake.yml +++ b/.github/workflows/.test-bake.yml @@ -31,6 +31,8 @@ jobs: contents: read id-token: write with: + cache: true + cache-scope: bake-aws-single context: test output: image push: ${{ github.event_name != 'pull_request' }} @@ -80,6 +82,8 @@ jobs: contents: read id-token: write with: + cache: true + cache-scope: bake-aws context: test output: image push: ${{ github.event_name != 'pull_request' }} @@ -129,6 +133,8 @@ jobs: contents: read id-token: write with: + cache: true + cache-scope: bake-aws-nosign context: test output: image push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/.test-build.yml b/.github/workflows/.test-build.yml index 4bd86d6..457c5ea 100644 --- a/.github/workflows/.test-build.yml +++ b/.github/workflows/.test-build.yml @@ -31,6 +31,8 @@ jobs: contents: read id-token: write with: + cache: true + cache-scope: build-aws-single file: test/hello.Dockerfile output: image push: ${{ github.event_name != 'pull_request' }} @@ -80,6 +82,8 @@ jobs: contents: read id-token: write with: + cache: true + cache-scope: build-aws file: test/hello.Dockerfile output: image platforms: linux/amd64,linux/arm64 @@ -129,6 +133,8 @@ jobs: contents: read id-token: write with: + cache: true + cache-scope: build-aws-nosign file: test/hello.Dockerfile output: image platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/bake.yml b/.github/workflows/bake.yml index aab966f..72df8a4 100644 --- a/.github/workflows/bake.yml +++ b/.github/workflows/bake.yml @@ -23,6 +23,20 @@ on: description: "Upload build output GitHub artifact (for local output)" required: false default: false + cache: + type: boolean + description: "Enable cache to GitHub Actions cache backend" + required: false + default: false + cache-scope: + type: string + description: "Which scope cache object belongs to if cache enabled (defaults to target name)" + required: false + cache-mode: + type: string + description: "Cache layers to export if cache enabled (min or max)" + required: false + default: 'min' context: type: string description: "Context to build from in the Git working tree" @@ -124,7 +138,7 @@ on: env: BUILDX_VERSION: "v0.30.1" - BUILDKIT_IMAGE: "moby/buildkit:v0.26.2" + BUILDKIT_IMAGE: "crazymax/buildkit:6397" DOCKER_ACTIONS_TOOLKIT_MODULE: "@docker/actions-toolkit@0.71.0" COSIGN_VERSION: "v3.0.2" LOCAL_EXPORT_DIR: "/tmp/buildx-output" @@ -136,6 +150,7 @@ jobs: outputs: includes: ${{ steps.set.outputs.includes }} sign: ${{ steps.set.outputs.sign }} + privateRepo: ${{ steps.set.outputs.privateRepo }} steps: - name: Install @docker/actions-toolkit @@ -247,6 +262,11 @@ jobs: } const privateRepo = GitHub.context.payload.repository?.private ?? false; + await core.group(`Set privateRepo output`, async () => { + core.info(`privateRepo: ${privateRepo}`); + core.setOutput('privateRepo', privateRepo); + }); + await core.group(`Set includes output`, async () => { let includes = []; if (platforms.length === 0) { @@ -329,14 +349,113 @@ jobs: if: ${{ inputs.setup-qemu }} with: cache-image: false + - + name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@3cb05d89e1f492524af3d41a1c98c83bc3025124 # v3.1.0 - name: Set up Docker Buildx + id: buildx uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 with: version: ${{ env.BUILDX_VERSION }} - buildkitd-flags: --debug - driver-opts: image=${{ env.BUILDKIT_IMAGE }} cache-binary: false + buildkitd-flags: --debug + driver-opts: | + image=${{ env.BUILDKIT_IMAGE }} + env.ACTIONS_ID_TOKEN_REQUEST_TOKEN=${{ env.ACTIONS_ID_TOKEN_REQUEST_TOKEN }} + env.ACTIONS_ID_TOKEN_REQUEST_URL=${{ env.ACTIONS_ID_TOKEN_REQUEST_URL }} + buildkitd-config-inline: | + [cache] + [cache.gha] + [cache.gha.sign] + command = ["ghacache-sign-script.sh"] + [cache.gha.verify] + required = true + [cache.gha.verify.policy] + timestampThreshold = 1 + tlogThreshold = ${{ matrix.privateRepo == 'true' && '0' || '1' }} + subjectAlternativeName = "https://github.com/docker/github-builder-experimental/.github/workflows/bake.yml*" + githubWorkflowRepository = "docker/github-builder-experimental" + issuer = "https://token.actions.githubusercontent.com" + runnerEnvironment = "github-hosted" + sourceRepositoryURI = "${{ github.server_url }}/${{ github.repository }}" + sourceRepositoryRef = "${{ github.event_name != 'pull_request' && github.ref || '' }}" + - + name: Install Cosign + if: ${{ needs.prepare.outputs.sign == 'true' || inputs.cache }} + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + INPUT_COSIGN-VERSION: ${{ env.COSIGN_VERSION }} + INPUT_BUILDER-NAME: ${{ steps.buildx.outputs.name }} + INPUT_GHA-CACHE-SIGN-SCRIPT: | + #!/bin/sh + set -e + + # Create temporary files + out_file=$(mktemp) + in_file=$(mktemp) + trap 'rm -f "$in_file" "$out_file"' EXIT + cat > "$in_file" + + set -x + + # Sign with cosign + cosign sign-blob \ + --yes \ + --oidc-provider github-actions \ + --new-bundle-format \ + --use-signing-config \ + --bundle "$out_file" \ + --tlog-upload=${{ matrix.privateRepo == 'false' }} \ + "$in_file" + + # Output bundle to stdout + cat "$out_file" + with: + script: | + const fs = require('fs'); + const os = require('os'); + const path = require('path'); + + const { Buildx } = require('@docker/actions-toolkit/lib/buildx/buildx'); + const { Cosign } = require('@docker/actions-toolkit/lib/cosign/cosign'); + const { Install } = require('@docker/actions-toolkit/lib/cosign/install'); + + const inpCosignVersion = core.getInput('cosign-version'); + const inpBuilderName = core.getInput('builder-name'); + const inpGHACacheSignScript = core.getInput('gha-cache-sign-script'); + + const cosignInstall = new Install(); + const cosignBinPath = await cosignInstall.download({ + version: core.getInput('cosign-version'), + ghaNoCache: true, + skipState: true, + verifySignature: true + }); + const cosignPath = await cosignInstall.install(cosignBinPath); + + const cosign = new Cosign(); + await cosign.printVersion(); + + const containerName = `${Buildx.containerNamePrefix}${inpBuilderName}0`; + + const ghaCacheSignScriptPath = path.join(os.tmpdir(), `ghacache-sign-script.sh`); + core.info(`Writing GitHub Actions cache sign script to ${ghaCacheSignScriptPath}`); + await fs.writeFileSync(ghaCacheSignScriptPath, inpGHACacheSignScript, {mode: 0o700}); + + core.info(`Copying GitHub Actions cache sign script to BuildKit container ${containerName}`); + await exec.exec('docker', [ + 'cp', + ghaCacheSignScriptPath, + `${containerName}:/usr/bin/ghacache-sign-script.sh` + ]); + + core.info(`Copying cosign binary to BuildKit container ${containerName}`); + await exec.exec('docker', [ + 'cp', + cosignPath, + `${containerName}:/usr/bin/cosign` + ]); - name: Prepare id: prepare @@ -344,6 +463,9 @@ jobs: env: INPUT_PLATFORM: ${{ matrix.platform }} INPUT_LOCAL-EXPORT-DIR: ${{ env.LOCAL_EXPORT_DIR }} + INPUT_CACHE: ${{ inputs.cache }} + INPUT_CACHE-SCOPE: ${{ inputs.cache-scope }} + INPUT_CACHE-MODE: ${{ inputs.cache-mode }} INPUT_CONTEXT: ${{ inputs.context }} INPUT_FILES: ${{ inputs.files }} INPUT_OUTPUT: ${{ inputs.output }} @@ -371,6 +493,9 @@ jobs: const inpLocalExportDir = core.getInput('local-export-dir'); + const inpCache = core.getBooleanInput('cache'); + const inpCacheScope = core.getInput('cache-scope'); + const inpCacheMode = core.getInput('cache-mode'); const inpContext = core.getInput('context'); const inpFiles = Util.getInputList('files'); const inpOutput = core.getInput('output'); @@ -468,6 +593,10 @@ jobs: if (inpPlatform) { bakeOverrides.push(`*.platform=${inpPlatform}`); } + if (inpCache) { + bakeOverrides.push(`*.cache-from=type=gha,scope=${inpCacheScope || target}${platformPairSuffix}`); + bakeOverrides.push(`*.cache-to=type=gha,scope=${inpCacheScope || target}${platformPairSuffix},mode=${inpCacheMode}`); + } core.info(JSON.stringify(bakeOverrides, null, 2)); core.setOutput('overrides', bakeOverrides.join(os.EOL)); }); @@ -505,28 +634,6 @@ jobs: const imageDigest = inpMetadata[inpTarget]['containerimage.digest']; core.info(imageDigest); core.setOutput('digest', imageDigest); - - - name: Install Cosign - if: ${{ needs.prepare.outputs.sign == 'true' }} - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - env: - INPUT_COSIGN-VERSION: ${{ env.COSIGN_VERSION }} - with: - script: | - const { Cosign } = require('@docker/actions-toolkit/lib/cosign/cosign'); - const { Install } = require('@docker/actions-toolkit/lib/cosign/install'); - - const cosignInstall = new Install(); - const cosignBinPath = await cosignInstall.download({ - version: core.getInput('cosign-version'), - ghaNoCache: true, - skipState: true, - verifySignature: true - }); - await cosignInstall.install(cosignBinPath); - - const cosign = new Cosign(); - await cosign.printVersion(); - name: Signing attestation manifests id: signing-attestation-manifests diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6cb5668..d788813 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,6 +31,20 @@ on: type: string description: "List of build-time variables" required: false + cache: + type: boolean + description: "Enable cache to GitHub Actions cache backend" + required: false + default: false + cache-scope: + type: string + description: "Which scope cache object belongs to if cache enabled (defaults to target name if set)" + required: false + cache-mode: + type: string + description: "Cache layers to export if cache enabled (min or max)" + required: false + default: 'min' context: type: string description: "Context to build from in the Git working tree" @@ -131,7 +145,7 @@ on: env: BUILDX_VERSION: "v0.30.1" - BUILDKIT_IMAGE: "moby/buildkit:v0.26.2" + BUILDKIT_IMAGE: "crazymax/buildkit:6397" DOCKER_ACTIONS_TOOLKIT_MODULE: "@docker/actions-toolkit@0.71.0" COSIGN_VERSION: "v3.0.2" LOCAL_EXPORT_DIR: "/tmp/buildx-output" @@ -143,6 +157,7 @@ jobs: outputs: includes: ${{ steps.set.outputs.includes }} sign: ${{ steps.set.outputs.sign }} + privateRepo: ${{ steps.set.outputs.privateRepo }} steps: - name: Install @docker/actions-toolkit @@ -209,6 +224,11 @@ jobs: } const privateRepo = GitHub.context.payload.repository?.private ?? false; + await core.group(`Set privateRepo output`, async () => { + core.info(`privateRepo: ${privateRepo}`); + core.setOutput('privateRepo', privateRepo); + }); + await core.group(`Set includes output`, async () => { let includes = []; if (inpPlatforms.length === 0) { @@ -290,14 +310,113 @@ jobs: if: ${{ inputs.setup-qemu }} with: cache-image: false + - + name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@3cb05d89e1f492524af3d41a1c98c83bc3025124 # v3.1.0 - name: Set up Docker Buildx + id: buildx uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 with: version: ${{ env.BUILDX_VERSION }} - buildkitd-flags: --debug - driver-opts: image=${{ env.BUILDKIT_IMAGE }} cache-binary: false + buildkitd-flags: --debug + driver-opts: | + image=${{ env.BUILDKIT_IMAGE }} + env.ACTIONS_ID_TOKEN_REQUEST_TOKEN=${{ env.ACTIONS_ID_TOKEN_REQUEST_TOKEN }} + env.ACTIONS_ID_TOKEN_REQUEST_URL=${{ env.ACTIONS_ID_TOKEN_REQUEST_URL }} + buildkitd-config-inline: | + [cache] + [cache.gha] + [cache.gha.sign] + command = ["ghacache-sign-script.sh"] + [cache.gha.verify] + required = true + [cache.gha.verify.policy] + timestampThreshold = 1 + tlogThreshold = ${{ matrix.privateRepo == 'true' && '0' || '1' }} + subjectAlternativeName = "https://github.com/docker/github-builder-experimental/.github/workflows/build.yml*" + githubWorkflowRepository = "docker/github-builder-experimental" + issuer = "https://token.actions.githubusercontent.com" + runnerEnvironment = "github-hosted" + sourceRepositoryURI = "${{ github.server_url }}/${{ github.repository }}" + sourceRepositoryRef = "${{ github.event_name != 'pull_request' && github.ref || '' }}" + - + name: Install Cosign + if: ${{ needs.prepare.outputs.sign == 'true' || inputs.cache }} + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + INPUT_COSIGN-VERSION: ${{ env.COSIGN_VERSION }} + INPUT_BUILDER-NAME: ${{ steps.buildx.outputs.name }} + INPUT_GHA-CACHE-SIGN-SCRIPT: | + #!/bin/sh + set -e + + # Create temporary files + out_file=$(mktemp) + in_file=$(mktemp) + trap 'rm -f "$in_file" "$out_file"' EXIT + cat > "$in_file" + + set -x + + # Sign with cosign + cosign sign-blob \ + --yes \ + --oidc-provider github-actions \ + --new-bundle-format \ + --use-signing-config \ + --bundle "$out_file" \ + --tlog-upload=${{ matrix.privateRepo == 'false' }} \ + "$in_file" + + # Output bundle to stdout + cat "$out_file" + with: + script: | + const fs = require('fs'); + const os = require('os'); + const path = require('path'); + + const { Buildx } = require('@docker/actions-toolkit/lib/buildx/buildx'); + const { Cosign } = require('@docker/actions-toolkit/lib/cosign/cosign'); + const { Install } = require('@docker/actions-toolkit/lib/cosign/install'); + + const inpCosignVersion = core.getInput('cosign-version'); + const inpBuilderName = core.getInput('builder-name'); + const inpGHACacheSignScript = core.getInput('gha-cache-sign-script'); + + const cosignInstall = new Install(); + const cosignBinPath = await cosignInstall.download({ + version: core.getInput('cosign-version'), + ghaNoCache: true, + skipState: true, + verifySignature: true + }); + const cosignPath = await cosignInstall.install(cosignBinPath); + + const cosign = new Cosign(); + await cosign.printVersion(); + + const containerName = `${Buildx.containerNamePrefix}${inpBuilderName}0`; + + const ghaCacheSignScriptPath = path.join(os.tmpdir(), `ghacache-sign-script.sh`); + core.info(`Writing GitHub Actions cache sign script to ${ghaCacheSignScriptPath}`); + await fs.writeFileSync(ghaCacheSignScriptPath, inpGHACacheSignScript, {mode: 0o700}); + + core.info(`Copying GitHub Actions cache sign script to BuildKit container ${containerName}`); + await exec.exec('docker', [ + 'cp', + ghaCacheSignScriptPath, + `${containerName}:/usr/bin/ghacache-sign-script.sh` + ]); + + core.info(`Copying cosign binary to BuildKit container ${containerName}`); + await exec.exec('docker', [ + 'cp', + cosignPath, + `${containerName}:/usr/bin/cosign` + ]); - name: Prepare id: prepare @@ -306,6 +425,9 @@ jobs: INPUT_PLATFORM: ${{ matrix.platform }} INPUT_LOCAL-EXPORT-DIR: ${{ env.LOCAL_EXPORT_DIR }} INPUT_ANNOTATIONS: ${{ inputs.annotations }} + INPUT_CACHE: ${{ inputs.cache }} + INPUT_CACHE-SCOPE: ${{ inputs.cache-scope }} + INPUT_CACHE-MODE: ${{ inputs.cache-mode }} INPUT_LABELS: ${{ inputs.labels }} INPUT_CONTEXT: ${{ inputs.context }} INPUT_OUTPUT: ${{ inputs.output }} @@ -326,6 +448,9 @@ jobs: const inpLocalExportDir = core.getInput('local-export-dir'); const inpAnnotations = core.getMultilineInput('annotations'); + const inpCache = core.getBooleanInput('cache'); + const inpCacheScope = core.getInput('cache-scope'); + const inpCacheMode = core.getInput('cache-mode'); const inpContext = core.getInput('context'); const inpLabels = core.getMultilineInput('labels'); const inpOutput = core.getInput('output'); @@ -361,6 +486,11 @@ jobs: core.setOutput('platform', inpPlatform); } + if (inpCache) { + core.setOutput('cache-from', `type=gha,scope=${inpCacheScope || inpTarget || 'buildkit'}${platformPairSuffix}`); + core.setOutput('cache-to', `type=gha,scope=${inpCacheScope || inpTarget || 'buildkit'}${platformPairSuffix},mode=${inpCacheMode}`); + } + if (inpSetMetaAnnotations && inpMetaAnnotations.length > 0) { inpAnnotations.push(...inpMetaAnnotations); } @@ -392,6 +522,8 @@ jobs: with: annotations: ${{ steps.prepare.outputs.annotations }} build-args: ${{ inputs.build-args }} + cache-from: ${{ steps.prepare.outputs.cache-from }} + cache-to: ${{ steps.prepare.outputs.cache-to }} context: ${{ steps.prepare.outputs.context }} file: ${{ inputs.file }} labels: ${{ steps.prepare.outputs.labels }} @@ -406,28 +538,6 @@ jobs: env: BUILDKIT_MULTI_PLATFORM: 1 GIT_AUTH_TOKEN: ${{ secrets.github-token || github.token }} - - - name: Install Cosign - if: ${{ needs.prepare.outputs.sign == 'true' }} - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - env: - INPUT_COSIGN-VERSION: ${{ env.COSIGN_VERSION }} - with: - script: | - const { Cosign } = require('@docker/actions-toolkit/lib/cosign/cosign'); - const { Install } = require('@docker/actions-toolkit/lib/cosign/install'); - - const cosignInstall = new Install(); - const cosignBinPath = await cosignInstall.download({ - version: core.getInput('cosign-version'), - ghaNoCache: true, - skipState: true, - verifySignature: true - }); - await cosignInstall.install(cosignBinPath); - - const cosign = new Cosign(); - await cosign.printVersion(); - name: Signing attestation manifests id: signing-attestation-manifests diff --git a/README.md b/README.md index 5aa58ce..bd721ec 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,12 @@ toward higher levels of security and trust. requiring emulation or [custom CI logic](https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners) or self-managed runners. +* **Optimized cache warming & reuse.** + The builder can use the GitHub Actions cache backend to persist layers across + branches, PRs, and rebuilds. This significantly reduces cold-start times and + avoids repeating expensive dependency installations, even for external + contributors' pull requests. + * **Centralized build configuration.** Repositories no longer need to configure buildx drivers, tune storage, or adjust resource limits. The reusable workflows encapsulate the recommended @@ -227,6 +233,9 @@ on: | `artifact-upload` | Bool | `false` | Upload build output GitHub artifact (for `local` output) | | `annotations` | List | | List of annotations to set to the image (for `image` output) | | `build-args` | List | `auto` | List of [build-time variables](https://docs.docker.com/engine/reference/commandline/buildx_build/#build-arg). If you want to set a build-arg through an environment variable, use the `envs` input | +| `cache` | Bool | `false` | Enable [GitHub Actions cache](https://docs.docker.com/build/cache/backends/gha/) exporter | +| `cache-scope` | String | target name or `buildkit` | Which [scope cache object belongs to](https://docs.docker.com/build/cache/backends/gha/#scope) if `cache` is enabled. This is the cache blob prefix name used when pushing cache to GitHub Actions cache backend | +| `cache-mode` | String | `min` | [Cache layers to export](https://docs.docker.com/build/cache/backends/#cache-mode) if cache enabled (`min` or `max`). In `min` cache mode, only layers that are exported into the resulting image are cached, while in `max` cache mode, all layers are cached, even those of intermediate steps | | `context` | String | `.` | Context to build from in the Git working tree | | `file` | String | `{context}/Dockerfile` | Path to the Dockerfile | | `labels` | List | | List of labels for an image (for `image` output) | @@ -329,6 +338,9 @@ on: | `setup-qemu` | Bool | `false` | Runs the `setup-qemu-action` step to install QEMU static binaries | | `artifact-name` | String | `docker-github-builder-assets` | Name of the uploaded GitHub artifact (for `local` output) | | `artifact-upload` | Bool | `false` | Upload build output GitHub artifact (for `local` output) | +| `cache` | Bool | `false` | Enable [GitHub Actions cache](https://docs.docker.com/build/cache/backends/gha/) exporter | +| `cache-scope` | String | target name or `buildkit` | Which [scope cache object belongs to](https://docs.docker.com/build/cache/backends/gha/#scope) if `cache` is enabled. This is the cache blob prefix name used when pushing cache to GitHub Actions cache backend | +| `cache-mode` | String | `min` | [Cache layers to export](https://docs.docker.com/build/cache/backends/#cache-mode) if cache enabled (`min` or `max`). In `min` cache mode, only layers that are exported into the resulting image are cached, while in `max` cache mode, all layers are cached, even those of intermediate steps | | `context` | String | `.` | Context to build from in the Git working tree | | `files` | List | `{context}/docker-bake.hcl` | List of bake definition files | | `output` | String | | Build output destination (one of [`image`](https://docs.docker.com/build/exporters/image-registry/) or [`local`](https://docs.docker.com/build/exporters/local-tar/)). |