diff --git a/README.md b/README.md index 0b6ae60..bfc03e1 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ steps: - label: "🐳 Build with ACR cache" command: "echo 'Building with cache'" plugins: - - docker-cache#v1.0.0: + - docker-cache#v1.1.0: provider: acr image: my-app acr: @@ -298,6 +298,43 @@ Environment variable name for exporting the final image reference. Enable verbose logging. +#### `fallback-tag` (string, default: `latest`) + +Tag used for layer caching "fallback" when no exact cache match exists. The plugin looks for an image with this tag to use for Docker layer caching, improving build performance even without an exact cache hit. Useful for registries with immutable tags where `:latest` cannot be overwritten. + +Using a static tag: + +```yaml +steps: + - plugins: + - docker-cache#v1.1.0: + provider: ecr + image: my-app + fallback-tag: cache-main +``` + +Using a tag containing the commit SHA: + +```yaml +steps: + - plugins: + - docker-cache#v1.1.0: + provider: acr + image: my-app + fallback-tag: cache-${BUILDKITE_COMMIT:0:7} +``` + +Using the build number as part of the tag: + +```yaml +steps: + - plugins: + - docker-cache#v1.1.0: + provider: gar + image: my-app + fallback-tag: build-${BUILDKITE_BUILD_NUMBER} +``` + #### `tag` (string) Custom tag for the cached image. If not provided, generated from git commit or pipeline context. diff --git a/hooks/pre-command b/hooks/pre-command index 73cde75..c3a7233 100755 --- a/hooks/pre-command +++ b/hooks/pre-command @@ -51,7 +51,7 @@ main() { # Check if we already have the required tags locally local target_with_key="${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:${BUILDKITE_PLUGIN_DOCKER_CACHE_KEY}" - local target_latest="${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:latest" + local target_latest="${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:${BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG}" local needs_key_tag=true local needs_latest_tag=true @@ -145,7 +145,7 @@ main() { log_success "Image built successfully" # Tag as latest - tag_image "${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:${BUILDKITE_PLUGIN_DOCKER_CACHE_KEY}" "${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:latest" + tag_image "${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:${BUILDKITE_PLUGIN_DOCKER_CACHE_KEY}" "${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:${BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG}" # Push cache echo "--- :docker: Pushing cache" diff --git a/lib/plugin.bash b/lib/plugin.bash index 4d4e207..6f6964f 100644 --- a/lib/plugin.bash +++ b/lib/plugin.bash @@ -5,6 +5,7 @@ plugin_read_config() { export BUILDKITE_PLUGIN_DOCKER_CACHE_SAVE="${BUILDKITE_PLUGIN_DOCKER_CACHE_SAVE:-true}" export BUILDKITE_PLUGIN_DOCKER_CACHE_RESTORE="${BUILDKITE_PLUGIN_DOCKER_CACHE_RESTORE:-true}" export BUILDKITE_PLUGIN_DOCKER_CACHE_TAG="${BUILDKITE_PLUGIN_DOCKER_CACHE_TAG:-cache}" + export BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG="${BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG:-latest}" export BUILDKITE_PLUGIN_DOCKER_CACHE_VERBOSE="${BUILDKITE_PLUGIN_DOCKER_CACHE_VERBOSE:-false}" export BUILDKITE_PLUGIN_DOCKER_CACHE_STRATEGY="${BUILDKITE_PLUGIN_DOCKER_CACHE_STRATEGY:-hybrid}" diff --git a/lib/providers/acr.bash b/lib/providers/acr.bash index f6d0c7d..3d109d3 100644 --- a/lib/providers/acr.bash +++ b/lib/providers/acr.bash @@ -111,8 +111,8 @@ restore_acr_cache() { else log_info "No cache found for key ${BUILDKITE_PLUGIN_DOCKER_CACHE_KEY} - will build from scratch" # Try to find any existing cache image for layer caching by checking for latest tag - local repository="${BUILDKITE_PLUGIN_DOCKER_CACHE_ACR_REPOSITORY:-${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}}" - local fallback_cache_image="${BUILDKITE_PLUGIN_DOCKER_CACHE_ACR_REGISTRY_URL}/${repository}/${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:latest" + local fallback_cache_image + fallback_cache_image=$(build_cache_image_name "${BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG}") if image_exists_in_registry "$fallback_cache_image"; then log_info "Using latest cache for layer caching: $fallback_cache_image" export BUILDKITE_PLUGIN_DOCKER_CACHE_FROM="$fallback_cache_image" @@ -155,8 +155,8 @@ save_acr_cache() { log_info "Ensuring ACR repository exists (auto-created if needed)" # Build cache image name with latest tag for layer caching - local repository="${BUILDKITE_PLUGIN_DOCKER_CACHE_ACR_REPOSITORY:-${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}}" - local latest_image="${BUILDKITE_PLUGIN_DOCKER_CACHE_ACR_REGISTRY_URL}/${repository}/${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:latest" + local latest_image + latest_image=$(build_cache_image_name "${BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG}") if tag_image "${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}" "$cache_image"; then if push_image "$cache_image"; then diff --git a/lib/providers/artifactory.bash b/lib/providers/artifactory.bash index b38d086..ab4d3a3 100644 --- a/lib/providers/artifactory.bash +++ b/lib/providers/artifactory.bash @@ -116,8 +116,8 @@ restore_artifactory_cache() { else log_info "No cache found for key ${BUILDKITE_PLUGIN_DOCKER_CACHE_KEY} - will build from scratch" # Try to find any existing cache image for layer caching by checking for latest tag - local repository="${BUILDKITE_PLUGIN_DOCKER_CACHE_ARTIFACTORY_REPOSITORY:-${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}}" - local fallback_cache_image="${BUILDKITE_PLUGIN_DOCKER_CACHE_ARTIFACTORY_REGISTRY_URL}/${repository}/${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:latest" + local fallback_cache_image + fallback_cache_image=$(build_cache_image_name "${BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG}") if image_exists_in_registry "$fallback_cache_image"; then log_info "Using latest cache for layer caching: $fallback_cache_image" export BUILDKITE_PLUGIN_DOCKER_CACHE_FROM="$fallback_cache_image" @@ -157,8 +157,8 @@ save_artifactory_cache() { fi # Build cache image name with latest tag for layer caching - local repository="${BUILDKITE_PLUGIN_DOCKER_CACHE_ARTIFACTORY_REPOSITORY:-${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}}" - local latest_image="${BUILDKITE_PLUGIN_DOCKER_CACHE_ARTIFACTORY_REGISTRY_URL}/${repository}/${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:latest" + local latest_image + latest_image=$(build_cache_image_name "${BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG}") if tag_image "${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}" "$cache_image"; then if push_image "$cache_image"; then diff --git a/lib/providers/buildkite.bash b/lib/providers/buildkite.bash index c449248..8394abc 100644 --- a/lib/providers/buildkite.bash +++ b/lib/providers/buildkite.bash @@ -161,7 +161,8 @@ restore_buildkite_cache() { else log_info "No cache found for key ${BUILDKITE_PLUGIN_DOCKER_CACHE_KEY} - will build from scratch" # Try to find any existing cache image for layer caching by checking for latest tag - local fallback_cache_image="${BUILDKITE_PLUGIN_DOCKER_CACHE_BUILDKITE_REGISTRY_URL}/${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:latest" + local fallback_cache_image + fallback_cache_image=$(build_cache_image_name "${BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG}") if image_exists_in_registry "$fallback_cache_image"; then log_info "Using latest cache for layer caching: $fallback_cache_image" export BUILDKITE_PLUGIN_DOCKER_CACHE_FROM="$fallback_cache_image" @@ -201,7 +202,8 @@ save_buildkite_cache() { fi # Build cache image name with latest tag for layer caching - local latest_image="${BUILDKITE_PLUGIN_DOCKER_CACHE_BUILDKITE_REGISTRY_URL}/${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:latest" + local latest_image + latest_image=$(build_cache_image_name "${BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG}") if tag_image "${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}" "$cache_image"; then if push_image "$cache_image"; then diff --git a/lib/providers/ecr.bash b/lib/providers/ecr.bash index 2a898c2..0f95e4a 100644 --- a/lib/providers/ecr.bash +++ b/lib/providers/ecr.bash @@ -99,7 +99,8 @@ restore_ecr_cache() { else log_info "No cache found for key ${BUILDKITE_PLUGIN_DOCKER_CACHE_KEY} - will build from scratch" # Try to find any existing cache image for layer caching by checking for latest tag - local fallback_cache_image="${BUILDKITE_PLUGIN_DOCKER_CACHE_ECR_REGISTRY_URL}/${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:latest" + local fallback_cache_image + fallback_cache_image=$(build_cache_image_name "${BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG}") if image_exists_in_registry "$fallback_cache_image"; then log_info "Using latest cache for layer caching: $fallback_cache_image" export BUILDKITE_PLUGIN_DOCKER_CACHE_FROM="$fallback_cache_image" @@ -151,7 +152,8 @@ save_ecr_cache() { fi # Build cache image name with latest tag for layer caching - local latest_image="${BUILDKITE_PLUGIN_DOCKER_CACHE_ECR_REGISTRY_URL}/${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:latest" + local latest_image + latest_image=$(build_cache_image_name "${BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG}") if tag_image "${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}" "$cache_image"; then if push_image "$cache_image"; then diff --git a/lib/providers/gar.bash b/lib/providers/gar.bash index 358ffc0..7b70686 100644 --- a/lib/providers/gar.bash +++ b/lib/providers/gar.bash @@ -91,7 +91,7 @@ restore_gar_cache() { log_info "No cache found for key ${BUILDKITE_PLUGIN_DOCKER_CACHE_KEY} - will build from scratch" # Try to find any existing cache image for layer caching by checking for latest tag local fallback_cache_image - fallback_cache_image=$(build_cache_image_name | sed 's/:cache-.*/:latest/') + fallback_cache_image=$(build_cache_image_name "${BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG}") if image_exists_in_registry "$fallback_cache_image"; then log_info "Using latest cache for layer caching: $fallback_cache_image" export BUILDKITE_PLUGIN_DOCKER_CACHE_FROM="$fallback_cache_image" @@ -148,7 +148,7 @@ save_gar_cache() { # Build latest image name for layer caching local latest_image - latest_image=$(build_cache_image_name | sed 's/:cache-.*/:latest/') + latest_image=$(build_cache_image_name "${BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG}") if tag_image "${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}" "$cache_image"; then if push_image "$cache_image"; then diff --git a/lib/shared.bash b/lib/shared.bash index ebd9a2c..48b0482 100644 --- a/lib/shared.bash +++ b/lib/shared.bash @@ -102,28 +102,32 @@ check_dependencies() { } build_cache_image_name() { + # Optional parameter: custom tag suffix to use instead of cache-KEY pattern + # Usage: build_cache_image_name "${BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG}" + local tag_suffix="${1:-${BUILDKITE_PLUGIN_DOCKER_CACHE_TAG:-cache}-${BUILDKITE_PLUGIN_DOCKER_CACHE_KEY}}" + case "${BUILDKITE_PLUGIN_DOCKER_CACHE_PROVIDER}" in ecr) - echo "${BUILDKITE_PLUGIN_DOCKER_CACHE_ECR_REGISTRY_URL}/${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:${BUILDKITE_PLUGIN_DOCKER_CACHE_TAG:-cache}-${BUILDKITE_PLUGIN_DOCKER_CACHE_KEY}" + echo "${BUILDKITE_PLUGIN_DOCKER_CACHE_ECR_REGISTRY_URL}/${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:${tag_suffix}" ;; gar) if [[ "${BUILDKITE_PLUGIN_DOCKER_CACHE_GAR_REGION:-us}" =~ \.pkg\.dev$ ]]; then # Google Artifact Registry host already specified - echo "${BUILDKITE_PLUGIN_DOCKER_CACHE_GAR_REGION:-us}/${BUILDKITE_PLUGIN_DOCKER_CACHE_GAR_PROJECT}/${BUILDKITE_PLUGIN_DOCKER_CACHE_GAR_REPOSITORY:-${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}}/${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:${BUILDKITE_PLUGIN_DOCKER_CACHE_TAG:-cache}-${BUILDKITE_PLUGIN_DOCKER_CACHE_KEY}" + echo "${BUILDKITE_PLUGIN_DOCKER_CACHE_GAR_REGION:-us}/${BUILDKITE_PLUGIN_DOCKER_CACHE_GAR_PROJECT}/${BUILDKITE_PLUGIN_DOCKER_CACHE_GAR_REPOSITORY:-${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}}/${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:${tag_suffix}" else - echo "${BUILDKITE_PLUGIN_DOCKER_CACHE_GAR_REGION:-us}.gar.io/${BUILDKITE_PLUGIN_DOCKER_CACHE_GAR_PROJECT}/${BUILDKITE_PLUGIN_DOCKER_CACHE_GAR_REPOSITORY:-${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}}/${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:${BUILDKITE_PLUGIN_DOCKER_CACHE_TAG:-cache}-${BUILDKITE_PLUGIN_DOCKER_CACHE_KEY}" + echo "${BUILDKITE_PLUGIN_DOCKER_CACHE_GAR_REGION:-us}.gcr.io/${BUILDKITE_PLUGIN_DOCKER_CACHE_GAR_PROJECT}/${BUILDKITE_PLUGIN_DOCKER_CACHE_GAR_REPOSITORY:-${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}}/${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:${tag_suffix}" fi ;; buildkite) - echo "${BUILDKITE_PLUGIN_DOCKER_CACHE_BUILDKITE_REGISTRY_URL}/${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:${BUILDKITE_PLUGIN_DOCKER_CACHE_TAG:-cache}-${BUILDKITE_PLUGIN_DOCKER_CACHE_KEY}" + echo "${BUILDKITE_PLUGIN_DOCKER_CACHE_BUILDKITE_REGISTRY_URL}/${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:${tag_suffix}" ;; artifactory) local repository="${BUILDKITE_PLUGIN_DOCKER_CACHE_ARTIFACTORY_REPOSITORY:-${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}}" - echo "${BUILDKITE_PLUGIN_DOCKER_CACHE_ARTIFACTORY_REGISTRY_URL}/${repository}/${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:${BUILDKITE_PLUGIN_DOCKER_CACHE_TAG:-cache}-${BUILDKITE_PLUGIN_DOCKER_CACHE_KEY}" + echo "${BUILDKITE_PLUGIN_DOCKER_CACHE_ARTIFACTORY_REGISTRY_URL}/${repository}/${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:${tag_suffix}" ;; acr) local repository="${BUILDKITE_PLUGIN_DOCKER_CACHE_ACR_REPOSITORY:-${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}}" - echo "${BUILDKITE_PLUGIN_DOCKER_CACHE_ACR_REGISTRY_URL}/${repository}/${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:${BUILDKITE_PLUGIN_DOCKER_CACHE_TAG:-cache}-${BUILDKITE_PLUGIN_DOCKER_CACHE_KEY}" + echo "${BUILDKITE_PLUGIN_DOCKER_CACHE_ACR_REGISTRY_URL}/${repository}/${BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE}:${tag_suffix}" ;; *) log_error "Unknown provider: ${BUILDKITE_PLUGIN_DOCKER_CACHE_PROVIDER}" diff --git a/plugin.yml b/plugin.yml index ba395ad..307030e 100644 --- a/plugin.yml +++ b/plugin.yml @@ -158,6 +158,10 @@ configuration: type: boolean description: Enable verbose logging default: false + fallback-tag: + type: string + description: Tag used for layer caching fallback when no exact cache match exists + default: "latest" required: - provider - image diff --git a/tests/environment.bats b/tests/environment.bats index 6a95d9d..89322bf 100644 --- a/tests/environment.bats +++ b/tests/environment.bats @@ -722,3 +722,37 @@ setup() { assert_success assert_output --partial 'Setting up Docker cache environment' } + +@test "Uses default fallback-tag when not specified" { + # Load the plugin library for testing + source "$PWD/lib/plugin.bash" + + unset BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG + plugin_read_config + + assert [ "$BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG" = "latest" ] +} + +@test "Accepts custom fallback-tag values" { + # Load the plugin library for testing + source "$PWD/lib/plugin.bash" + + for tag in "cache-main" "build-123" "cache-abc123" "stable" "v1.0.0"; do + export BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG="$tag" + plugin_read_config + + assert [ "$BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG" = "$tag" ] + done +} + +@test "Accepts fallback-tag with environment variable expansion" { + # Load the plugin library for testing + source "$PWD/lib/plugin.bash" + + export BUILDKITE_COMMIT="abc123def456" + export BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG='cache-${BUILDKITE_COMMIT:0:7}' + plugin_read_config + + # Variable expansion happens when the tag is used, not when read + assert [ "$BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG" = 'cache-${BUILDKITE_COMMIT:0:7}' ] +} diff --git a/tests/pre-command.bats b/tests/pre-command.bats index 5572cf8..8ee3018 100644 --- a/tests/pre-command.bats +++ b/tests/pre-command.bats @@ -528,3 +528,70 @@ setup() { assert_output --partial 'Docker cache build' assert_output --partial 'Complete cache hit' } + +@test "Custom fallback-tag is used instead of latest" { + export BUILDKITE_PLUGIN_DOCKER_CACHE_PROVIDER='ecr' + export BUILDKITE_PLUGIN_DOCKER_CACHE_IMAGE='test-app' + export BUILDKITE_PLUGIN_DOCKER_CACHE_FALLBACK_TAG='cache-main' + export BUILDKITE_PLUGIN_DOCKER_CACHE_ECR_REGION='us-east-1' + export BUILDKITE_PLUGIN_DOCKER_CACHE_ECR_ACCOUNT_ID='123456789012' + + stub aws \ + "ecr get-login-password --region us-east-1 : echo password" \ + "ecr describe-repositories --repository-names test-app --region us-east-1 : exit 1" \ + "ecr create-repository --repository-name test-app --region us-east-1 : echo '{\"repository\":{\"repositoryUri\":\"123456789012.dkr.ecr.us-east-1.amazonaws.com/test-app\"}}'" + + # Create temp file to track tag usage + local tag_marker="${BATS_TEST_TMPDIR}/tag_marker" + + function docker() { + case "$1" in + login) + if [[ "$2" == "--username" ]]; then + cat > /dev/null + echo "Login Succeeded" + return 0 + fi + ;; + manifest) + return 1 # Simulate cache miss + ;; + build) + echo "Successfully built abc123" + return 0 + ;; + tag) + # Check if cache-main tag is used instead of latest + if [[ "$3" =~ :cache-main$ ]]; then + echo "true" > "$tag_marker" + fi + return 0 + ;; + push) + echo "Successfully pushed image" + return 0 + ;; + image) + if [[ "$2" == "inspect" ]]; then + return 0 + fi + ;; + *) + command docker "$@" + ;; + esac + } + export -f docker + export tag_marker + + run "$PWD"/hooks/pre-command + + assert_success + assert_output --partial 'Docker cache build' + + # Verify that cache-main tag was used + assert [ -f "$tag_marker" ] + assert [ "$(cat "$tag_marker")" = "true" ] + + unstub aws +}