From 0ee466d381c8261cf4f478411de2d4eb1ed06b08 Mon Sep 17 00:00:00 2001 From: smrtfl Date: Thu, 15 May 2025 19:31:03 +0200 Subject: [PATCH 1/2] enable cloud storage prefix --- .../build/gradle/core/BuildCacheKey.kt | 7 +++++ .../gradle/core/FileSystemStorageService.kt | 13 +++++++++- .../gradle/core/RemoteGradleBuildCache.kt | 2 ++ .../gradle/core/FileStorageServiceTest.kt | 20 ++++++++++++++ .../gcpbuildcache/GcpBuildCacheService.kt | 26 ++++++++++++++++--- .../GcpBuildCacheServiceFactory.kt | 14 +++++----- .../s3buildcache/S3BuildCacheService.kt | 26 ++++++++++++++++--- .../S3BuildCacheServiceFactory.kt | 2 ++ 8 files changed, 95 insertions(+), 15 deletions(-) diff --git a/core/src/main/kotlin/androidx/build/gradle/core/BuildCacheKey.kt b/core/src/main/kotlin/androidx/build/gradle/core/BuildCacheKey.kt index 7fd8a29..17b97fd 100644 --- a/core/src/main/kotlin/androidx/build/gradle/core/BuildCacheKey.kt +++ b/core/src/main/kotlin/androidx/build/gradle/core/BuildCacheKey.kt @@ -31,3 +31,10 @@ fun BuildCacheKey.blobKey(): String { // a single `/`. return hashCode.replace(slashes, "/") } + +fun String.withPrefix(prefix: String?): String { + if (prefix.isNullOrBlank()) return this + + val sanitizedPrefix = prefix.trimEnd('/', '\\') + return "$sanitizedPrefix/$this" +} diff --git a/core/src/main/kotlin/androidx/build/gradle/core/FileSystemStorageService.kt b/core/src/main/kotlin/androidx/build/gradle/core/FileSystemStorageService.kt index 2704596..9e5de15 100644 --- a/core/src/main/kotlin/androidx/build/gradle/core/FileSystemStorageService.kt +++ b/core/src/main/kotlin/androidx/build/gradle/core/FileSystemStorageService.kt @@ -26,11 +26,20 @@ import java.nio.file.Files */ class FileSystemStorageService( override val bucketName: String, + private val prefix: String?, override val isPush: Boolean, override val isEnabled: Boolean ) : StorageService { - private val location = Files.createTempDirectory("tmp$bucketName").toFile() + private val location: File = createTempDirectory() + + private fun createTempDirectory(): File { + val baseDir = prefix?.takeIf { it.isNotBlank() }?.let { + File(it).apply { if (!exists()) mkdirs() } + } ?: File(System.getProperty(JAVA_IO_TMPDIR)) + + return Files.createTempDirectory(baseDir.toPath(), "tmp$bucketName").toFile() + } override fun load(cacheKey: String): InputStream? { if (!isEnabled) { @@ -84,6 +93,8 @@ class FileSystemStorageService( } companion object { + private const val JAVA_IO_TMPDIR = "java.io.tmpdir" + private fun File.deleteRecursively() { val files = listFiles() for (file in files) { diff --git a/core/src/main/kotlin/androidx/build/gradle/core/RemoteGradleBuildCache.kt b/core/src/main/kotlin/androidx/build/gradle/core/RemoteGradleBuildCache.kt index c5cd198..5170321 100644 --- a/core/src/main/kotlin/androidx/build/gradle/core/RemoteGradleBuildCache.kt +++ b/core/src/main/kotlin/androidx/build/gradle/core/RemoteGradleBuildCache.kt @@ -30,6 +30,8 @@ abstract class RemoteGradleBuildCache : AbstractBuildCache() { */ lateinit var bucketName: String + lateinit var prefix: String + /** * The type of credentials to use to connect to authenticate to your project instance. */ diff --git a/core/src/test/kotlin/androidx/build/gradle/core/FileStorageServiceTest.kt b/core/src/test/kotlin/androidx/build/gradle/core/FileStorageServiceTest.kt index 63e0001..631a741 100644 --- a/core/src/test/kotlin/androidx/build/gradle/core/FileStorageServiceTest.kt +++ b/core/src/test/kotlin/androidx/build/gradle/core/FileStorageServiceTest.kt @@ -24,6 +24,23 @@ class FileStorageServiceTest { fun testStoreBlob() { val storageService = FileSystemStorageService( bucketName = BUCKET_NAME, + prefix = null, + isPush = true, + isEnabled = true + ) + storageService.use { + val cacheKey = "test-store.txt" + val contents = "The quick brown fox jumped over the lazy dog" + val result = storageService.store(cacheKey, contents.toByteArray(Charsets.UTF_8)) + assert(result) + } + } + + @Test + fun testStoreBlob_withPrefix() { + val storageService = FileSystemStorageService( + bucketName = BUCKET_NAME, + prefix = "test-prefix", isPush = true, isEnabled = true ) @@ -39,6 +56,7 @@ class FileStorageServiceTest { fun testLoadBlob() { val storageService = FileSystemStorageService( bucketName = BUCKET_NAME, + prefix = null, isPush = true, isEnabled = true ) @@ -57,6 +75,7 @@ class FileStorageServiceTest { fun testStoreBlob_noPushSupport() { val storageService = FileSystemStorageService( bucketName = BUCKET_NAME, + prefix = null, isPush = false, isEnabled = true ) @@ -72,6 +91,7 @@ class FileStorageServiceTest { fun testStoreBlob_disabled() { val storageService = FileSystemStorageService( bucketName = BUCKET_NAME, + prefix = null, isPush = true, isEnabled = false ) diff --git a/gcpbuildcache/src/main/kotlin/androidx/build/gradle/gcpbuildcache/GcpBuildCacheService.kt b/gcpbuildcache/src/main/kotlin/androidx/build/gradle/gcpbuildcache/GcpBuildCacheService.kt index 61d058c..a0f6168 100644 --- a/gcpbuildcache/src/main/kotlin/androidx/build/gradle/gcpbuildcache/GcpBuildCacheService.kt +++ b/gcpbuildcache/src/main/kotlin/androidx/build/gradle/gcpbuildcache/GcpBuildCacheService.kt @@ -19,6 +19,7 @@ package androidx.build.gradle.gcpbuildcache import androidx.build.gradle.core.FileSystemStorageService import androidx.build.gradle.core.blobKey +import androidx.build.gradle.core.withPrefix import org.gradle.api.logging.Logging import org.gradle.caching.BuildCacheEntryReader import org.gradle.caching.BuildCacheEntryWriter @@ -37,6 +38,7 @@ import java.io.ByteArrayOutputStream internal class GcpBuildCacheService( private val projectId: String, private val bucketName: String, + private val prefix: String?, gcpCredentials: GcpCredentials, messageOnAuthenticationFailure: String, isPush: Boolean, @@ -46,9 +48,21 @@ internal class GcpBuildCacheService( private val storageService = if (inTestMode) { // Use an implementation backed by the File System when in test mode. - FileSystemStorageService(bucketName, isPush, isEnabled) + FileSystemStorageService( + bucketName = bucketName, + prefix = prefix, + isPush = isPush, + isEnabled = isEnabled, + ) } else { - GcpStorageService(projectId, bucketName, gcpCredentials, messageOnAuthenticationFailure, isPush, isEnabled) + GcpStorageService( + projectId = projectId, + bucketName = bucketName, + gcpCredentials = gcpCredentials, + messageOnAuthenticationFailure = messageOnAuthenticationFailure, + isPush = isPush, + isEnabled = isEnabled + ) } override fun close() { @@ -57,7 +71,9 @@ internal class GcpBuildCacheService( override fun load(key: BuildCacheKey, reader: BuildCacheEntryReader): Boolean { logger.info("Loading ${key.blobKey()}") - val cacheKey = key.blobKey() + val cacheKey = key + .blobKey() + .apply { if (prefix != null) withPrefix(prefix) } val input = storageService.load(cacheKey) ?: return false reader.readFrom(input) return true @@ -66,7 +82,9 @@ internal class GcpBuildCacheService( override fun store(key: BuildCacheKey, writer: BuildCacheEntryWriter) { if (writer.size == 0L) return // do not store empty entries into the cache logger.info("Storing ${key.blobKey()}") - val cacheKey = key.blobKey() + val cacheKey = key + .blobKey() + .apply { if (prefix != null) withPrefix(prefix) } val output = ByteArrayOutputStream() output.use { writer.writeTo(output) diff --git a/gcpbuildcache/src/main/kotlin/androidx/build/gradle/gcpbuildcache/GcpBuildCacheServiceFactory.kt b/gcpbuildcache/src/main/kotlin/androidx/build/gradle/gcpbuildcache/GcpBuildCacheServiceFactory.kt index 846367e..eb45f0f 100644 --- a/gcpbuildcache/src/main/kotlin/androidx/build/gradle/gcpbuildcache/GcpBuildCacheServiceFactory.kt +++ b/gcpbuildcache/src/main/kotlin/androidx/build/gradle/gcpbuildcache/GcpBuildCacheServiceFactory.kt @@ -32,6 +32,7 @@ class GcpBuildCacheServiceFactory : BuildCacheServiceFactory { .type("GCP-backed") .config("projectId", buildCache.projectId) .config("bucketName", buildCache.bucketName) + .config("prefix", buildCache.prefix) .config("isPushSupported", "${buildCache.isPush}") .config("isEnabled", "${buildCache.isEnabled}") .config( @@ -40,12 +41,13 @@ class GcpBuildCacheServiceFactory : BuildCacheServiceFactory { ) val service = GcpBuildCacheService( - buildCache.projectId, - buildCache.bucketName, - buildCache.credentials, - buildCache.messageOnAuthenticationFailure, - buildCache.isPush, - buildCache.isEnabled + projectId = buildCache.projectId, + bucketName = buildCache.bucketName, + prefix = buildCache.prefix, + gcpCredentials = buildCache.credentials, + messageOnAuthenticationFailure = buildCache.messageOnAuthenticationFailure, + isPush = buildCache.isPush, + isEnabled = buildCache.isEnabled ) service.validateConfiguration() return service diff --git a/s3buildcache/src/main/kotlin/androidx/build/gradle/s3buildcache/S3BuildCacheService.kt b/s3buildcache/src/main/kotlin/androidx/build/gradle/s3buildcache/S3BuildCacheService.kt index be2b814..e1af9cc 100644 --- a/s3buildcache/src/main/kotlin/androidx/build/gradle/s3buildcache/S3BuildCacheService.kt +++ b/s3buildcache/src/main/kotlin/androidx/build/gradle/s3buildcache/S3BuildCacheService.kt @@ -19,6 +19,7 @@ package androidx.build.gradle.s3buildcache import androidx.build.gradle.core.FileSystemStorageService import androidx.build.gradle.core.blobKey +import androidx.build.gradle.core.withPrefix import org.gradle.api.logging.Logging import org.gradle.caching.BuildCacheEntryReader import org.gradle.caching.BuildCacheEntryWriter @@ -42,6 +43,7 @@ class S3BuildCacheService( credentials: S3Credentials, region: String, bucketName: String, + private val prefix: String?, isPush: Boolean, isEnabled: Boolean, reducedRedundancy: Boolean, @@ -52,14 +54,28 @@ class S3BuildCacheService( clientOptions(credentials(credentials), region) } private val storageService = if (inTestMode) { - FileSystemStorageService(bucketName, isPush, isEnabled) + FileSystemStorageService( + bucketName = bucketName, + prefix = prefix, + isPush = isPush, + isEnabled = isEnabled, + ) } else { - S3StorageService(bucketName, isPush, isEnabled, client, region, reducedRedundancy) + S3StorageService( + bucketName = bucketName, + isPush = isPush, + isEnabled = isEnabled, + client = client, + region = region, + reducedRedundancy = reducedRedundancy + ) } override fun load(key: BuildCacheKey, reader: BuildCacheEntryReader): Boolean { logger.info("Loading ${key.blobKey()}") - val cacheKey = key.blobKey() + val cacheKey = key + .blobKey() + .apply { if (prefix != null) withPrefix(prefix) } val input = storageService.load(cacheKey) ?: return false reader.readFrom(input) return true @@ -67,8 +83,10 @@ class S3BuildCacheService( override fun store(key: BuildCacheKey, writer: BuildCacheEntryWriter) { if (writer.size == 0L) return // do not store empty entries into the cache + val cacheKey = key + .blobKey() + .apply { if (prefix != null) withPrefix(prefix) } logger.info("Storing ${key.blobKey()}") - val cacheKey = key.blobKey() val output = ByteArrayOutputStream() output.use { writer.writeTo(output) diff --git a/s3buildcache/src/main/kotlin/androidx/build/gradle/s3buildcache/S3BuildCacheServiceFactory.kt b/s3buildcache/src/main/kotlin/androidx/build/gradle/s3buildcache/S3BuildCacheServiceFactory.kt index 2685a71..7027b67 100644 --- a/s3buildcache/src/main/kotlin/androidx/build/gradle/s3buildcache/S3BuildCacheServiceFactory.kt +++ b/s3buildcache/src/main/kotlin/androidx/build/gradle/s3buildcache/S3BuildCacheServiceFactory.kt @@ -33,6 +33,7 @@ class S3BuildCacheServiceFactory : BuildCacheServiceFactory { .type("AWS-S3-backed") .config("region", buildCache.region) .config("bucketName", buildCache.bucketName) + .config("prefix", buildCache.prefix) .config("reducedRedundancy", "${buildCache.reducedRedundancy}") .config("isPushSupported", "${buildCache.isPush}") .config("isEnabled", "${buildCache.isEnabled}") @@ -41,6 +42,7 @@ class S3BuildCacheServiceFactory : BuildCacheServiceFactory { val service = S3BuildCacheService( region = buildCache.region, bucketName = buildCache.bucketName, + prefix = buildCache.prefix, isPush = buildCache.isPush, isEnabled = buildCache.isEnabled, reducedRedundancy = buildCache.reducedRedundancy, From 7d6970e1651e129cdfa8acd125f4dceb6d17026f Mon Sep 17 00:00:00 2001 From: smrtfl Date: Fri, 16 May 2025 12:05:41 +0200 Subject: [PATCH 2/2] document prefix variable --- .../androidx/build/gradle/core/RemoteGradleBuildCache.kt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/kotlin/androidx/build/gradle/core/RemoteGradleBuildCache.kt b/core/src/main/kotlin/androidx/build/gradle/core/RemoteGradleBuildCache.kt index 5170321..a3c5cb5 100644 --- a/core/src/main/kotlin/androidx/build/gradle/core/RemoteGradleBuildCache.kt +++ b/core/src/main/kotlin/androidx/build/gradle/core/RemoteGradleBuildCache.kt @@ -30,6 +30,11 @@ abstract class RemoteGradleBuildCache : AbstractBuildCache() { */ lateinit var bucketName: String + /** + * The prefix to use when storing cache entries in the bucket. + * It becomes new root for all cache entries. + * If not specified, the cache entries will be stored at the root of the bucket. + */ lateinit var prefix: String /**