diff --git a/.github/workflows/gcs-integration.yml b/.github/workflows/gcs-integration.yml new file mode 100644 index 000000000..a7a856eec --- /dev/null +++ b/.github/workflows/gcs-integration.yml @@ -0,0 +1,43 @@ +name: GCS BlobStore Integration + +on: + push: + tags: + - "**" + pull_request: + paths: + - ".github/workflows/gcs-integration.yml" + - "pom.xml" + - "geowebcache/pom.xml" + - "geowebcache/core/**" + - "geowebcache/gcsblob/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + fake-gcs-server: + name: Fake GCS Server container + runs-on: ubuntu-latest + strategy: + matrix: + java-version: [ 17, 21 ] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.java-version }} + cache: 'maven' + + - name: Tests against Fake GCS Server TestContainers + run: | + mvn verify -f geowebcache/pom.xml -pl :gcs-blob -am \ + -Ponline \ + -DskipTests=true \ + -DskipITs=false -B -ntp + + - name: Remove SNAPSHOT jars from repository + run: | + find .m2/repository -name "*SNAPSHOT*" -type d | xargs rm -rf {} diff --git a/documentation/en/user/source/configuration/storage.rst b/documentation/en/user/source/configuration/storage.rst index 3c2d9fa8f..9a5df27af 100644 --- a/documentation/en/user/source/configuration/storage.rst +++ b/documentation/en/user/source/configuration/storage.rst @@ -11,11 +11,13 @@ Storage Cache ----- -Starting with version 1.8.0, there are two types of persistent storage mechanisms for tiles: +Starting with version 1.8.0, GeoWebCache supports multiple persistent storage mechanisms for tiles: -* File blob store: stores tiles in a directory structure consisting of various image files organized by layer and zoom level. -* S3 blob store: stores tiles in an `Amazon Simple Storage Service `_ bucket, as individual "objects" following a +* File blob store: stores tiles in a directory structure consisting of various image files organized by layer and zoom level. +* S3 blob store: stores tiles in an `Amazon Simple Storage Service `_ bucket, as individual "objects" following a `TMS `_-like key structure. +* Google Cloud Storage blob store: stores tiles in a GCS bucket using the same TMS-like structure as S3. +* Azure blob store, MBTiles blob store, Swift blob store: additional storage backends described below. Zero or more blobstores can be configured in the configuration file to store tiles at different locations and on different storage back-ends. One of the configured blobstores will be the **default** one. Meaning that it will be used to store the tiles of every layer whose configuration @@ -287,7 +289,54 @@ GeoServer ``topp:states`` sample layer on a fictitious ``my-geowebcache-bucket`` zoom: 2 }) }); - + + +Google Cloud Storage (GCS) Blob Store ++++++++++++++++++++++++++++++++++++++ + +This blob store allows to configure a cache for layers on a Google Cloud Storage bucket with the same TMS-like key structure as S3: + + [prefix]///////. + +Configuration example: + +.. code-block:: xml + + + myGcsCache + true + my-gwc-bucket + test-cache + my-gcp-project + true + + +Properties: + +* **bucket**: Mandatory. The name of the GCS bucket where to store tiles. +* **prefix**: Optional. A prefix path to use as the "root folder" to store tiles at. +* **projectId**: Optional. The GCP project ID. Can be omitted if using service account credentials that already specify the project. +* **quotaProjectId**: Optional. Project to bill for quota when using requester-pays buckets. +* **endpointUrl**: Optional. Custom endpoint URL for use with GCS emulators or compatible services. +* **useDefaultCredentialsChain**: Optional. Set to ``true`` to use Application Default Credentials. This will look for credentials in the following order: environment variable GOOGLE_APPLICATION_CREDENTIALS pointing to a service account key file, GCE/GKE metadata service, or gcloud CLI credentials. +* **apiKey**: Optional. API key for authentication. If both apiKey and useDefaultCredentialsChain are provided, apiKey takes precedence. + +**Note**: Like S3, all configuration properties support environment variable expansion using the ``${VARIABLE_NAME}`` syntax: + +.. code-block:: xml + + ${GCS_BUCKET} + ${GCS_PROJECT_ID} + +Authentication options: + +* **Application Default Credentials** (recommended): Set ``useDefaultCredentialsChain`` to ``true``. This works automatically on GCE/GKE and when GOOGLE_APPLICATION_CREDENTIALS points to a service account key. +* **API Key**: Set the ``apiKey`` property. Less secure, mainly for testing. +* **No auth**: For use with emulators only. Leave both auth options unset. + +Implementation notes: + +Delete operations run asynchronously in a background thread pool. When deleting tile ranges or layers, tiles are removed in batches using the GCS batch API for efficiency. The thread pool is sized based on available processors and shuts down gracefully on blob store destruction. Microsoft Azure Blob Store +++++++++++++++++++++++++++++++++++++++++++++ @@ -861,4 +910,4 @@ Additional Information: * The package makes use of the open source multi-cloud toolkit `jclouds `_ * Jclouds documentation for `getting started with Openstack `_ -* Jclouds documentation for `OpenStack Keystone V3 Support `_ used in config \ No newline at end of file +* Jclouds documentation for `OpenStack Keystone V3 Support `_ used in config diff --git a/geowebcache/gcsblob/README.md b/geowebcache/gcsblob/README.md new file mode 100644 index 000000000..b897a0902 --- /dev/null +++ b/geowebcache/gcsblob/README.md @@ -0,0 +1,68 @@ +# GCS Blob Store + +BlobStore implementation for Google Cloud Storage. + +## Overview + +This module provides a `BlobStore` that stores tiles in a GCS bucket. Tiles are organized using the standard TMS key structure: `///////.` + +## Components + +- `GoogleCloudStorageBlobStore` - Main BlobStore implementation +- `GoogleCloudStorageClient` - Low-level GCS operations, handles batch deletes via background thread pool +- `GoogleCloudStorageBlobStoreInfo` - XStream-serializable configuration +- `GoogleCloudStorageConfigProvider` - Spring integration for config management + +## Building + +```bash +mvn clean install +``` + +## Testing + +Unit tests run against a fake GCS server via testcontainers: + +```bash +mvn test +``` + +Integration tests use the same fake GCS server but run through the failsafe plugin: + +```bash +mvn verify -Ponline +``` + +## Configuration + +Supports environment variable expansion in all config parameters. Example: + +```xml + + gcs-store + true + ${GCS_BUCKET} + gwc + ${GCS_PROJECT_ID} + true + +``` + +### Parameters + +- `bucket` (required) - GCS bucket name +- `prefix` (optional) - Path prefix within the bucket. If not set, operates at bucket root +- `projectId` (optional) - GCP project ID +- `quotaProjectId` (optional) - Project to bill for quota (for requester-pays buckets) +- `endpointUrl` (optional) - Custom endpoint URL for emulators or GCS-compatible services +- `useDefaultCredentialsChain` (optional) - Set to `true` to use Application Default Credentials +- `apiKey` (optional) - API key for authentication + +Authentication options (pick one): +- `useDefaultCredentialsChain` - Uses Application Default Credentials +- `apiKey` - API key for simple auth +- No auth specified - Anonymous access (useful for emulators) + +## Notes + +Delete operations run asynchronously on a background thread pool sized to available processors. The pool shuts down gracefully on blob store destruction with a 60s timeout. diff --git a/geowebcache/gcsblob/pom.xml b/geowebcache/gcsblob/pom.xml new file mode 100644 index 000000000..e70c4a006 --- /dev/null +++ b/geowebcache/gcsblob/pom.xml @@ -0,0 +1,90 @@ + + + 4.0.0 + + org.geowebcache + geowebcache + 2.0-SNAPSHOT + + gcs-blob + Google Cloud Storage blob store + + + 2.55.0 + 0.2.0 + + + + + + io.aiven + testcontainers-fake-gcs-server + ${testcontainers-fake-gcs-server.version} + test + + + + + + org.geowebcache + gwc-core + ${project.version} + + + com.google.cloud + google-cloud-storage + ${google-cloud-storage.version} + + + + org.mockito + mockito-core + test + + + org.easymock + easymock + test + + + org.geowebcache + gwc-core + tests + test + + + javax.servlet + javax.servlet-api + provided + + + io.aiven + testcontainers-fake-gcs-server + test + + + org.awaitility + awaitility + test + + + + + online + + false + + + + + maven-failsafe-plugin + + 1 + false + + + + + + + diff --git a/geowebcache/gcsblob/src/main/java/org/geowebcache/storage/blobstore/gcs/CacheId.java b/geowebcache/gcsblob/src/main/java/org/geowebcache/storage/blobstore/gcs/CacheId.java new file mode 100644 index 000000000..37e07cff7 --- /dev/null +++ b/geowebcache/gcsblob/src/main/java/org/geowebcache/storage/blobstore/gcs/CacheId.java @@ -0,0 +1,33 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General + * Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this program. If not, see + * . + * + * @author Gabriel Roldan, Camptocamp, Copyright 2025 + */ +package org.geowebcache.storage.blobstore.gcs; + +import org.geowebcache.mime.MimeType; + +/** + * A record to hold the components of a tile's cache identity. + * + *

Note {@code layerName} is used to provide the layer name to callbacks (see + * {@link GoogleCloudStorageBlobStore#sendTileDeleted(TileLocation, long)}, may the layer id be different than the layer + * name like in GeoServer tile layers. For all other purposes, {@code layerId} uniquely identifies the layer (e.g. for + * layer cache prefixes) + * + * @param layerId The unique identifier of the layer. + * @param layerName The name of the layer. + * @param gridsetId The identifier of the gridset. + * @param format The MIME type of the tile. + * @param parametersId The identifier for the tile's parameter set, can be {@code null}. + * @since 1.28 + */ +record CacheId(String layerId, String layerName, String gridsetId, MimeType format, String parametersId) {} diff --git a/geowebcache/gcsblob/src/main/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageBlobStore.java b/geowebcache/gcsblob/src/main/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageBlobStore.java new file mode 100644 index 000000000..bea77deac --- /dev/null +++ b/geowebcache/gcsblob/src/main/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageBlobStore.java @@ -0,0 +1,488 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General + * Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this program. If not, see + * . + * + * @author Gabriel Roldan, Camptocamp, Copyright 2025 + */ +package org.geowebcache.storage.blobstore.gcs; + +import static com.google.common.base.Preconditions.checkNotNull; +import static java.util.Objects.isNull; +import static java.util.Objects.requireNonNull; + +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.StorageException; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.time.OffsetDateTime; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.BiConsumer; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.annotation.Nullable; +import org.geotools.util.logging.Logging; +import org.geowebcache.GeoWebCacheException; +import org.geowebcache.filter.parameters.ParametersUtils; +import org.geowebcache.io.ByteArrayResource; +import org.geowebcache.io.Resource; +import org.geowebcache.layer.TileLayer; +import org.geowebcache.layer.TileLayerDispatcher; +import org.geowebcache.mime.MimeException; +import org.geowebcache.mime.MimeType; +import org.geowebcache.storage.BlobStore; +import org.geowebcache.storage.BlobStoreListener; +import org.geowebcache.storage.BlobStoreListenerList; +import org.geowebcache.storage.CompositeBlobStore; +import org.geowebcache.storage.TileObject; +import org.geowebcache.storage.TileRange; +import org.geowebcache.storage.TileRangeIterator; +import org.geowebcache.util.TMSKeyBuilder; + +/** + * A {@link BlobStore} implementation that stores tiles in a Google Cloud Storage bucket. + * + * @since 1.28 + */ +public class GoogleCloudStorageBlobStore implements BlobStore { + + static Logger log = Logging.getLogger(GoogleCloudStorageBlobStore.class.getName()); + + private final TMSKeyBuilder keyBuilder; + private final BlobStoreListenerList listeners = new BlobStoreListenerList(); + final GoogleCloudStorageClient client; + + private TileLayerDispatcher layers; + + /** + * @param client a pre-configured {@link GoogleCloudStorageClient} + * @param layers the tile layer dispatcher to build tile keys from + * @throws org.geowebcache.storage.StorageException if the target bucket is not suitable for a cache (e.g. it's not + * empty and does not contain a {@code metadata.properties} marker file. + */ + public GoogleCloudStorageBlobStore(GoogleCloudStorageClient client, TileLayerDispatcher layers) + throws org.geowebcache.storage.StorageException { + + this.client = requireNonNull(client); + this.layers = requireNonNull(layers); + + String prefix = Optional.ofNullable(client.getPrefix()).orElse(""); + this.keyBuilder = new TMSKeyBuilder(prefix, layers); + + ensureCacheSuitability(prefix); + } + + void ensureCacheSuitability(String prefix) throws org.geowebcache.storage.StorageException { + + // check target is suitable for a cache + boolean emptyFolder = !client.directoryExists(prefix); + boolean existingMetadata = false; + + final String storeMetadataKey = keyBuilder.storeMetadata(); + if (!emptyFolder) { + existingMetadata = client.blobExists(storeMetadataKey); + } + CompositeBlobStore.checkSuitability(client.getLocation(), existingMetadata, emptyFolder); + + // Just a marker to indicate this is a GWC cache. + if (!existingMetadata) { + putProperties(storeMetadataKey, new Properties()); + } + } + + private void putProperties(String key, Properties properties) throws org.geowebcache.storage.StorageException { + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + properties.store(out, ""); + byte[] bytes = out.toByteArray(); + client.put(key, bytes, "text/plain"); + } catch (IOException | StorageException e) { + throw new org.geowebcache.storage.StorageException("Failed to write properties to " + key, e); + } + } + + /** + * Retrieves a tile from the blob store. + * + * @param obj The {@link TileObject} to populate with tile data. + * @return {@code true} if the tile was found, {@code false} otherwise. + * @throws org.geowebcache.storage.StorageException if an error occurs while accessing the blob store. + */ + @Override + public boolean get(TileObject obj) throws org.geowebcache.storage.StorageException { + final String key = keyBuilder.forTile(obj); + Optional blob = client.get(key); + if (blob.isEmpty()) { + obj.setBlob(null); + obj.setBlobSize(0); + return false; + } + + Blob found = blob.orElseThrow(); + byte[] bytes = found.getContent(); + obj.setBlobSize(bytes.length); + obj.setBlob(new ByteArrayResource(bytes)); + OffsetDateTime updateTime = found.getUpdateTimeOffsetDateTime(); + obj.setCreated(updateTime.toInstant().toEpochMilli()); + return true; + } + + /** + * Stores a tile in the blob store. + * + * @param obj The {@link TileObject} containing the tile data to store. + * @throws org.geowebcache.storage.StorageException if an error occurs while writing to the blob store. + */ + @Override + public void put(TileObject obj) throws org.geowebcache.storage.StorageException { + final Resource content = checkNotNull(obj).getBlob(); + checkNotNull(content); + checkNotNull(obj.getBlobFormat()); + + final String tileKey = keyBuilder.forTile(obj); + final String contentType = getMimeType(obj); + + final long oldSize = listeners.isEmpty() ? -1 : client.getSize(tileKey).orElse(-2L); + + client.put(tileKey, content, contentType); + + putParametersMetadata(obj.getLayerName(), obj.getParametersId(), obj.getParameters()); + + if (oldSize < 0) { + listeners.sendTileStored(obj); + } else { + listeners.sendTileUpdated(obj, oldSize); + } + } + + /** + * Deletes all tiles for a given layer asynchronously. + * + *

This method submits the delete operation to a background task and returns immediately. + * + * @param layerName The name of the layer to delete. + * @return {@code true} if the layer existed and the delete task was submitted, {@code false} otherwise. + * @throws org.geowebcache.storage.StorageException if an error occurs during the delete operation. + */ + @Override + public boolean delete(String layerName) throws org.geowebcache.storage.StorageException { + checkNotNull(layerName, "layerName"); + + final String metadataKey = keyBuilder.layerMetadata(layerName); + final String layerPrefix = keyBuilder.forLayer(layerName); + + // this might not be there, tolerant delete + client.deleteBlob(metadataKey); + + boolean layerExists = client.deleteDirectory(layerPrefix); + if (layerExists) { + listeners.sendLayerDeleted(layerName); + } + return layerExists; + } + + /** + * Deletes a set of tiles for a layer identified by a parameters ID, asynchronously. + * + *

This method submits the delete operation to a background task and returns immediately. + * + * @param layerName The name of the layer. + * @param parametersId The ID of the parameter set. + * @return {@code true} if any tiles were deleted, {@code false} otherwise. + * @throws org.geowebcache.storage.StorageException if an error occurs. + */ + @Override + public boolean deleteByParametersId(String layerName, String parametersId) + throws org.geowebcache.storage.StorageException { + checkNotNull(layerName, "layerName"); + checkNotNull(parametersId, "parametersId"); + + Set gridsetAndFormatPrefixes = keyBuilder.forParameters(layerName, parametersId); + // for each ///// + boolean prefixExists = gridsetAndFormatPrefixes.stream() + .map(client::deleteDirectory) + .reduce(Boolean::logicalOr) + .orElse(false); + if (prefixExists) { + listeners.sendParametersDeleted(layerName, parametersId); + } + return prefixExists; + } + + /** + * Deletes all tiles for a specific gridset within a layer, asynchronously. + * + *

This method submits the delete operation to a background task and returns immediately. + * + * @param layerName The name of the layer. + * @param gridSetId The ID of the gridset. + * @return {@code true} if the gridset existed and was deleted, {@code false} otherwise. + * @throws org.geowebcache.storage.StorageException if an error occurs. + */ + @Override + public boolean deleteByGridsetId(final String layerName, final String gridSetId) + throws org.geowebcache.storage.StorageException { + checkNotNull(layerName, "layerName"); + checkNotNull(gridSetId, "gridSetId"); + + final String gridsetPrefix = keyBuilder.forGridset(layerName, gridSetId); + + boolean prefixExists = client.deleteDirectory(gridsetPrefix); + if (prefixExists) { + listeners.sendGridSubsetDeleted(layerName, gridSetId); + } + return prefixExists; + } + + /** + * Deletes a single tile synchronously. + * + * @param obj The {@link TileObject} to delete. + * @return {@code true} if the tile was deleted, {@code false} if it did not exist. + * @throws org.geowebcache.storage.StorageException if an error occurs. + */ + @Override + public boolean delete(TileObject obj) throws org.geowebcache.storage.StorageException { + final String tileKey = keyBuilder.forTile(obj); + + // don't bother for the extra call if there are no listeners + if (listeners.isEmpty()) { + return client.deleteBlob(tileKey); + } + + // if there are listeners, gather extra information + final long oldSize = client.getSize(tileKey).orElse(0L); + final boolean deleted = client.deleteBlob(tileKey); + if (deleted && oldSize > 0L) { + obj.setBlobSize((int) oldSize); + listeners.sendTileDeleted(obj); + } + return deleted; + } + + /** + * Deletes a range of tiles asynchronously. + * + *

This method submits the delete operation to a background task and returns immediately. + * + * @param tileRange The range of tiles to delete. + * @return {@code true} if the tile range existed and the delete task was submitted, {@code false} otherwise. + * @throws org.geowebcache.storage.StorageException if an error occurs. + */ + @Override + public boolean delete(TileRange tileRange) throws org.geowebcache.storage.StorageException { + requireNonNull(tileRange); + final boolean endWithSlash = false; + // the key prefix up to the coordinates (i.e. {@code "////"}) + final String coordsPrefix = keyBuilder.coordinatesPrefix(tileRange, endWithSlash); + + if (client.directoryExists(coordsPrefix)) { + Stream tiles = toTileLocations(tileRange); + if (listeners.isEmpty()) { + client.delete(tiles); + } else { + BiConsumer callback = this::sendTileDeleted; + client.delete(tiles, callback); + } + return true; + } + return false; + } + + void sendTileDeleted(TileLocation tile, long size) { + String layerName = tile.cache().layerName(); + String gridsetId = tile.cache().gridsetId(); + MimeType mimeType = tile.cache().format(); + String format = mimeType.getFormat(); + String parametersId = tile.cache().parametersId(); + long x = tile.tile().x(); + long y = tile.tile().y(); + int z = tile.tile().z(); + listeners.sendTileDeleted(layerName, gridsetId, format, parametersId, x, y, z, size); + } + + private Stream toTileLocations(TileRange tileRange) { + String layerName = tileRange.getLayerName(); + String layerId; + // we need the layer name and the layer id, for the most part they seem to be the same, but not in + // GeoServerTileLayers + try { + TileLayer tileLayer = layers.getTileLayer(layerName); + layerId = tileLayer.getId(); + } catch (GeoWebCacheException e) { + throw new IllegalStateException(e); + } + + String gridSetId = tileRange.getGridSetId(); + MimeType mimeType = tileRange.getMimeType(); + String parametersId = tileRange.getParametersId(); + + final CacheId tileCacheId = new CacheId(layerId, layerName, gridSetId, mimeType, parametersId); + Stream tileIndices = toTileIndices(tileRange); + String prefix = client.getPrefix(); + return tileIndices.map(ti -> new TileLocation(prefix, tileCacheId, ti)); + } + + static Stream toTileIndices(TileRange tileRange) { + + final int[] metaTilingFactors = {1, 1}; + final TileRangeIterator trIter = new TileRangeIterator(tileRange, metaTilingFactors); + + // optimization for TileRangeIterator.nextMetaGridLocation() to avoid creating many arrays + final long[] reusedGridLoc = new long[3]; + + Stream gridLocations = Stream.generate(() -> trIter.nextMetaGridLocation(reusedGridLoc)) + // Stream.generate() creates an infinite stream, we need to end it on null + .takeWhile(Objects::nonNull); + + return gridLocations.map(gridLoc -> new TileIndex(gridLoc[0], gridLoc[1], (int) gridLoc[2])); + } + + private String getMimeType(TileObject obj) { + try { + return MimeType.createFromFormat(obj.getBlobFormat()).getMimeType(); + } catch (MimeException e) { + throw new IllegalArgumentException(e); + } + } + + /** + * Local cache of existing parameters metadata to avoid every {@link #put(TileObject)} call to store the same + * properties file. + * + *

This is ok since the parameters id is a {@link ParametersUtils#getId(Map) hash code} of the contents + */ + private Map existingParametersMedatata = new ConcurrentHashMap<>(); + + void putParametersMetadata(String layerName, String parametersId, Map parameters) { + assert (isNull(parametersId) == isNull(parameters)); + if (isNull(parametersId)) { + return; + } + if (existingParametersMedatata.containsKey(parametersId)) { + return; + } + + Properties properties = new Properties(); + parameters.forEach(properties::setProperty); + String resourceKey = keyBuilder.parametersMetadata(layerName, parametersId); + try { + putProperties(resourceKey, properties); + existingParametersMedatata.put(layerName, parametersId); + } catch (org.geowebcache.storage.StorageException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public void clear() throws org.geowebcache.storage.StorageException { + throw new UnsupportedOperationException("clear() should not be called"); + } + + @Override + public void destroy() { + client.close(); + } + + @Override + public void addListener(BlobStoreListener listener) { + listeners.addListener(listener); + } + + @Override + public boolean removeListener(BlobStoreListener listener) { + return listeners.removeListener(listener); + } + + @Override + public boolean rename(String oldLayerName, String newLayerName) throws org.geowebcache.storage.StorageException { + log.fine("No need to rename layers, GoogleCloudStorageBlobStore uses layer id as key root"); + if (client.directoryExists(oldLayerName)) { + listeners.sendLayerRenamed(oldLayerName, newLayerName); + } + return true; + } + + @Override + @Nullable + public String getLayerMetadata(String layerName, String key) { + Properties properties = getLayerMetadata(layerName); + return properties.getProperty(key); + } + + @Override + public void putLayerMetadata(String layerName, String key, String value) { + Properties properties = getLayerMetadata(layerName); + properties.setProperty(key, value); + String resourceKey = keyBuilder.layerMetadata(layerName); + try { + putProperties(resourceKey, properties); + } catch (org.geowebcache.storage.StorageException e) { + throw new UncheckedIOException(e); + } + } + + private Properties getLayerMetadata(String layerName) { + String key = keyBuilder.layerMetadata(layerName); + return findProperties(key).orElseGet(Properties::new); + } + + @Override + public boolean layerExists(String layerName) { + final String layerPrefix = keyBuilder.forLayer(layerName); + return client.directoryExists(layerPrefix); + } + + @Override + public Map>> getParametersMapping(String layerName) { + String parametersMetadataPrefix = keyBuilder.parametersMetadataPrefix(layerName); + Stream blobStream = client.list(parametersMetadataPrefix); + + return blobStream + .map(Blob::getName) + .map(this::loadProperties) + .collect(Collectors.toMap(ParametersUtils::getId, Optional::ofNullable)); + } + + private Optional findProperties(String key) { + try { + return client.get(key).map(this::toProperties); + } catch (Exception e) { + log.log(Level.WARNING, "Failed to read properties from " + key, e); + } + return Optional.empty(); + } + + private Map loadProperties(String blobKey) { + return findProperties(blobKey).map(this::toMap).orElse(Map.of()); + } + + private Properties toProperties(Blob blob) { + try { + Properties properties = new Properties(); + properties.load(new ByteArrayInputStream(blob.getContent())); + return properties; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private Map toMap(Properties properties) { + return properties.entrySet().stream() + .collect(Collectors.toMap(e -> (String) e.getKey(), e -> (String) e.getValue())); + } +} diff --git a/geowebcache/gcsblob/src/main/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageBlobStoreInfo.java b/geowebcache/gcsblob/src/main/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageBlobStoreInfo.java new file mode 100644 index 000000000..0c0ed47dc --- /dev/null +++ b/geowebcache/gcsblob/src/main/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageBlobStoreInfo.java @@ -0,0 +1,178 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General + * Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this program. If not, see + * . + * + * @author Gabriel Roldan, Camptocamp, Copyright 2025 + */ +package org.geowebcache.storage.blobstore.gcs; + +import java.util.Objects; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.geowebcache.GeoWebCacheEnvironment; +import org.geowebcache.GeoWebCacheExtensions; +import org.geowebcache.config.BlobStoreInfo; +import org.geowebcache.layer.TileLayerDispatcher; +import org.geowebcache.locks.LockProvider; +import org.geowebcache.storage.StorageException; +import org.springframework.util.StringUtils; + +/** + * Configuration and factory for a {@link GoogleCloudStorageBlobStore}. + * + *

This class holds the necessary connection and authentication details for accessing a Google Cloud Storage bucket. + * It is typically configured in {@code geowebcache.xml} and used by GeoWebCache to instantiate the blob store. + * + * @implNote All configuration properties are strings to allow for environment parameterization (e.g., using + * {@code ${name}} variable placeholders). The + * {@link GoogleCloudStorageClient#builder(GoogleCloudStorageBlobStoreInfo, GeoWebCacheEnvironment)} dereferences + * these placeholders. + * @since 1.28 + */ +@SuppressWarnings("serial") +public class GoogleCloudStorageBlobStoreInfo extends BlobStoreInfo { + + private String projectId; + private String quotaProjectId; + private String bucket; + private String prefix; + private String endpointUrl; // Custom endpoint for emulators or non-standard GCS endpoints + private String apiKey; + private String useDefaultCredentialsChain; + + @Override + public GoogleCloudStorageBlobStore createInstance(TileLayerDispatcher layers, LockProvider lockProvider) + throws StorageException { + GeoWebCacheEnvironment environment = GeoWebCacheExtensions.bean(GeoWebCacheEnvironment.class); + return createInstance(layers, environment); + } + + GoogleCloudStorageBlobStore createInstance(TileLayerDispatcher layers, GeoWebCacheEnvironment environment) + throws StorageException { + + GoogleCloudStorageClient client = + GoogleCloudStorageClient.builder(this, environment).build(); + return new GoogleCloudStorageBlobStore(client, layers); + } + + @Override + public String getLocation() { + if (StringUtils.hasText(prefix)) { + return "gs://" + bucket + "/" + prefix; + } + return "gs://" + bucket; + } + + /** @return The Google Cloud Storage project ID. */ + public String getProjectId() { + return projectId; + } + + public void setProjectId(String projectId) { + this.projectId = projectId; + } + + /** + * @return The ID of the project to bill for quota purposes. + * @see Requester Pays + */ + public String getQuotaProjectId() { + return quotaProjectId; + } + + public void setQuotaProjectId(String quotaProjectId) { + this.quotaProjectId = quotaProjectId; + } + + /** @return The name of the GCS bucket. */ + public String getBucket() { + return bucket; + } + + public void setBucket(String bucket) { + this.bucket = bucket; + } + + /** @return The blob name prefix within the bucket, or {@code null} if the cache operates at the bucket's root. */ + public String getPrefix() { + return prefix; + } + + /** @param prefix the blob name prefix applied to the cache, if not set, the cache operates at the bucket's root */ + public void setPrefix(String prefix) { + this.prefix = prefix; + } + + /** @return An alternative GCS endpoint URL, for use with emulators or GCS-compatible services. */ + public String getEndpointUrl() { + return endpointUrl; + } + + /** + * Set an alternative GCS endpoint URL, for example, for compatible services + * + * @param endpoint + */ + public void setEndpointUrl(String endpoint) { + this.endpointUrl = endpoint; + } + + /** + * @return The API key for authentication. If provided, it is used in preference to the default credentials chain. + */ + public String getApiKey() { + return apiKey; + } + + public void setApiKey(String apiKey) { + this.apiKey = apiKey; + } + + /** + * @return {@code "true"} if the default Google Cloud credentials chain should be used for authentication. + * @see com.google.auth.oauth2.GoogleCredentials#getApplicationDefault() + */ + public String getUseDefaultCredentialsChain() { + return useDefaultCredentialsChain; + } + + /** + * Sets whether to use the default Google Cloud credentials chain. + * + * @param defaultCredentialsChain {@code "true"} to enable, {@code "false"} to disable. + */ + public void setUseDefaultCredentialsChain(String defaultCredentialsChain) { + this.useDefaultCredentialsChain = defaultCredentialsChain; + } + + @Override + public String toString() { + return ToStringBuilder.reflectionToString(this); + } + + @Override + public boolean equals(Object o) { + if (super.equals(o)) { + GoogleCloudStorageBlobStoreInfo other = (GoogleCloudStorageBlobStoreInfo) o; + return Objects.equals(projectId, other.projectId) + && Objects.equals(bucket, other.bucket) + && Objects.equals(endpointUrl, other.endpointUrl) + && Objects.equals(apiKey, other.apiKey) + && Objects.equals(useDefaultCredentialsChain, other.useDefaultCredentialsChain) + && Objects.equals(quotaProjectId, other.quotaProjectId); + } + return false; + } + + @Override + public int hashCode() { + return 31 * super.hashCode() + + Objects.hash(projectId, bucket, endpointUrl, apiKey, useDefaultCredentialsChain, quotaProjectId); + } +} diff --git a/geowebcache/gcsblob/src/main/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageClient.java b/geowebcache/gcsblob/src/main/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageClient.java new file mode 100644 index 000000000..6f4fbd577 --- /dev/null +++ b/geowebcache/gcsblob/src/main/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageClient.java @@ -0,0 +1,560 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General + * Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this program. If not, see + * . + * + * @author Gabriel Roldan, Camptocamp, Copyright 2025 + */ +package org.geowebcache.storage.blobstore.gcs; + +import static java.util.Objects.requireNonNull; + +import com.google.api.gax.paging.Page; +import com.google.auth.ApiKeyCredentials; +import com.google.auth.Credentials; +import com.google.auth.oauth2.GoogleCredentials; +import com.google.cloud.BatchResult; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BlobField; +import com.google.cloud.storage.Storage.BlobGetOption; +import com.google.cloud.storage.Storage.BlobListOption; +import com.google.cloud.storage.StorageBatch; +import com.google.cloud.storage.StorageBatchResult; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.StorageOptions; +import com.google.common.collect.Iterators; +import com.google.common.io.ByteStreams; +import java.io.IOException; +import java.io.InputStream; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; +import java.util.function.BooleanSupplier; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Stream; +import org.geotools.util.logging.Logging; +import org.geowebcache.GeoWebCacheEnvironment; +import org.geowebcache.io.Resource; + +/** + * A low-level client to interact with a Google Cloud Storage bucket, tailored for {@link GoogleCloudStorageBlobStore}'s + * needs. + * + *

This client handles the communication with the GCS API, including authentication, and performs bulk delete + * operations asynchronously in a background thread pool. + * + * @since 1.28 + */ +class GoogleCloudStorageClient { + static Logger log = Logging.getLogger(GoogleCloudStorageClient.class.getName()); + + private static final int DEFAULT_BATCH_SIZE = 1000; + private final ExecutorService deleteService; + + private final Storage storage; + + private final String bucket; + + /** Prefix prepended to all blob paths */ + private final String prefix; + + private volatile boolean closed; + + private GoogleCloudStorageClient(Storage storageClient, String bucket, String prefix) { + this.storage = storageClient; + this.bucket = bucket; + this.prefix = prefix; + int poolSize = Runtime.getRuntime().availableProcessors(); + this.deleteService = Executors.newFixedThreadPool(poolSize); + } + + /** + * Creates a builder initialized with configuration values from the provided blob store info. + * + *

This factory method extracts configuration parameters from {@link GoogleCloudStorageBlobStoreInfo}, resolving + * environment variables if enabled through the {@link GeoWebCacheEnvironment}. + * + * @param config The Google Cloud Storage blob store configuration. + * @param environment The GeoWebCache environment for resolving configuration placeholders. + * @return A builder instance configured with the resolved parameters. + * @throws org.geowebcache.storage.StorageException if the bucket parameter is not provided or resolution fails. + */ + public static GoogleCloudStorageClient.Builder builder( + GoogleCloudStorageBlobStoreInfo config, GeoWebCacheEnvironment environment) + throws org.geowebcache.storage.StorageException { + + requireNonNull(config); + requireNonNull(environment); + + GoogleCloudStorageClient.Builder builder = GoogleCloudStorageClient.builder(); + builder.bucket(environment + .resolveValueIfEnabled(config.getBucket(), String.class) + .orElseThrow(() -> new IllegalArgumentException("parameter bucket is mandatory"))); + + builder.prefix(environment + .resolveValueIfEnabled(config.getPrefix(), String.class) + .orElse(null)); + + builder.projectId(environment + .resolveValueIfEnabled(config.getProjectId(), String.class) + .orElse(null)); + + builder.quotaProjectId(environment + .resolveValueIfEnabled(config.getQuotaProjectId(), String.class) + .orElse(null)); + + builder.endpointUrl(environment + .resolveValueIfEnabled(config.getEndpointUrl(), String.class) + .orElse(null)); + + builder.apiKey(environment + .resolveValueIfEnabled(config.getApiKey(), String.class) + .orElse(null)); + + builder.useDefaultCredentialsChain(environment + .resolveValueIfEnabled(config.getUseDefaultCredentialsChain(), Boolean.class) + .orElse(false)); + + return builder; + } + + /** + * Creates a new builder for configuring a {@link GoogleCloudStorageClient}. + * + * @return A new builder instance with default values. + */ + public static GoogleCloudStorageClient.Builder builder() { + return new Builder(); + } + + static class Builder { + String projectId; + String quotaProjectId; + String bucket; + String prefix; + String endpointUrl; + String apiKey; + boolean useDefaultCredentialsChain; + + public Builder projectId(String projectId) { + this.projectId = projectId; + return this; + } + + public Builder quotaProjectId(String quotaProjectId) { + this.quotaProjectId = quotaProjectId; + return this; + } + + public Builder bucket(String bucket) { + this.bucket = bucket; + return this; + } + + public Builder prefix(String prefix) { + this.prefix = prefix; + return this; + } + + public Builder endpointUrl(String endpointUrl) { + this.endpointUrl = endpointUrl; + return this; + } + + public Builder apiKey(String apiKey) { + this.apiKey = apiKey; + return this; + } + + public Builder useDefaultCredentialsChain(boolean useDefaultCredentialsChain) { + this.useDefaultCredentialsChain = useDefaultCredentialsChain; + return this; + } + + public GoogleCloudStorageClient build() throws org.geowebcache.storage.StorageException { + StorageOptions.Builder builder = StorageOptions.getDefaultInstance().toBuilder(); + + if (projectId != null) { + builder.setProjectId(projectId); + } + if (quotaProjectId != null) { + builder.setQuotaProjectId(quotaProjectId); + } + if (endpointUrl != null) { + // Set custom endpoint for emulators or non-standard GCS endpoints + builder.setHost(endpointUrl); + } + Credentials credentials = null; + if (apiKey != null) { + credentials = ApiKeyCredentials.create(apiKey); + } else if (useDefaultCredentialsChain) { + try { + credentials = GoogleCredentials.getApplicationDefault(); + } catch (IOException e) { + throw new org.geowebcache.storage.StorageException("Error obtaining default credentials", e); + } + } + if (credentials != null) { + // credentials need to be set after projectId and quotaProjectId so its setter will + // check whether projectId is null and get it from credentials if its a ServiceAccountCredentials + // or quotaProjectId is null and get it from credentials if it's a QuotaProjectIdProvider + builder.setCredentials(credentials); + } + Storage storageClient = builder.build().getService(); + + return new GoogleCloudStorageClient(storageClient, bucket, prefix); + } + } + + /** + * Gets the name of the Google Cloud Storage bucket this client operates on. + * + * @return The bucket name. + */ + public String getBucket() { + return bucket; + } + + /** + * Gets the prefix prepended to all blob paths in the bucket. + * + * @return The prefix string, or {@code null} if no prefix is configured. + */ + public String getPrefix() { + return prefix; + } + + /** + * Gets the full Google Cloud Storage location in {@code gs://} URI format. + * + * @return The location as {@code gs:///}. + */ + public String getLocation() { + return "gs://%s/%s".formatted(bucket, prefix); + } + + /** + * Checks if a blob with the given key exists in the bucket. + * + * @param key The blob key to check. + * @return {@code true} if the blob exists, {@code false} otherwise. + * @throws org.geowebcache.storage.StorageException if an error occurs accessing the storage. + */ + public boolean blobExists(String key) throws org.geowebcache.storage.StorageException { + return get(key).isPresent(); + } + + /** + * Lists all blobs whose names begin with the specified prefix. + * + * @param prefix The prefix to filter blobs by. + * @return A stream of matching {@link Blob} objects. + */ + public Stream list(String prefix) { + return storage.list(bucket, BlobListOption.prefix(requireNonNull(prefix))) + .streamAll(); + } + + /** + * Checks if a directory (path prefix) exists in the bucket. + * + *

This method checks if there are any blobs whose names start with the given path (with a trailing slash added + * if not present). It only fetches one blob to efficiently determine existence. + * + * @param path The directory path to check. + * @return {@code true} if at least one blob exists with this path prefix, {@code false} otherwise. + */ + public boolean directoryExists(final String path) { + requireNonNull(path); + String dirPrefix = dirPrefix(path); + Page blobs = storage.list(bucket, BlobListOption.prefix(dirPrefix), BlobListOption.pageSize(1)); + Iterator iterator = blobs.getValues().iterator(); + boolean hasNext = iterator.hasNext(); + if (hasNext) { + log.fine("Directory exists: " + path); + } else { + log.fine("Directory does not exist: " + path); + } + return hasNext; + } + + /** + * Deletes all blobs under a given path prefix asynchronously. + * + *

This method checks if the directory exists and, if so, submits a task to a background thread pool to delete + * all blobs whose names start with the given path. The method returns immediately. + * + *

The provided {@code path} is treated as a directory prefix. A trailing slash ({@code /}) is added if not + * already present to ensure only blobs *within* the directory are matched, avoiding accidental deletion of sibling + * blobs with a similar prefix. + * + * @param path The directory path to delete. + * @return {@code true} if the directory existed and the delete task was submitted, {@code false} if the directory + * did not exist. + */ + public boolean deleteDirectory(String path) { + requireNonNull(path); + if (directoryExists(path)) { + String dirPrefix = dirPrefix(path); + deleteService.submit(() -> deleteAllByPrefix(dirPrefix)); + return true; + } + return false; + } + + private String dirPrefix(String path) { + // Add trailing slash to ensure we only delete blobs within this specific directory + String dirPrefix = path.isEmpty() || path.endsWith("/") ? path : path + "/"; + return dirPrefix; + } + + /** + * Closes this client and shuts down the background delete service. + * + *

This method initiates an orderly shutdown of the delete thread pool, waiting up to 60 seconds for in-progress + * deletions to complete. If tasks don't complete within that time, a forced shutdown is attempted. Any ongoing or + * queued delete operations after this point will be cancelled. + * + *

This method should be called when the blob store is being destroyed to ensure proper cleanup of resources. + */ + public void close() { + closed = true; + if (!deleteService.isShutdown()) { + deleteService.shutdown(); + try { + if (!deleteService.awaitTermination(60, TimeUnit.SECONDS)) { + deleteService.shutdownNow(); + } + } catch (InterruptedException e) { + deleteService.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + } + + /** + * Gets the size in bytes of a blob without retrieving its full content. + * + *

This method performs an optimized request that only fetches the blob's size metadata, making it efficient for + * checking blob sizes without downloading content. + * + * @param key The blob key. + * @return An {@link OptionalLong} containing the blob size in bytes, or empty if the blob does not exist. + * @throws org.geowebcache.storage.StorageException if an error occurs accessing the storage. + */ + public OptionalLong getSize(String key) throws org.geowebcache.storage.StorageException { + try { + Blob blob = storage.get(bucket, requireNonNull(key), BlobGetOption.fields(BlobField.SIZE)); + return blob == null ? OptionalLong.empty() : OptionalLong.of(blob.getSize()); + } catch (StorageException e) { + throw new org.geowebcache.storage.StorageException("Failed to get blob " + key, e); + } + } + + /** + * Retrieves a blob by its key. + * + * @param key The blob key. + * @return An {@link Optional} containing the {@link Blob} if found, or empty if not found. + * @throws org.geowebcache.storage.StorageException if an error occurs accessing the storage. + */ + public Optional get(String key) throws org.geowebcache.storage.StorageException { + try { + return Optional.ofNullable(storage.get(bucket, requireNonNull(key))); + } catch (StorageException e) { + throw new org.geowebcache.storage.StorageException("Failed to get blob " + key, e); + } + } + + /** + * Stores a blob in the bucket from a byte array. + * + * @param key The blob key (path) under which to store the content. + * @param bytes The content to store. + * @param contentType The MIME type of the content (e.g., "image/png"). + * @return The created {@link Blob} object. + * @throws org.geowebcache.storage.StorageException if the upload fails. + */ + public Blob put(String key, byte[] bytes, String contentType) throws org.geowebcache.storage.StorageException { + BlobInfo blobInfo = BlobInfo.newBuilder(bucket, requireNonNull(key)) + .setContentType(requireNonNull(contentType)) + .build(); + try { + return storage.create(blobInfo, bytes); + } catch (Exception e) { + throw new org.geowebcache.storage.StorageException("Failed to upload tile to GCS with key " + key, e); + } + } + + /** + * Stores a blob in the bucket from a {@link Resource}. + * + *

This method reads the entire resource into memory before uploading to GCS. + * + * @param key The blob key (path) under which to store the content. + * @param blob The resource containing the content to store. + * @param contentType The MIME type of the content (e.g., "image/png"). + * @return The created {@link Blob} object. + * @throws org.geowebcache.storage.StorageException if reading the resource or uploading fails. + */ + public Blob put(String key, Resource blob, String contentType) throws org.geowebcache.storage.StorageException { + byte[] bytes; + try (InputStream is = requireNonNull(blob).getInputStream()) { + bytes = ByteStreams.toByteArray(is); + } catch (IOException e) { + throw new org.geowebcache.storage.StorageException("Failed to upload tile to GCS with key " + key, e); + } + return put(key, bytes, contentType); + } + + /** + * Deletes a single blob synchronously. + * + * @param key The blob key to delete. + * @return {@code true} if the blob was deleted, {@code false} if it didn't exist. + * @throws org.geowebcache.storage.StorageException if the delete operation fails. + */ + public boolean deleteBlob(String key) throws org.geowebcache.storage.StorageException { + try { + return storage.delete(bucket, requireNonNull(key)); + } catch (com.google.cloud.storage.StorageException e) { + throw new org.geowebcache.storage.StorageException("Failed to delete blob " + key, e); + } + } + + /** + * Asynchronously deletes a stream of tiles. + * + * @param keys A stream of {@link TileLocation} objects to delete. + * @throws org.geowebcache.storage.StorageException if an error occurs submitting the delete task. + */ + public void delete(Stream keys) throws org.geowebcache.storage.StorageException { + Stream ids = requireNonNull(keys).map(this::toBlobId); + try { + // deleteInternal(ids); + deleteService.submit(() -> deleteInternal(ids)); + } catch (com.google.cloud.storage.StorageException gcsException) { + throw new org.geowebcache.storage.StorageException("Failed to delete tiles", gcsException); + } + } + + /** + * Asynchronously deletes a stream of tiles, invoking a callback for each successfully deleted tile. + * + * @param keys A stream of {@link TileLocation} objects to delete. + * @param callback A {@link BiConsumer} that accepts the {@link TileLocation} and size of each deleted tile. + */ + public void delete(Stream keys, BiConsumer callback) { + requireNonNull(keys); + requireNonNull(callback); + // deleteInternal(keys, callback); + deleteService.submit(() -> deleteInternal(keys, callback)); + } + + /** + * Internal method that performs batched deletions with callbacks for each tile. + * + *

This method partitions the stream of tiles into batches of up to {@value #DEFAULT_BATCH_SIZE} tiles and + * processes each batch using the GCS batch API. For each successfully deleted tile, the callback is invoked with + * the tile location and its size. + * + *

This method is typically called from a background thread via {@link #delete(Stream, BiConsumer)}. + * + * @param keys The stream of tile locations to delete. + * @param callback The callback invoked for each successfully deleted tile. + */ + public void deleteInternal(Stream keys, BiConsumer callback) { + Iterator> partitions = Iterators.partition(keys.iterator(), DEFAULT_BATCH_SIZE); + + while (partitions.hasNext() && !closed) { + deleteInternal(partitions.next(), callback); + } + } + + private void deleteInternal(List partition, BiConsumer callback) { + List ids = partition.stream().map(this::toBlobId).toList(); + + StorageBatch batch = storage.batch(); + for (int i = 0; i < partition.size(); i++) { + TileLocation tile = partition.get(i); + + BlobId blobId = ids.get(i); + StorageBatchResult size = batch.get(blobId, BlobGetOption.fields(BlobField.SIZE)); + batch.delete(blobId).notify(new TileDeleteCallback(tile, size, callback, () -> this.closed)); + } + batch.submit(); + } + + private record TileDeleteCallback( + TileLocation tile, + StorageBatchResult sizeResult, + BiConsumer callback, + BooleanSupplier closedCheck) + implements BatchResult.Callback { + + @Override + public void success(Boolean result) { + if (closedCheck.getAsBoolean()) return; + if (!result) { + if (log.isLoggable(Level.FINEST)) { + log.finest("Tile didn't exist while deleting " + tile.getStorageKey()); + } + return; + } + + if (sizeResult.completed()) { + Blob sizeOnlyBlob = sizeResult.get(); + Long size = sizeOnlyBlob.getSize(); + callback.accept(tile, size); + } else { + throw new IllegalArgumentException("%s size fetch didn't complete".formatted(tile)); + } + } + + @Override + public void error(StorageException exception) { + if (!closedCheck.getAsBoolean()) { + log.log(Level.FINER, "Exception deleting tile " + tile.getStorageKey(), exception); + } + } + } + + private void deleteAllByPrefix(String prefix) { + Stream prefixedBlobs = storage.list( + bucket, BlobListOption.prefix(prefix), BlobListOption.fields(BlobField.SIZE)) + .streamAll(); + + Stream blobIds = prefixedBlobs.map(Blob::getBlobId); + + deleteInternal(blobIds); + } + + private void deleteInternal(Stream blobIds) { + Iterator> partitions = Iterators.partition(blobIds.iterator(), DEFAULT_BATCH_SIZE); + + while (partitions.hasNext() && !closed) { + List ids = partitions.next(); + storage.delete(ids); // A batch request is used to perform this call + } + } + + private BlobId toBlobId(TileLocation loc) { + String storageKey = loc.getStorageKey(); + return BlobId.of(bucket, storageKey); + } +} diff --git a/geowebcache/gcsblob/src/main/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageConfigProvider.java b/geowebcache/gcsblob/src/main/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageConfigProvider.java new file mode 100644 index 000000000..f305c3b6a --- /dev/null +++ b/geowebcache/gcsblob/src/main/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageConfigProvider.java @@ -0,0 +1,43 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General + * Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this program. If not, see + * . + * + * @author Gabriel Roldan, Camptocamp, Copyright 2025 + */ +package org.geowebcache.storage.blobstore.gcs; + +import com.thoughtworks.xstream.XStream; +import org.geowebcache.config.Info; +import org.geowebcache.config.XMLConfigurationProvider; + +/** + * Configures XStream for XML persistence of {@link GoogleCloudStorageBlobStoreInfo} objects. + * + *

This class sets up the necessary aliases for serializing and deserializing the blob store configuration to and + * from {@code geowebcache.xml}. + * + * @since 1.28 + */ +public class GoogleCloudStorageConfigProvider implements XMLConfigurationProvider { + + @Override + public XStream getConfiguredXStream(XStream xs) { + Class clazz = GoogleCloudStorageBlobStoreInfo.class; + xs.alias("GoogleCloudStorageBlobStore", clazz); + xs.aliasField("id", clazz, "name"); + xs.allowTypes(new Class[] {GoogleCloudStorageBlobStoreInfo.class}); + return xs; + } + + @Override + public boolean canSave(Info i) { + return i instanceof GoogleCloudStorageBlobStoreInfo; + } +} diff --git a/geowebcache/gcsblob/src/main/java/org/geowebcache/storage/blobstore/gcs/TileIndex.java b/geowebcache/gcsblob/src/main/java/org/geowebcache/storage/blobstore/gcs/TileIndex.java new file mode 100644 index 000000000..c5f120321 --- /dev/null +++ b/geowebcache/gcsblob/src/main/java/org/geowebcache/storage/blobstore/gcs/TileIndex.java @@ -0,0 +1,39 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General + * Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this program. If not, see + * . + * + * @author Gabriel Roldan, Camptocamp, Copyright 2025 + */ +package org.geowebcache.storage.blobstore.gcs; + +/** + * A record to hold the spatial coordinates of a tile in a tile matrix set. + * + *

This record represents the position of a tile within a gridset using the standard TMS (Tile Map Service) + * coordinate system. The coordinates follow the convention where: + * + *

    + *
  • {@code x} - the column index of the tile (west to east) + *
  • {@code y} - the row index of the tile (south to north in TMS, north to south in some other systems) + *
  • {@code z} - the zoom level, where higher values represent higher zoom levels (more detail) + *
+ * + *

TileIndex instances are typically created from a {@link org.geowebcache.storage.TileRange} using + * {@link org.geowebcache.storage.TileRangeIterator} to iterate over all tiles in a range. See + * {@link GoogleCloudStorageBlobStore#toTileIndices(org.geowebcache.storage.TileRange)}. + * + * @param x The column index of the tile (horizontal position). + * @param y The row index of the tile (vertical position). + * @param z The zoom level of the tile. + * @see TileLocation + * @see CacheId + * @since 1.28 + */ +record TileIndex(long x, long y, int z) {} diff --git a/geowebcache/gcsblob/src/main/java/org/geowebcache/storage/blobstore/gcs/TileLocation.java b/geowebcache/gcsblob/src/main/java/org/geowebcache/storage/blobstore/gcs/TileLocation.java new file mode 100644 index 000000000..fa625498a --- /dev/null +++ b/geowebcache/gcsblob/src/main/java/org/geowebcache/storage/blobstore/gcs/TileLocation.java @@ -0,0 +1,93 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General + * Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this program. If not, see + * . + * + * @author Gabriel Roldan, Camptocamp, Copyright 2025 + */ +package org.geowebcache.storage.blobstore.gcs; + +import org.geowebcache.storage.TileObject; +import org.geowebcache.util.TMSKeyBuilder; +import org.springframework.util.StringUtils; + +/** + * A record to hold the complete location information for a tile in Google Cloud Storage. + * + *

This record combines all the components necessary to uniquely identify and locate a tile in a blob store: + * + *

    + *
  • {@code prefix} - the storage prefix path (bucket-level prefix for organizing caches) + *
  • {@code cache} - a {@link CacheId} that identifies the layer, gridset, format, and parameters + *
  • {@code tile} - a {@link TileIndex} that identifies the tile's spatial coordinates + *
+ * + *

TileLocation instances are typically created from a {@link org.geowebcache.storage.TileRange} by combining layer + * information into a {@link CacheId}, extracting tile coordinates into {@link TileIndex} instances, and adding the + * storage prefix. See {@link GoogleCloudStorageBlobStore#toTileLocations(org.geowebcache.storage.TileRange)}. + * + *

The primary purposes of this record are: + * + *

    + *
  • Generate the complete storage key path for a tile using {@link #getStorageKey()}, which follows the TMS (Tile + * Map Service) key structure + *
  • Provide a convenient way to pass complete tile information to callbacks during bulk operations, such as in + * {@link GoogleCloudStorageBlobStore#sendTileDeleted(TileLocation, long)}, where all tile metadata is extracted + * and forwarded to blob store listeners + *
+ * + * @param prefix The storage prefix path, can be {@code null} or empty. + * @param cache The cache identity containing layer, gridset, format, and parameters information. + * @param tile The tile's spatial coordinates (x, y, z). + * @see CacheId + * @see TileIndex + * @see TMSKeyBuilder + * @since 1.28 + */ +record TileLocation(String prefix, CacheId cache, TileIndex tile) { + + /** + * Same as {@link TMSKeyBuilder#forTile(TileObject)} but using this record's data + * + * @return {@code ///////.} + */ + public String getStorageKey() { + String parametersId = cache.parametersId(); + if (parametersId == null) { + parametersId = "default"; + } + String layerId = cache.layerId(); + String gridset = cache.gridsetId(); + String shortFormat = cache.format().getFileExtension(); + String extension = cache.format().getInternalName(); // png, jpeg, etc. + + StringBuilder sb = new StringBuilder(); + if (StringUtils.hasText(prefix)) { + sb.append(prefix).append('/'); + } + + sb.append(layerId) + .append('/') + .append(gridset) + .append('/') + .append(shortFormat) + .append('/') + .append(parametersId) + .append('/') + .append(tile.z()) + .append('/') + .append(tile.x()) + .append('/') + .append(tile.y()) + .append('.') + .append(extension); + + return sb.toString(); + } +} diff --git a/geowebcache/gcsblob/src/test/java/org/geowebcache/storage/blobstore/gcs/CollectingListener.java b/geowebcache/gcsblob/src/test/java/org/geowebcache/storage/blobstore/gcs/CollectingListener.java new file mode 100644 index 000000000..0fa1b4c25 --- /dev/null +++ b/geowebcache/gcsblob/src/test/java/org/geowebcache/storage/blobstore/gcs/CollectingListener.java @@ -0,0 +1,123 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General + * Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this program. If not, see + * . + * + * @author Gabriel Roldan, Camptocamp, Copyright 2025 + */ +package org.geowebcache.storage.blobstore.gcs; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import org.geowebcache.mime.MimeException; +import org.geowebcache.mime.MimeType; +import org.geowebcache.storage.BlobStoreListener; + +class CollectingListener implements BlobStoreListener { + + Map tilesStored = new ConcurrentHashMap<>(); + Map tilesDeleted = new ConcurrentHashMap<>(); + Map tilesUpdated = new ConcurrentHashMap<>(); + + Map layersRenamed = new ConcurrentHashMap<>(); + + List layersDeleted = new CopyOnWriteArrayList<>(); + Map> gridSubsetsDeleted = new ConcurrentHashMap<>(); + Map> parametersDeleted = new ConcurrentHashMap<>(); + + @Override + public void tileStored( + String layerName, + String gridSetId, + String blobFormat, + String parametersId, + long x, + long y, + int z, + long blobSize) { + add(tilesStored, layerName, gridSetId, blobFormat, parametersId, x, y, z, blobSize); + } + + @Override + public void tileDeleted( + String layerName, + String gridSetId, + String blobFormat, + String parametersId, + long x, + long y, + int z, + long blobSize) { + add(tilesDeleted, layerName, gridSetId, blobFormat, parametersId, x, y, z, blobSize); + } + + @Override + public void tileUpdated( + String layerName, + String gridSetId, + String blobFormat, + String parametersId, + long x, + long y, + int z, + long blobSize, + long oldSize) { + add(tilesUpdated, layerName, gridSetId, blobFormat, parametersId, x, y, z, blobSize); + } + + private void add( + Map target, + String layerName, + String gridSetId, + String blobFormat, + String parametersId, + long x, + long y, + int z, + long blobSize) { + + MimeType format; + try { + format = MimeType.createFromFormat(blobFormat); + } catch (MimeException e) { + throw new IllegalArgumentException(e); + } + + CacheId cache = new CacheId(layerName, layerName, gridSetId, format, parametersId); + TileIndex index = new TileIndex(x, y, z); + TileLocation tileLocation = new TileLocation(null, cache, index); + target.put(tileLocation, blobSize); + } + + @Override + public void layerDeleted(String layerName) { + layersDeleted.add(layerName); + } + + @Override + public void layerRenamed(String oldLayerName, String newLayerName) { + layersRenamed.put(oldLayerName, newLayerName); + } + + @Override + public void gridSubsetDeleted(String layerName, String gridSetId) { + gridSubsetsDeleted + .computeIfAbsent(layerName, l -> new CopyOnWriteArrayList<>()) + .add(gridSetId); + } + + @Override + public void parametersDeleted(String layerName, String parametersId) { + parametersDeleted + .computeIfAbsent(layerName, l -> new CopyOnWriteArrayList<>()) + .add(parametersId); + } +} diff --git a/geowebcache/gcsblob/src/test/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageBlobStoreConformanceIT.java b/geowebcache/gcsblob/src/test/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageBlobStoreConformanceIT.java new file mode 100644 index 000000000..f6a8a3f70 --- /dev/null +++ b/geowebcache/gcsblob/src/test/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageBlobStoreConformanceIT.java @@ -0,0 +1,123 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General + * Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this program. If not, see + * . + * + * @author Gabriel Roldan, Camptocamp, Copyright 2025 + */ +package org.geowebcache.storage.blobstore.gcs; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.awaitility.Awaitility.await; +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.eq; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; +import java.util.Collections; +import java.util.stream.Stream; +import org.easymock.EasyMock; +import org.geowebcache.GeoWebCacheException; +import org.geowebcache.config.DefaultGridsets; +import org.geowebcache.grid.GridSet; +import org.geowebcache.layer.TileLayer; +import org.geowebcache.layer.TileLayerDispatcher; +import org.geowebcache.mime.ImageMime; +import org.geowebcache.storage.AbstractBlobStoreTest; +import org.geowebcache.storage.StorageException; +import org.geowebcache.storage.TileRange; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; + +public class GoogleCloudStorageBlobStoreConformanceIT extends AbstractBlobStoreTest { + + @ClassRule + public static GoogleCloudStorageContainerSupport containerSupport = new GoogleCloudStorageContainerSupport(); + + /** + * Used to create a new {@link GoogleCloudStorageBlobStore} with a different prefix for each test at + * {@link #createTestUnit()} to isolate tests from each other + */ + @Rule + public TestName testName = new TestName(); + + @Override + @Before + public void createTestUnit() throws Exception { + TileLayerDispatcher layers = createMockLayerDispatcher(); + + String prefix = testName.getMethodName(); + super.store = containerSupport.createBlobStore(prefix, layers); + } + + private TileLayerDispatcher createMockLayerDispatcher() { + TileLayerDispatcher layers = createMock(TileLayerDispatcher.class); + Stream.of("testLayer", "testLayer1", "testLayer2") + .map(name -> { + TileLayer mock = createMock(name, TileLayer.class); + expect(mock.getName()).andStubReturn(name); + expect(mock.getId()).andStubReturn(name); + expect(mock.getGridSubsets()).andStubReturn(Collections.singleton("testGridSet")); + expect(mock.getMimeTypes()).andStubReturn(Arrays.asList(org.geowebcache.mime.ImageMime.png)); + try { + expect(layers.getTileLayer(eq(name))).andStubReturn(mock); + } catch (GeoWebCacheException e) { + throw new IllegalStateException(e); + } + return mock; + }) + .forEach(EasyMock::replay); + replay(layers); + return layers; + } + + @Test + public void testDeleteRangeWithListener() throws StorageException { + TileLayer layer = EasyMock.createNiceMock("layer", TileLayer.class); + final String layerName = "testLayer"; + EasyMock.expect(layer.getName()).andStubReturn(layerName); + GridSet gridSet = new DefaultGridsets(true, false).worldEpsg4326(); + final String format = ImageMime.png.getFormat(); + String content = "sample".repeat(1000); + String gridsetId = gridSet.getName(); + + CollectingListener listener = new CollectingListener(); + store.addListener(listener); + + // store full world coverage for zoom levels 0...2 + setupFullCoverage(layerName, gridSet, format, content, gridsetId, 0, 2); + + // delete full range at zoom level 2 + int z = 2; + long tilesWide = gridSet.getGrid(z).getNumTilesWide(); + long tilesHigh = gridSet.getGrid(z).getNumTilesHigh(); + + long[][] rangeBounds = {{0, 0, tilesWide - 1, tilesHigh - 1, 2}}; + TileRange range = new TileRange(layerName, gridsetId, 2, 2, rangeBounds, ImageMime.png, null); + + store.delete(range); + + final int expectedDeletes = tileCount(range); + await().atMost(10, SECONDS).untilAsserted(() -> assertEquals(expectedDeletes, listener.tilesDeleted.size())); + + // check tiles in range have have been deleted, but others are there + assertTileRangeEmpty(layerName, gridSet, format, range); + assertTile(layerName, 0, 0, 0, gridsetId, format, null, content); + assertTile(layerName, 1, 0, 0, gridsetId, format, null, content); + } + + private int tileCount(TileRange range) { + return (int) GoogleCloudStorageBlobStore.toTileIndices(range).count(); + } +} diff --git a/geowebcache/gcsblob/src/test/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageBlobStoreSuitabilityIT.java b/geowebcache/gcsblob/src/test/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageBlobStoreSuitabilityIT.java new file mode 100644 index 000000000..58964121a --- /dev/null +++ b/geowebcache/gcsblob/src/test/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageBlobStoreSuitabilityIT.java @@ -0,0 +1,90 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General + * Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this program. If not, see + * . + * + * @author Gabriel Roldan, Camptocamp, Copyright 2025 + */ +package org.geowebcache.storage.blobstore.gcs; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItemInArray; +import static org.junit.Assert.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.util.UUID; +import org.easymock.EasyMock; +import org.geowebcache.layer.TileLayerDispatcher; +import org.geowebcache.locks.LockProvider; +import org.geowebcache.locks.NoOpLockProvider; +import org.geowebcache.storage.BlobStore; +import org.geowebcache.storage.BlobStoreSuitabilityTest; +import org.hamcrest.Matcher; +import org.hamcrest.Matchers; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.experimental.theories.DataPoints; +import org.junit.rules.TestName; + +public class GoogleCloudStorageBlobStoreSuitabilityIT extends BlobStoreSuitabilityTest { + + @ClassRule + public static GoogleCloudStorageContainerSupport containerSupport = new GoogleCloudStorageContainerSupport(); + + @Rule + public TestName testName = new TestName(); + + @DataPoints + public static String[][] persistenceLocations = { + {}, + {"metadata.properties"}, + {"something"}, + {"something", "metadata.properties"}, + {"something/metadata.properties"} + }; + + TileLayerDispatcher tld; + LockProvider locks; + + @Before + public void setup() throws Exception { + tld = EasyMock.createMock("tld", TileLayerDispatcher.class); + locks = new NoOpLockProvider(); + EasyMock.replay(tld); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + @Override + protected Matcher existing() { + return (Matcher) hasItemInArray(equalTo("metadata.properties")); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + @Override + protected Matcher empty() { + return (Matcher) Matchers.emptyArray(); + } + + @Override + public BlobStore create(Object dir) throws Exception { + // Use unique prefix per theory invocation to avoid interference between test runs + final String prefix = testName.getMethodName() + "-" + UUID.randomUUID(); + // Create a client just for uploading test files + GoogleCloudStorageClient uploadClient = containerSupport.createClient(prefix); + for (String path : (String[]) dir) { + String fullPath = prefix + "/" + path; + byte[] data = "testAbc".getBytes(StandardCharsets.UTF_8); + uploadClient.put(fullPath, data, "text/plain"); + assertTrue(uploadClient.get(fullPath).isPresent()); + } + // Now create the blob store which will perform suitability checks + return containerSupport.createBlobStore(prefix, tld); + } +} diff --git a/geowebcache/gcsblob/src/test/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageBlobStoreTest.java b/geowebcache/gcsblob/src/test/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageBlobStoreTest.java new file mode 100644 index 000000000..05dfb3551 --- /dev/null +++ b/geowebcache/gcsblob/src/test/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageBlobStoreTest.java @@ -0,0 +1,489 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General + * Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this program. If not, see + * . + * + * @author Gabriel Roldan, Camptocamp, Copyright 2025 + */ +package org.geowebcache.storage.blobstore.gcs; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.cloud.storage.Blob; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Properties; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.geowebcache.GeoWebCacheException; +import org.geowebcache.filter.parameters.ParametersUtils; +import org.geowebcache.io.ByteArrayResource; +import org.geowebcache.io.Resource; +import org.geowebcache.layer.TileLayer; +import org.geowebcache.layer.TileLayerDispatcher; +import org.geowebcache.mime.ImageMime; +import org.geowebcache.storage.CompositeBlobStore; +import org.geowebcache.storage.StorageException; +import org.geowebcache.storage.SuitabilityCheckRule; +import org.geowebcache.storage.TileObject; +import org.geowebcache.storage.TileRange; +import org.geowebcache.storage.UnsuitableStorageException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +public class GoogleCloudStorageBlobStoreTest { + + private GoogleCloudStorageClient mockClient; + private TileLayerDispatcher mockTld; + private TileLayer testLayer; + + private CollectingListener collectingListener = new CollectingListener(); + + @Rule + public SuitabilityCheckRule suitability = SuitabilityCheckRule.system(); + + @Before + public void setUp() throws StorageException, GeoWebCacheException { + mockClient = mock(GoogleCloudStorageClient.class); + when(mockClient.getLocation()).thenReturn("gs://test-bucket/test-prefix"); + when(mockClient.getPrefix()).thenReturn("test-prefix"); + when(mockClient.getBucket()).thenReturn("test-bucket"); + mockTld = mock(TileLayerDispatcher.class); + testLayer = mock(TileLayer.class); + when(testLayer.getId()).thenReturn("testLayer"); + when(mockTld.getTileLayer("testLayer")).thenReturn(testLayer); + } + + private GoogleCloudStorageBlobStore store() throws StorageException { + GoogleCloudStorageBlobStore store = new GoogleCloudStorageBlobStore(mockClient, mockTld); + store.addListener(collectingListener); + return store; + } + + @Test + public void testSuitability_None_EmptyDirectory() throws StorageException { + suitability.setValue(CompositeBlobStore.StoreSuitabilityCheck.NONE); + when(mockClient.directoryExists("test-prefix")).thenReturn(false); // empty + when(mockClient.blobExists("test-prefix/metadata.properties")).thenReturn(false); + // Should succeed + assertNotNull(store()); + } + + @Test + public void testSuitability_None_DirectoryWithFiles() throws StorageException { + suitability.setValue(CompositeBlobStore.StoreSuitabilityCheck.NONE); + when(mockClient.directoryExists("test-prefix")).thenReturn(true); // has files + when(mockClient.blobExists("test-prefix/metadata.properties")).thenReturn(false); // no metadata + // Should succeed (NONE mode allows anything) + assertNotNull(store()); + } + + @Test + public void testSuitability_None_DirectoryWithMetadata() throws StorageException { + suitability.setValue(CompositeBlobStore.StoreSuitabilityCheck.NONE); + when(mockClient.directoryExists("test-prefix")).thenReturn(true); + when(mockClient.blobExists("test-prefix/metadata.properties")).thenReturn(true); // has metadata + // Should succeed + assertNotNull(store()); + } + + @Test + public void testSuitability_Empty_EmptyDirectory() throws StorageException { + suitability.setValue(CompositeBlobStore.StoreSuitabilityCheck.EMPTY); + when(mockClient.directoryExists("test-prefix")).thenReturn(false); // empty + when(mockClient.blobExists("test-prefix/metadata.properties")).thenReturn(false); + // Should succeed + assertNotNull(store()); + } + + @Test + public void testSuitability_Empty_DirectoryWithFiles_NoMetadata() throws StorageException { + suitability.setValue(CompositeBlobStore.StoreSuitabilityCheck.EMPTY); + when(mockClient.directoryExists("test-prefix")).thenReturn(true); // not empty + when(mockClient.blobExists("test-prefix/metadata.properties")).thenReturn(false); + // Should fail - directory not empty + assertThrows(UnsuitableStorageException.class, this::store); + } + + @Test + public void testSuitability_Empty_DirectoryWithMetadata() throws StorageException { + suitability.setValue(CompositeBlobStore.StoreSuitabilityCheck.EMPTY); + when(mockClient.directoryExists("test-prefix")).thenReturn(true); // not empty + when(mockClient.blobExists("test-prefix/metadata.properties")).thenReturn(true); + // Should fail - directory not empty + assertThrows(UnsuitableStorageException.class, this::store); + } + + @Test + public void testSuitability_Existing_EmptyDirectory() throws StorageException { + suitability.setValue(CompositeBlobStore.StoreSuitabilityCheck.EXISTING); + when(mockClient.directoryExists("test-prefix")).thenReturn(false); // empty + when(mockClient.blobExists("test-prefix/metadata.properties")).thenReturn(false); + // Should succeed (fallthrough to EMPTY case) + assertNotNull(store()); + } + + @Test + public void testSuitability_Existing_DirectoryWithFiles_NoMetadata() throws StorageException { + suitability.setValue(CompositeBlobStore.StoreSuitabilityCheck.EXISTING); + when(mockClient.directoryExists("test-prefix")).thenReturn(true); // not empty + when(mockClient.blobExists("test-prefix/metadata.properties")).thenReturn(false); // no metadata + // Should fail - has files but no metadata + assertThrows(UnsuitableStorageException.class, this::store); + } + + @Test + public void testSuitability_Existing_DirectoryWithMetadata() throws StorageException { + suitability.setValue(CompositeBlobStore.StoreSuitabilityCheck.EXISTING); + when(mockClient.directoryExists("test-prefix")).thenReturn(true); + when(mockClient.blobExists("test-prefix/metadata.properties")).thenReturn(true); // has metadata + // Should succeed - is existing GWC cache + assertNotNull(store()); + } + + @Test + public void testLayerExists() throws StorageException, GeoWebCacheException { + suitability.setValue(CompositeBlobStore.StoreSuitabilityCheck.NONE); + when(mockClient.directoryExists("test-prefix/testLayer/")).thenReturn(true); + + GoogleCloudStorageBlobStore store = store(); + assertTrue(store.layerExists("testLayer")); + verify(mockClient).directoryExists("test-prefix/testLayer/"); + + TileLayer testLayer2 = mock(TileLayer.class); + when(testLayer2.getId()).thenReturn("testLayer2"); + when(mockTld.getTileLayer("testLayer2")).thenReturn(testLayer2); + assertFalse(store.layerExists("testLayer2")); + } + + @Test + public void testLayerExists_NotFound() throws StorageException { + suitability.setValue(CompositeBlobStore.StoreSuitabilityCheck.NONE); + when(mockClient.directoryExists("test-prefix")).thenReturn(false); + when(mockClient.directoryExists("test-prefix/testLayer")).thenReturn(false); + + GoogleCloudStorageBlobStore store = store(); + assertFalse(store.layerExists("testLayer")); + } + + @Test + public void testGet_ExistingTile() throws StorageException { + suitability.setValue(CompositeBlobStore.StoreSuitabilityCheck.NONE); + when(mockClient.directoryExists("test-prefix")).thenReturn(false); + + Blob mockBlob = mock(Blob.class); + byte[] content = "tile-data".getBytes(); + when(mockBlob.getContent()).thenReturn(content); + when(mockBlob.getUpdateTimeOffsetDateTime()).thenReturn(OffsetDateTime.now()); + when(mockClient.get(anyString())).thenReturn(Optional.of(mockBlob)); + + GoogleCloudStorageBlobStore store = store(); + TileObject tile = + TileObject.createQueryTileObject("testLayer", new long[] {0, 0, 0}, "EPSG:4326", "image/png", null); + + boolean found = store.get(tile); + + assertTrue(found); + assertNotNull(tile.getBlob()); + assertEquals(content.length, tile.getBlobSize()); + } + + @Test + public void testGet_NonExistingTile() throws StorageException { + suitability.setValue(CompositeBlobStore.StoreSuitabilityCheck.NONE); + when(mockClient.directoryExists("test-prefix")).thenReturn(false); + when(mockClient.get(anyString())).thenReturn(Optional.empty()); + + GoogleCloudStorageBlobStore store = store(); + TileObject tile = + TileObject.createQueryTileObject("testLayer", new long[] {0, 0, 0}, "EPSG:4326", "image/png", null); + + boolean found = store.get(tile); + + assertFalse(found); + assertNull(tile.getBlob()); + assertEquals(0, tile.getBlobSize()); + } + + @Test + public void testPut() throws StorageException { + suitability.setValue(CompositeBlobStore.StoreSuitabilityCheck.NONE); + when(mockClient.directoryExists("test-prefix")).thenReturn(false); + + GoogleCloudStorageBlobStore store = store(); + TileObject tile = TileObject.createCompleteTileObject( + "testLayer", + new long[] {0, 0, 0}, + "EPSG:4326", + "image/png", + null, + new ByteArrayResource("data".getBytes())); + + store.put(tile); + + verify(mockClient) + .put(eq("test-prefix/testLayer/EPSG:4326/png/default/0/0/0.png"), any(Resource.class), eq("image/png")); + assertEquals(1, collectingListener.tilesStored.size()); + } + + @Test + public void testPutOverrides() throws StorageException { + suitability.setValue(CompositeBlobStore.StoreSuitabilityCheck.NONE); + when(mockClient.directoryExists("test-prefix")).thenReturn(false); + + GoogleCloudStorageBlobStore store = store(); + TileObject tile = TileObject.createCompleteTileObject( + "testLayer", + new long[] {0, 0, 0}, + "EPSG:4326", + "image/png", + null, + new ByteArrayResource("data".getBytes())); + + store.put(tile); + + String tileKey = "test-prefix/testLayer/EPSG:4326/png/default/0/0/0.png"; + verify(mockClient).put(eq(tileKey), any(Resource.class), eq("image/png")); + + tile.setBlob(new ByteArrayResource("updated".getBytes())); + when(mockClient.getSize(tileKey)).thenReturn(OptionalLong.of(100)); + store.put(tile); + + assertEquals(1, collectingListener.tilesStored.size()); + assertEquals(1, collectingListener.tilesUpdated.size()); + } + + @Test + public void testDeleteLayer() throws StorageException { + suitability.setValue(CompositeBlobStore.StoreSuitabilityCheck.NONE); + when(mockClient.directoryExists("test-prefix")).thenReturn(false); + when(mockClient.deleteDirectory("test-prefix/testLayer")).thenReturn(true); + + GoogleCloudStorageBlobStore store = store(); + when(mockClient.deleteDirectory("test-prefix/testLayer/")).thenReturn(true); + + boolean deleted = store.delete("testLayer"); + + assertTrue(deleted); + verify(mockClient).deleteBlob("test-prefix/testLayer/metadata.properties"); + verify(mockClient).deleteDirectory("test-prefix/testLayer/"); + assertEquals(List.of("testLayer"), collectingListener.layersDeleted); + } + + @Test + public void testDeleteTile() throws StorageException { + suitability.setValue(CompositeBlobStore.StoreSuitabilityCheck.NONE); + when(mockClient.directoryExists("test-prefix")).thenReturn(false); + when(mockClient.deleteBlob(anyString())).thenReturn(true); + + GoogleCloudStorageBlobStore store = store(); + TileObject tile = + TileObject.createQueryTileObject("testLayer", new long[] {0, 0, 0}, "EPSG:4326", "image/png", null); + + String tileKey = "test-prefix/testLayer/EPSG:4326/png/default/0/0/0.png"; + when(mockClient.getSize(tileKey)).thenReturn(OptionalLong.of(100)); + + boolean deleted = store.delete(tile); + assertTrue(deleted); + verify(mockClient).deleteBlob(anyString()); + assertEquals(1, collectingListener.tilesDeleted.size()); + } + + @Test + public void testRename() throws StorageException { + suitability.setValue(CompositeBlobStore.StoreSuitabilityCheck.NONE); + when(mockClient.directoryExists("test-prefix")).thenReturn(false); + when(mockClient.directoryExists("oldLayer")).thenReturn(true); + + GoogleCloudStorageBlobStore store = store(); + boolean renamed = store.rename("oldLayer", "newLayer"); + + assertTrue(renamed); + // GCS store uses layer ID, so no actual rename needed + verify(mockClient, never()).deleteDirectory(anyString()); + } + + @Test + public void testDestroy() throws StorageException { + suitability.setValue(CompositeBlobStore.StoreSuitabilityCheck.NONE); + when(mockClient.directoryExists("test-prefix")).thenReturn(false); + + GoogleCloudStorageBlobStore store = store(); + store.destroy(); + + verify(mockClient).close(); + } + + @Test + public void testDeleteByParametersId() throws StorageException { + when(testLayer.getGridSubsets()).thenReturn(Set.of("EPSG:4326", "EPSG:3857")); + when(testLayer.getMimeTypes()).thenReturn(List.of(ImageMime.png, ImageMime.jpeg)); + + GoogleCloudStorageBlobStore store = store(); + + when(mockClient.deleteDirectory("test-prefix/testLayer/EPSG:4326/png/test-params-id/")) + .thenReturn(true); + + assertTrue(store.deleteByParametersId("testLayer", "test-params-id")); + + verify(mockClient).deleteDirectory("test-prefix/testLayer/EPSG:4326/png/test-params-id/"); + verify(mockClient).deleteDirectory("test-prefix/testLayer/EPSG:3857/jpeg/test-params-id/"); + verify(mockClient).deleteDirectory("test-prefix/testLayer/EPSG:3857/png/test-params-id/"); + verify(mockClient).deleteDirectory("test-prefix/testLayer/EPSG:4326/jpeg/test-params-id/"); + + Map> expected = Map.of("testLayer", List.of("test-params-id")); + assertEquals(expected, collectingListener.parametersDeleted); + } + + @Test + public void testDeleteByGridsetId() throws StorageException { + when(testLayer.getGridSubsets()).thenReturn(Set.of("EPSG:4326", "EPSG:3857")); + + when(mockClient.deleteDirectory("test-prefix/testLayer/EPSG:4326/")).thenReturn(true); + when(mockClient.deleteDirectory("test-prefix/testLayer/EPSG:3857/")).thenReturn(true); + + GoogleCloudStorageBlobStore store = store(); + assertTrue(store.deleteByGridsetId("testLayer", "EPSG:4326")); + assertTrue(store.deleteByGridsetId("testLayer", "EPSG:3857")); + + Map> expected = Map.of("testLayer", List.of("EPSG:4326", "EPSG:3857")); + assertEquals(expected, collectingListener.gridSubsetsDeleted); + } + + @Test + @SuppressWarnings("unchecked") + public void testDeleteTileRange() throws StorageException { + long[][] rangeBounds = {{0, 0, 4 - 1, 4 - 1, 2}}; + TileRange range = new TileRange("testLayer", "EPSG:4326", 2, 2, rangeBounds, ImageMime.png, null); + + when(mockClient.directoryExists("test-prefix/testLayer/EPSG:4326/png/default")) + .thenReturn(true); + + // Capture the arguments passed to client.delete() + ArgumentCaptor> streamCaptor = ArgumentCaptor.forClass(Stream.class); + ArgumentCaptor> callbackCaptor = ArgumentCaptor.forClass(BiConsumer.class); + + GoogleCloudStorageBlobStore store = store(); + assertTrue(store.delete(range)); + + verify(mockClient).directoryExists("test-prefix/testLayer/EPSG:4326/png/default"); + verify(mockClient).delete(streamCaptor.capture(), callbackCaptor.capture()); + + // Verify the stream contains the expected tiles + List tiles = streamCaptor.getValue().collect(Collectors.toList()); + assertEquals(16, tiles.size()); // 4x4 grid at zoom level 2 + + // Verify the callback works - it should notify the listener + // simulate deleteService.submit(() -> deleteInternal(keys, callback)); at client.delete(Stream, Biconsumer); + BiConsumer callback = callbackCaptor.getValue(); + + tiles.forEach(tile -> callback.accept(tile, 100L)); + + // Verify the listener was notified + assertEquals(16, collectingListener.tilesDeleted.size()); + } + + @Test + public void testGetLayerMetadata() throws StorageException { + GoogleCloudStorageBlobStore store = store(); + when(mockClient.get("test-prefix/testLayer/metadata.properties")).thenReturn(Optional.empty()); + assertNull(store.getLayerMetadata("testLayer", "testKey")); + + Blob mockBlob = mock(Blob.class); + byte[] value = "testKey=testValue".getBytes(StandardCharsets.UTF_8); + when(mockBlob.getContent()).thenReturn(value); + when(mockClient.get("test-prefix/testLayer/metadata.properties")).thenReturn(Optional.of(mockBlob)); + + assertEquals("testValue", store.getLayerMetadata("testLayer", "testKey")); + } + + @Test + public void testPutLayerMetadata() throws IOException { + mockBlob("test-prefix/testLayer/metadata.properties", "existingKey=testValue"); + + ArgumentCaptor contentCaptor = ArgumentCaptor.forClass(byte[].class); + + GoogleCloudStorageBlobStore store = store(); + store.putLayerMetadata("testLayer", "newKey", "value2"); + + verify(mockClient) + .put(eq("test-prefix/testLayer/metadata.properties"), contentCaptor.capture(), eq("text/plain")); + + byte[] content = contentCaptor.getValue(); + Properties props = new Properties(); + props.load(new ByteArrayInputStream(content)); + assertEquals("testValue", props.getProperty("existingKey")); + assertEquals("value2", props.getProperty("newKey")); + } + + @Test + public void testGetParametersMapping() throws StorageException { + GoogleCloudStorageBlobStore store = store(); + + Map>> mappings = store.getParametersMapping("testLayer"); + assertNotNull(mappings); + assertTrue(mappings.isEmpty()); + + final String paramsPrefix = "test-prefix/testLayer/parameters-"; + + Map params1 = Map.of("key1", "value1"); + Map params2 = Map.of("key2", "value2"); + + String paramsId1 = ParametersUtils.getId(params1); + String paramsId2 = ParametersUtils.getId(params2); + + Blob paramsBlob1 = mockBlob(paramsPrefix + paramsId1, "key1=value1"); + Blob paramsBlob2 = mockBlob(paramsPrefix + paramsId2, "key2=value2"); + + when(mockClient.list(paramsPrefix)).thenReturn(Stream.of(paramsBlob1, paramsBlob2)); + + mappings = store.getParametersMapping("testLayer"); + assertNotNull(mappings); + assertEquals(Optional.of(params1), mappings.get(paramsId1)); + assertEquals(Optional.of(params2), mappings.get(paramsId2)); + } + + @Test + public void testClearUnsupported() throws StorageException { + GoogleCloudStorageBlobStore store = store(); + assertThrows(UnsupportedOperationException.class, store::clear); + } + + private Blob mockBlob(String key, String val) throws StorageException { + Blob mockBlob = mock(Blob.class); + byte[] value = val.getBytes(StandardCharsets.UTF_8); + when(mockBlob.getContent()).thenReturn(value); + when(mockClient.get(key)).thenReturn(Optional.of(mockBlob)); + when(mockBlob.getSize()).thenReturn((long) value.length); + when(mockBlob.getName()).thenReturn(key); + return mockBlob; + } +} diff --git a/geowebcache/gcsblob/src/test/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageConfigProviderTest.java b/geowebcache/gcsblob/src/test/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageConfigProviderTest.java new file mode 100644 index 000000000..c01846e6f --- /dev/null +++ b/geowebcache/gcsblob/src/test/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageConfigProviderTest.java @@ -0,0 +1,117 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General + * Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this program. If not, see + * . + * + * @author Gabriel Roldan, Camptocamp, Copyright 2025 + */ +package org.geowebcache.storage.blobstore.gcs; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.thoughtworks.xstream.XStream; +import java.io.StringReader; +import java.util.HashMap; +import java.util.Map; +import org.geowebcache.GeoWebCacheEnvironment; +import org.geowebcache.config.FileBlobStoreInfo; +import org.geowebcache.storage.StorageException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.context.support.ClassPathXmlApplicationContext; + +public class GoogleCloudStorageConfigProviderTest { + + private ClassPathXmlApplicationContext context; + + private Map properties; + + @Before + public void setUp() throws Exception { + properties = new HashMap<>(); + setProperty("ALLOW_ENV_PARAMETRIZATION", "true"); + context = new ClassPathXmlApplicationContext("appContextTestGcs.xml"); + } + + private void setProperty(String name, String value) { + System.setProperty(name, value); + properties.put(name, value); + } + + @After + public void tearDown() throws Exception { + properties.keySet().forEach(System::clearProperty); + } + + @Test + public void testCanSave() { + GoogleCloudStorageConfigProvider provider = context.getBean(GoogleCloudStorageConfigProvider.class); + assertFalse(provider.canSave(null)); + assertFalse(provider.canSave(new FileBlobStoreInfo())); + assertTrue(provider.canSave(new GoogleCloudStorageBlobStoreInfo())); + } + + @Test + public void testValuesFromEnvironment() throws StorageException { + GoogleCloudStorageConfigProvider provider = new GoogleCloudStorageConfigProvider(); + XStream stream = new XStream(); + stream = provider.getConfiguredXStream(stream); + String xml = + """ + + myGcsBlobStore + true + ${GCS_PROJECT_ID} + ${GCS_QUOTA_PROJECT_ID} + ${GCS_BUCKET} + ${GCS_PREFIX} + ${GCS_ENDPOINT} + ${GCS_APIKEY} + ${GCS_DEFAULT_CREDENTIALS} + + """; + + Object parsed = stream.fromXML(new StringReader(xml)); + assertTrue(parsed instanceof GoogleCloudStorageBlobStoreInfo); + + GoogleCloudStorageBlobStoreInfo config = (GoogleCloudStorageBlobStoreInfo) parsed; + assertEquals("myGcsBlobStore", config.getId()); + assertEquals("myGcsBlobStore", config.getName()); + assertTrue(config.isEnabled()); + + assertEquals("${GCS_PROJECT_ID}", config.getProjectId()); + assertEquals("${GCS_QUOTA_PROJECT_ID}", config.getQuotaProjectId()); + assertEquals("${GCS_BUCKET}", config.getBucket()); + assertEquals("${GCS_PREFIX}", config.getPrefix()); + assertEquals("${GCS_ENDPOINT}", config.getEndpointUrl()); + assertEquals("${GCS_APIKEY}", config.getApiKey()); + assertEquals("${GCS_DEFAULT_CREDENTIALS}", config.getUseDefaultCredentialsChain()); + + setProperty("GCS_PROJECT_ID", "my-project"); + setProperty("GCS_QUOTA_PROJECT_ID", "bills-go-here"); + setProperty("GCS_BUCKET", "test-bucket"); + setProperty("GCS_PREFIX", "gwc"); + setProperty("GCS_ENDPOINT", "https://goog.compatible.storage/api"); + setProperty("GCS_APIKEY", "goog-fake-api-key"); + setProperty("GCS_DEFAULT_CREDENTIALS", "true"); + + GeoWebCacheEnvironment environment = context.getBean(GeoWebCacheEnvironment.class); + GoogleCloudStorageClient.Builder builder = GoogleCloudStorageClient.builder(config, environment); + assertEquals("my-project", builder.projectId); + assertEquals("bills-go-here", builder.quotaProjectId); + assertEquals("test-bucket", builder.bucket); + assertEquals("gwc", builder.prefix); + assertEquals("https://goog.compatible.storage/api", builder.endpointUrl); + assertEquals("goog-fake-api-key", builder.apiKey); + assertTrue(builder.useDefaultCredentialsChain); + } +} diff --git a/geowebcache/gcsblob/src/test/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageContainerSupport.java b/geowebcache/gcsblob/src/test/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageContainerSupport.java new file mode 100644 index 000000000..d82c43b0a --- /dev/null +++ b/geowebcache/gcsblob/src/test/java/org/geowebcache/storage/blobstore/gcs/GoogleCloudStorageContainerSupport.java @@ -0,0 +1,127 @@ +/** + * This program is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General + * Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + *

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + * + *

You should have received a copy of the GNU Lesser General Public License along with this program. If not, see + * . + * + * @author Gabriel Roldan, Camptocamp, Copyright 2025 + */ +package org.geowebcache.storage.blobstore.gcs; + +import com.google.cloud.storage.BucketInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import io.aiven.testcontainers.fakegcsserver.FakeGcsServerContainer; +import javax.annotation.Nullable; +import org.geowebcache.GeoWebCacheEnvironment; +import org.geowebcache.layer.TileLayerDispatcher; +import org.geowebcache.storage.StorageException; +import org.junit.Assume; +import org.junit.rules.ExternalResource; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.utility.DockerImageName; + +public class GoogleCloudStorageContainerSupport extends ExternalResource { + + private static final String BUCKET_NAME = "test-bucket"; + private static final String PROJECT_ID = "test-project"; + + public static final DockerImageName IMAGE_NAME = DockerImageName.parse("fsouza/fake-gcs-server:latest"); + + private FakeGcsServerContainer gcsEmulator; + + private String emulatorEndpoint; + + @Override + protected void after() { + if (gcsEmulator != null) { + gcsEmulator.stop(); + gcsEmulator = null; + } + } + + @Override + protected void before() throws Throwable { + // Skip tests if Docker is not available + Assume.assumeTrue( + "Docker is not available, skipping Testcontainers tests.", + DockerClientFactory.instance().isDockerAvailable()); + + gcsEmulator = new FakeGcsServerContainer(IMAGE_NAME); + + // Wait for the emulator to be ready + gcsEmulator.start(); + + // Configure the Storage client to use the emulator + String emulatorHost = gcsEmulator.getHost(); + Integer emulatorPort = gcsEmulator.getFirstMappedPort(); + emulatorEndpoint = "http://" + emulatorHost + ":" + emulatorPort; + + // Create Storage client + try (Storage storage = StorageOptions.newBuilder() + .setProjectId(PROJECT_ID) + .setHost(emulatorEndpoint) + .build() + .getService()) { + + // Create a bucket + BucketInfo bucketInfo = BucketInfo.newBuilder(BUCKET_NAME).build(); + storage.create(bucketInfo); + } + } + + public String getEmulatorEndpoint() { + return emulatorEndpoint; + } + + public String getBucket() { + return BUCKET_NAME; + } + + public String getProjectId() { + return PROJECT_ID; + } + + public GoogleCloudStorageBlobStoreInfo createBlobstoreInfo() { + GoogleCloudStorageBlobStoreInfo info = new GoogleCloudStorageBlobStoreInfo(); + + info.setEndpointUrl(getEmulatorEndpoint()); + info.setBucket(getBucket()); + info.setProjectId(getProjectId()); + return info; + } + + public GoogleCloudStorageBlobStoreInfo createBlobstoreInfo(String prefix) { + GoogleCloudStorageBlobStoreInfo info = createBlobstoreInfo(); + info.setPrefix(prefix); + return info; + } + + public GoogleCloudStorageBlobStore createBlobStore(TileLayerDispatcher layers) throws StorageException { + return createBlobStore(null, layers); + } + + /** + * @param prefix the blob name prefix applied to the cache, if {@code null}, the cache operates at the bucket's root + * @param layers + * @return + * @throws StorageException + */ + public GoogleCloudStorageBlobStore createBlobStore(@Nullable String prefix, TileLayerDispatcher layers) + throws StorageException { + GoogleCloudStorageClient client = createClient(prefix); + + return new GoogleCloudStorageBlobStore(client, layers); + } + + public GoogleCloudStorageClient createClient(String prefix) throws StorageException { + GeoWebCacheEnvironment env = new GeoWebCacheEnvironment(); + GoogleCloudStorageBlobStoreInfo info = createBlobstoreInfo(prefix); + return GoogleCloudStorageClient.builder(info, env).build(); + } +} diff --git a/geowebcache/gcsblob/src/test/resources/appContextTestGcs.xml b/geowebcache/gcsblob/src/test/resources/appContextTestGcs.xml new file mode 100644 index 000000000..c75b7908c --- /dev/null +++ b/geowebcache/gcsblob/src/test/resources/appContextTestGcs.xml @@ -0,0 +1,22 @@ + + + + + + + Bean configuration file for the gwc-gcs-blob tests + + + + + + + + \ No newline at end of file diff --git a/geowebcache/pom.xml b/geowebcache/pom.xml index ee905967d..ce896149c 100644 --- a/geowebcache/pom.xml +++ b/geowebcache/pom.xml @@ -29,6 +29,7 @@ azureblob swiftblob mbtiles + gcsblob