From 4e6eb18352824706d88102efdb13986af77380d1 Mon Sep 17 00:00:00 2001 From: Yavor16 Date: Mon, 10 Nov 2025 10:45:40 +0200 Subject: [PATCH] Revert "Add GCP Objectstore SDK instead of JClouds (#1723)" This reverts commit 54d7e0d75bea0a5969a1231b1efc35e5401d1c6d. --- .../health/ApplicationHealthCalculator.java | 7 +- .../ApplicationHealthCalculatorTest.java | 20 +- multiapps-controller-persistence/pom.xml | 23 +- .../src/main/java/module-info.java | 7 +- .../services/GcpObjectStoreFileStorage.java | 237 ------------------ ...orage.java => ObjectStoreFileStorage.java} | 95 +++++-- .../persistence/util/ObjectStoreFilter.java | 58 ----- .../persistence/util/ObjectStoreMapper.java | 35 --- .../GcpObjectStoreFileStorageTest.java | 95 ------- ...t.java => ObjectStoreFileStorageTest.java} | 56 ++--- .../util/ObjectStoreFilterTest.java | 79 ------ .../util/ObjectStoreMapperTest.java | 52 ---- multiapps-controller-web/pom.xml | 8 - .../bean/factory/FileServiceFactoryBean.java | 10 +- .../ObjectStoreFileStorageFactoryBean.java | 116 +++------ .../service/ObjectStoreServiceInfo.java | 1 + .../ObjectStoreServiceInfoCreator.java | 36 ++- ...ObjectStoreFileStorageFactoryBeanTest.java | 50 +--- .../ObjectStoreServiceInfoCreatorTest.java | 45 +++- pom.xml | 13 - 20 files changed, 253 insertions(+), 790 deletions(-) delete mode 100644 multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java rename multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/{JCloudsObjectStoreFileStorage.java => ObjectStoreFileStorage.java} (68%) delete mode 100644 multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreFilter.java delete mode 100644 multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapper.java delete mode 100644 multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java rename multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/{JCloudsObjectStoreFileStorageTest.java => ObjectStoreFileStorageTest.java} (88%) delete mode 100644 multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreFilterTest.java delete mode 100644 multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapperTest.java diff --git a/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculator.java b/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculator.java index 70a85311c2..40272e7d90 100644 --- a/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculator.java +++ b/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculator.java @@ -15,7 +15,6 @@ import java.util.concurrent.TimeoutException; import java.util.function.Consumer; import java.util.function.Supplier; - import jakarta.inject.Inject; import jakarta.inject.Named; import org.cloudfoundry.multiapps.common.SLException; @@ -30,7 +29,7 @@ import org.cloudfoundry.multiapps.controller.core.util.ApplicationInstanceNameUtil; import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseHealthService; import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseMonitoringService; -import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; +import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -47,7 +46,7 @@ public class ApplicationHealthCalculator { // timeout private static final int TOTAL_TASK_TIMEOUT_IN_SECONDS = 3 * SINGLE_TASK_TIMEOUT_IN_SECONDS; - private final FileStorage objectStoreFileStorage; + private final ObjectStoreFileStorage objectStoreFileStorage; private final ApplicationConfiguration applicationConfiguration; private final DatabaseHealthService databaseHealthService; private final DatabaseMonitoringService databaseMonitoringService; @@ -75,7 +74,7 @@ public class ApplicationHealthCalculator { private final ResilientOperationExecutor resilientOperationExecutor = getResilienceExecutor(); @Inject - public ApplicationHealthCalculator(@Autowired(required = false) FileStorage objectStoreFileStorage, + public ApplicationHealthCalculator(@Autowired(required = false) ObjectStoreFileStorage objectStoreFileStorage, ApplicationConfiguration applicationConfiguration, DatabaseHealthService databaseHealthService, DatabaseMonitoringService databaseMonitoringService, DatabaseWaitingLocksAnalyzer databaseWaitingLocksAnalyzer) { diff --git a/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculatorTest.java b/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculatorTest.java index 8a67a4bda3..e067eb7e88 100644 --- a/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculatorTest.java +++ b/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculatorTest.java @@ -1,5 +1,9 @@ package org.cloudfoundry.multiapps.controller.core.application.health; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + import org.cloudfoundry.multiapps.common.SLException; import org.cloudfoundry.multiapps.controller.client.util.ResilientOperationExecutor; import org.cloudfoundry.multiapps.controller.core.application.health.database.DatabaseWaitingLocksAnalyzer; @@ -7,7 +11,7 @@ import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseHealthService; import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseMonitoringService; -import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage; +import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; @@ -16,14 +20,10 @@ import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - class ApplicationHealthCalculatorTest { @Mock - private JCloudsObjectStoreFileStorage jCloudsObjectStoreFileStorage; + private ObjectStoreFileStorage objectStoreFileStorage; @Mock private ApplicationConfiguration applicationConfiguration; @Mock @@ -41,7 +41,7 @@ void setUp() throws Exception { .close(); Mockito.when(applicationConfiguration.isHealthCheckEnabled()) .thenReturn(true); - applicationHealthCalculator = new ApplicationHealthCalculatorMock(jCloudsObjectStoreFileStorage, + applicationHealthCalculator = new ApplicationHealthCalculatorMock(objectStoreFileStorage, applicationConfiguration, databaseHealthService, databaseMonitoringService, @@ -51,7 +51,7 @@ void setUp() throws Exception { @Test void testUpdateWithFailingObjectStore() { Mockito.doThrow(new SLException("Object store not working")) - .when(jCloudsObjectStoreFileStorage) + .when(objectStoreFileStorage) .testConnection(); applicationHealthCalculator.updateHealthStatus(); ResponseEntity applicationHealthResultResponseEntity = applicationHealthCalculator.calculateApplicationHealth(); @@ -117,12 +117,12 @@ void testSuccessfulUpdateWithMissingObjectStore() { } private static class ApplicationHealthCalculatorMock extends ApplicationHealthCalculator { - public ApplicationHealthCalculatorMock(JCloudsObjectStoreFileStorage JCloudsObjectStoreFileStorage, + public ApplicationHealthCalculatorMock(ObjectStoreFileStorage objectStoreFileStorage, ApplicationConfiguration applicationConfiguration, DatabaseHealthService databaseHealthService, DatabaseMonitoringService databaseMonitoringService, DatabaseWaitingLocksAnalyzer databaseWaitingLocksAnalyzer) { - super(JCloudsObjectStoreFileStorage, + super(objectStoreFileStorage, applicationConfiguration, databaseHealthService, databaseMonitoringService, diff --git a/multiapps-controller-persistence/pom.xml b/multiapps-controller-persistence/pom.xml index 6a28c8b0e9..05f592ab36 100644 --- a/multiapps-controller-persistence/pom.xml +++ b/multiapps-controller-persistence/pom.xml @@ -99,18 +99,16 @@ azureblob - org.apache.jclouds - jclouds-blobstore - - - org.apache.jclouds.provider - google-cloud-storage - - + org.apache.jclouds.provider + google-cloud-storage - com.google.cloud - google-cloud-storage + org.apache.jclouds.common + googlecloud + + + org.apache.jclouds + jclouds-blobstore com.aliyun.oss @@ -153,11 +151,6 @@ org.apache.logging.log4j log4j-core - - com.google.cloud - google-cloud-nio - compile - org.slf4j slf4j-api diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index 875fb82e2d..a1ce012fad 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -30,8 +30,6 @@ requires aliyun.sdk.oss; requires com.fasterxml.jackson.annotation; requires com.fasterxml.jackson.databind; - requires com.google.auth; - requires com.google.auth.oauth2; requires com.google.auto.service; requires com.google.common; requires com.google.guice; @@ -39,10 +37,6 @@ requires flowable.engine; requires flowable.engine.common.api; requires flowable.variable.service.api; - requires gax; - requires google.cloud.core; - requires google.cloud.nio; - requires google.cloud.storage; requires jakarta.inject; requires org.apache.logging.log4j; requires org.apache.logging.log4j.core; @@ -59,4 +53,5 @@ requires static org.immutables.value; requires jakarta.xml.bind; requires org.bouncycastle.fips.pkix; + } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java deleted file mode 100644 index ab81b1750f..0000000000 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java +++ /dev/null @@ -1,237 +0,0 @@ -package org.cloudfoundry.multiapps.controller.persistence.services; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.channels.Channels; -import java.text.MessageFormat; -import java.time.Duration; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Base64; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import com.google.api.gax.retrying.RetrySettings; -import com.google.auth.Credentials; -import com.google.auth.oauth2.GoogleCredentials; -import com.google.cloud.ReadChannel; -import com.google.cloud.storage.Blob; -import com.google.cloud.storage.BlobId; -import com.google.cloud.storage.BlobInfo; -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.StorageException; -import com.google.cloud.storage.StorageOptions; -import com.google.cloud.storage.StorageRetryStrategy; -import org.cloudfoundry.multiapps.common.util.MiscUtil; -import org.cloudfoundry.multiapps.controller.persistence.Messages; -import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; -import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; -import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; -import org.springframework.http.MediaType; - -public class GcpObjectStoreFileStorage implements FileStorage { - - private final String bucketName; - private final Storage storage; - private static final String BUCKET = "bucket"; - private static final int OBJECT_STORE_MAX_ATTEMPTS_CONFIG = 6; - private static final double OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG = 2.0; - private static final Duration OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES = Duration.ofMinutes(10); - private static final Duration OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS = Duration.ofSeconds(10); - private static final Duration OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS = Duration.ofMillis(250); - private static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData"; - - public GcpObjectStoreFileStorage(Map credentials) { - this.bucketName = (String) credentials.get(BUCKET); - this.storage = createObjectStoreStorage(credentials); - } - - protected Storage createObjectStoreStorage(Map credentials) { - return StorageOptions.http() - .setCredentials(getGcpCredentialsSupplier(credentials)) - .setStorageRetryStrategy(StorageRetryStrategy.getDefaultStorageRetryStrategy()) - .setRetrySettings( - RetrySettings.newBuilder() - .setMaxAttempts(OBJECT_STORE_MAX_ATTEMPTS_CONFIG) - .setTotalTimeoutDuration(OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES) - .setMaxRetryDelayDuration(OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS) - .setInitialRetryDelayDuration(OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS) - .setRetryDelayMultiplier(OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG) - .build()) - .build() - .getService(); - } - - private Credentials getGcpCredentialsSupplier(Map credentials) { - if (!credentials.containsKey(BASE_64_ENCODED_PRIVATE_KEY_DATA)) { - return null; - } - byte[] decodedKey = Base64.getDecoder() - .decode((String) credentials.get(BASE_64_ENCODED_PRIVATE_KEY_DATA)); - try { - return GoogleCredentials.fromStream(new ByteArrayInputStream(decodedKey)); - } catch (IOException e) { - throw new IllegalStateException(e); - } - } - - @Override - public void addFile(FileEntry fileEntry, InputStream content) throws FileStorageException { - BlobId blobId = BlobId.of(bucketName, fileEntry.getId()); - BlobInfo blobInfo = BlobInfo.newBuilder(blobId) - .setContentDisposition(fileEntry.getName()) - .setContentType(MediaType.APPLICATION_OCTET_STREAM_VALUE) - .setMetadata(ObjectStoreMapper.createFileEntryMetadata(fileEntry)) - .build(); - - putBlob(blobInfo, content); - } - - private void putBlob(BlobInfo blobInfo, InputStream content) throws FileStorageException { - try { - storage.createFrom(blobInfo, content); - } catch (IOException | StorageException e) { - throw new FileStorageException(e); - } - } - - @Override - public List getFileEntriesWithoutContent(List fileEntries) { - Set existingFiles = getAllEntries().stream() - .map(Blob::getName) - .collect(Collectors.toSet()); - return fileEntries.stream() - .filter(fileEntry -> !existingFiles.contains(fileEntry.getId())) - .toList(); - } - - @Override - public void deleteFile(String id, String space) { - storage.delete(bucketName, id); - } - - @Override - public void deleteFilesBySpaceIds(List spaceIds) { - removeBlobsByFilter(blob -> ObjectStoreFilter.filterBySpaceIds(blob.getMetadata(), spaceIds)); - } - - @Override - public void deleteFilesBySpaceAndNamespace(String space, String namespace) { - removeBlobsByFilter(blob -> ObjectStoreFilter.filterBySpaceAndNamespace(blob.getMetadata(), space, namespace)); - } - - @Override - public int deleteFilesModifiedBefore(LocalDateTime modificationTime) { - return removeBlobsByFilter( - blob -> ObjectStoreFilter.filterByModificationTime(blob.getMetadata(), blob.getName(), modificationTime)); - } - - @Override - public T processFileContent(String space, String id, - FileContentProcessor fileContentProcessor) throws FileStorageException { - FileEntry fileEntry = ObjectStoreMapper.createFileEntry(space, id); - try (InputStream inputStream = openBlobStream(fileEntry)) { - return fileContentProcessor.process(inputStream); - } catch (Exception e) { - throw new FileStorageException(e); - } - } - - private InputStream openBlobStream(FileEntry fileEntry) throws FileStorageException { - Blob blob = getBlob(fileEntry); - return Channels.newInputStream(blob.reader()); - } - - private Blob getBlob(FileEntry fileEntry) throws FileStorageException { - try { - Blob blob = storage.get(bucketName, fileEntry.getId()); - if (blob == null) { - throw new FileStorageException( - MessageFormat.format(Messages.FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST, - fileEntry.getId(), fileEntry.getSpace())); - } - return blob; - } catch (StorageException e) { - throw new FileStorageException(e); - } - } - - @Override - public InputStream openInputStream(String space, String id) throws FileStorageException { - FileEntry fileEntry = ObjectStoreMapper.createFileEntry(space, id); - return openBlobStream(fileEntry); - } - - @Override - public void testConnection() { - storage.get(bucketName, "test"); - } - - @Override - public void deleteFilesByIds(List fileIds) { - removeBlobsByFilter(blob -> fileIds.contains(blob.getName())); - } - - @Override - public T processArchiveEntryContent(FileContentToProcess fileContentToProcess, FileContentProcessor fileContentProcessor) - throws FileStorageException { - FileEntry fileEntry = ObjectStoreMapper.createFileEntry(fileContentToProcess.getSpaceGuid(), fileContentToProcess.getGuid()); - InputStream blobPayload = getBlobPayloadWithOffset(fileEntry, fileContentToProcess.getStartOffset(), - fileContentToProcess.getEndOffset()); - return processContent(fileContentProcessor, blobPayload); - } - - private T processContent(FileContentProcessor fileContentProcessor, InputStream inputStream) - throws FileStorageException { - try { - return fileContentProcessor.process(inputStream); - } catch (IOException e) { - throw new FileStorageException(e); - } - } - - public Set getAllEntries() { - return storage.list(bucketName) - .streamAll() - .collect(Collectors.toSet()); - } - - protected int removeBlobsByFilter(Predicate filter) { - List blobIds = getEntryNames(filter).stream() - .map(entry -> BlobId.of(bucketName, entry)) - .toList(); - List deletedBlobsResults = new ArrayList<>(); - if (!blobIds.isEmpty()) { - deletedBlobsResults = storage.delete(blobIds); - } - return MiscUtil.cast(deletedBlobsResults.stream() - .filter(Boolean::booleanValue) - .count()); - } - - protected Set getEntryNames(Predicate filter) { - return storage.list(bucketName) - .streamAll() - .filter(filter) - .map(Blob::getName) - .collect(Collectors.toSet()); - } - - private InputStream getBlobPayloadWithOffset(FileEntry fileEntry, long startOffset, long endOffset) - throws FileStorageException { - try { - Blob blob = getBlob(fileEntry); - ReadChannel reader = storage.reader(blob.getBlobId()); - reader.seek(startOffset); - reader.limit(endOffset + 1); - - return Channels.newInputStream(reader); - } catch (IOException | StorageException e) { - throw new FileStorageException(e); - } - } -} diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java similarity index 68% rename from multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java rename to multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java index 2e5ec53acb..977b5968df 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java @@ -3,19 +3,23 @@ import java.io.IOException; import java.io.InputStream; import java.text.MessageFormat; +import java.time.Instant; import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import org.cloudfoundry.multiapps.common.util.MiscUtil; +import org.cloudfoundry.multiapps.controller.persistence.Constants; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; -import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; -import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; +import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.ContainerNotFoundException; import org.jclouds.blobstore.domain.Blob; @@ -29,17 +33,18 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; +import org.springframework.util.CollectionUtils; -public class JCloudsObjectStoreFileStorage implements FileStorage { +public class ObjectStoreFileStorage implements FileStorage { - private static final Logger LOGGER = LoggerFactory.getLogger(JCloudsObjectStoreFileStorage.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ObjectStoreFileStorage.class); private static final int MAX_RETRIES_COUNT = 3; private static final long RETRY_BASE_WAIT_TIME_IN_MILLIS = 5000L; private final BlobStore blobStore; private final String container; - public JCloudsObjectStoreFileStorage(BlobStore blobStore, String container) { + public ObjectStoreFileStorage(BlobStore blobStore, String container) { this.blobStore = blobStore; this.container = container; } @@ -54,10 +59,10 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage .contentDisposition(fileEntry.getName()) .contentType(MediaType.APPLICATION_OCTET_STREAM_VALUE) .contentLength(fileSize) - .userMetadata(ObjectStoreMapper.createFileEntryMetadata(fileEntry)) + .userMetadata(createFileEntryMetadata(fileEntry)) .build(); try { - putBlobWithRetries(blob, MAX_RETRIES_COUNT); + putBlobWithRetries(blob, 3); LOGGER.debug(MessageFormat.format(Messages.STORED_FILE_0_WITH_SIZE_1, fileEntry.getId(), fileSize)); } catch (ContainerNotFoundException e) { throw new FileStorageException(MessageFormat.format(Messages.FILE_UPLOAD_FAILED, fileEntry.getName(), @@ -82,23 +87,22 @@ public void deleteFile(String id, String space) { @Override public void deleteFilesBySpaceIds(List spaceIds) { - removeBlobsByFilter(blob -> ObjectStoreFilter.filterBySpaceIds(blob.getUserMetadata(), spaceIds)); + removeBlobsByFilter(blob -> filterBySpaceIds(blob, spaceIds)); } @Override public void deleteFilesBySpaceAndNamespace(String space, String namespace) { - removeBlobsByFilter(blob -> ObjectStoreFilter.filterBySpaceAndNamespace(blob.getUserMetadata(), space, namespace)); + removeBlobsByFilter(blob -> filterBySpaceAndNamespace(blob, space, namespace)); } @Override public int deleteFilesModifiedBefore(LocalDateTime modificationTime) { - return removeBlobsByFilter( - blob -> ObjectStoreFilter.filterByModificationTime(blob.getUserMetadata(), blob.getName(), modificationTime)); + return removeBlobsByFilter(blob -> filterByModificationTime(blob, modificationTime)); } @Override public T processFileContent(String space, String id, FileContentProcessor fileContentProcessor) throws FileStorageException { - FileEntry fileEntry = ObjectStoreMapper.createFileEntry(space, id); + FileEntry fileEntry = createFileEntry(space, id); try { Payload payload = getBlobPayload(fileEntry); return processContent(fileContentProcessor, payload); @@ -110,7 +114,7 @@ public T processFileContent(String space, String id, FileContentProcessor @Override public T processArchiveEntryContent(FileContentToProcess fileContentToProcess, FileContentProcessor fileContentProcessor) throws FileStorageException { - FileEntry fileEntry = ObjectStoreMapper.createFileEntry(fileContentToProcess.getSpaceGuid(), fileContentToProcess.getGuid()); + FileEntry fileEntry = createFileEntry(fileContentToProcess.getSpaceGuid(), fileContentToProcess.getGuid()); try { Payload payload = getBlobPayloadWithOffset(fileEntry, fileContentToProcess.getStartOffset(), fileContentToProcess.getEndOffset()); @@ -146,7 +150,7 @@ private Blob getBlobWithRetriesWithOffset(FileEntry fileEntry, int retries, long } private Payload getBlobPayload(FileEntry fileEntry) throws FileStorageException { - Blob blob = getBlobWithRetries(fileEntry, MAX_RETRIES_COUNT); + Blob blob = getBlobWithRetries(fileEntry, 3); if (blob == null) { throw new FileStorageException(MessageFormat.format(Messages.FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST, fileEntry.getId(), fileEntry.getSpace())); @@ -156,7 +160,7 @@ private Payload getBlobPayload(FileEntry fileEntry) throws FileStorageException @Override public InputStream openInputStream(String space, String id) throws FileStorageException { - FileEntry fileEntry = ObjectStoreMapper.createFileEntry(space, id); + FileEntry fileEntry = createFileEntry(space, id); Payload payload = getBlobPayload(fileEntry); return openPayloadInputStream(payload); } @@ -179,6 +183,13 @@ public void deleteFilesByIds(List fileIds) { removeBlobsByFilter(blob -> fileIds.contains(blob.getName())); } + private FileEntry createFileEntry(String space, String id) { + return ImmutableFileEntry.builder() + .space(space) + .id(id) + .build(); + } + private T processContent(FileContentProcessor fileContentProcessor, Payload payload) throws FileStorageException { try (InputStream fileContentStream = payload.openStream()) { return fileContentProcessor.process(fileContentStream); @@ -221,6 +232,19 @@ protected long getRetryWaitTime() { return RETRY_BASE_WAIT_TIME_IN_MILLIS; } + private Map createFileEntryMetadata(FileEntry fileEntry) { + Map metadata = new HashMap<>(); + metadata.put(Constants.FILE_ENTRY_SPACE.toLowerCase(), fileEntry.getSpace()); + metadata.put(Constants.FILE_ENTRY_MODIFIED.toLowerCase(), Long.toString(fileEntry.getModified() + .atZone(ZoneId.systemDefault()) + .toInstant() + .toEpochMilli())); + if (fileEntry.getNamespace() != null) { + metadata.put(Constants.FILE_ENTRY_NAMESPACE.toLowerCase(), fileEntry.getNamespace()); + } + return metadata; + } + private int removeBlobsByFilter(Predicate filter) { Set entries = getEntryNames(filter); if (!entries.isEmpty()) { @@ -247,4 +271,45 @@ private Set getAllEntries(ListContainerOptions options) { } return entries; } + + private boolean filterByModificationTime(StorageMetadata blobMetadata, LocalDateTime modificationTime) { + Map userMetadata = blobMetadata.getUserMetadata(); + // Clean up any blobStore entries that don't have any metadata as we can't check their creation date + if (CollectionUtils.isEmpty(userMetadata)) { + LOGGER.warn(MessageFormat.format(Messages.USER_METADATA_OF_BLOB_0_EMPTY_AND_WILL_BE_DELETED, blobMetadata.getName())); + return true; + } + String longString = userMetadata.get(Constants.FILE_ENTRY_MODIFIED.toLowerCase()); + try { + long dateLong = Long.parseLong(longString); + LocalDateTime date = LocalDateTime.ofInstant(Instant.ofEpochMilli(dateLong), ZoneId.systemDefault()); + return date.isBefore(modificationTime); + } catch (NumberFormatException e) { + // Clean up any blobStore entries that have invalid timestamp + LOGGER.warn(MessageFormat.format(Messages.DATE_METADATA_OF_BLOB_0_IS_NOT_IN_PROPER_FORMAT_AND_WILL_BE_DELETED, + blobMetadata.getName()), + e); + return true; + } + } + + private boolean filterBySpaceIds(StorageMetadata blobMetadata, List spaceIds) { + Map userMetadata = blobMetadata.getUserMetadata(); + if (CollectionUtils.isEmpty(userMetadata)) { + return false; + } + String spaceParameter = userMetadata.get(Constants.FILE_ENTRY_SPACE.toLowerCase()); + return spaceIds.contains(spaceParameter); + } + + private boolean filterBySpaceAndNamespace(StorageMetadata blobMetadata, String space, String namespace) { + Map userMetadata = blobMetadata.getUserMetadata(); + if (CollectionUtils.isEmpty(userMetadata)) { + return false; + } + String spaceParameter = userMetadata.get(Constants.FILE_ENTRY_SPACE.toLowerCase()); + String namespaceParameter = userMetadata.get(Constants.FILE_ENTRY_NAMESPACE.toLowerCase()); + return space.equals(spaceParameter) && namespace.equals(namespaceParameter); + } + } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreFilter.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreFilter.java deleted file mode 100644 index 22ee92687e..0000000000 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreFilter.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.cloudfoundry.multiapps.controller.persistence.util; - -import java.text.MessageFormat; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.List; -import java.util.Map; - -import org.cloudfoundry.multiapps.controller.persistence.Constants; -import org.cloudfoundry.multiapps.controller.persistence.Messages; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.util.CollectionUtils; - -public class ObjectStoreFilter { - - private static final Logger LOGGER = LoggerFactory.getLogger(ObjectStoreFilter.class); - - public static boolean filterBySpaceIds(Map metadata, List spaceIds) { - if (CollectionUtils.isEmpty(metadata)) { - return false; - } - String spaceParameter = metadata.get(Constants.FILE_ENTRY_SPACE.toLowerCase()); - - return spaceIds.contains(spaceParameter); - } - - public static boolean filterBySpaceAndNamespace(Map metadata, String space, String namespace) { - if (CollectionUtils.isEmpty(metadata)) { - return false; - } - - String spaceParameter = metadata.get(Constants.FILE_ENTRY_SPACE.toLowerCase()); - String namespaceParameter = metadata.get(Constants.FILE_ENTRY_NAMESPACE.toLowerCase()); - return space.equals(spaceParameter) && namespace.equals(namespaceParameter); - } - - public static boolean filterByModificationTime(Map metadata, String blobName, LocalDateTime modificationTime) { - // Clean up any blobStore entries that don't have any metadata as we can't check their creation date - if (CollectionUtils.isEmpty(metadata)) { - LOGGER.warn(MessageFormat.format(Messages.USER_METADATA_OF_BLOB_0_EMPTY_AND_WILL_BE_DELETED, blobName)); - return true; - } - String longString = metadata.get(Constants.FILE_ENTRY_MODIFIED.toLowerCase()); - try { - long dateLong = Long.parseLong(longString); - LocalDateTime date = LocalDateTime.ofInstant(Instant.ofEpochMilli(dateLong), ZoneId.systemDefault()); - return date.isBefore(modificationTime); - } catch (NumberFormatException e) { - // Clean up any blobStore entries that have invalid timestamp - LOGGER.warn(MessageFormat.format(Messages.DATE_METADATA_OF_BLOB_0_IS_NOT_IN_PROPER_FORMAT_AND_WILL_BE_DELETED, - blobName), - e); - return true; - } - } -} diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapper.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapper.java deleted file mode 100644 index 1875b88a50..0000000000 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapper.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.cloudfoundry.multiapps.controller.persistence.util; - -import java.time.ZoneId; -import java.util.HashMap; -import java.util.Map; - -import org.cloudfoundry.multiapps.controller.persistence.Constants; -import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; -import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; - -public class ObjectStoreMapper { - - public static Map createFileEntryMetadata(FileEntry fileEntry) { - Map metadata = new HashMap<>(); - metadata.put(Constants.FILE_ENTRY_SPACE.toLowerCase(), fileEntry.getSpace()); - metadata.put(Constants.FILE_ENTRY_MODIFIED.toLowerCase(), - Long.toString(fileEntry.getModified() - .atZone( - ZoneId.systemDefault()) - .toInstant() - .toEpochMilli())); - if (fileEntry.getNamespace() != null) { - metadata.put(Constants.FILE_ENTRY_NAMESPACE.toLowerCase(), fileEntry.getNamespace()); - } - return metadata; - } - - public static FileEntry createFileEntry(String space, String id) { - return ImmutableFileEntry.builder() - .space(space) - .id(id) - .build(); - } - -} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java deleted file mode 100644 index cc4b0aa773..0000000000 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java +++ /dev/null @@ -1,95 +0,0 @@ -package org.cloudfoundry.multiapps.controller.persistence.services; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.function.Predicate; - -import com.google.cloud.storage.Blob; -import com.google.cloud.storage.BlobId; -import com.google.cloud.storage.BlobInfo; -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; -import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.springframework.http.MediaType; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; - -class GcpObjectStoreFileStorageTest extends JCloudsObjectStoreFileStorageTest { - - private Storage storage; - - @Override - @BeforeEach - public void setUp() { - storage = LocalStorageHelper.getOptions() - .getService(); - fileStorage = new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER)) { - - @Override - protected Storage createObjectStoreStorage(Map credentials) { - return storage; - } - - @Override - protected int removeBlobsByFilter(Predicate filter) { - Set entries = getEntryNames(filter); - for (String entry : entries) { - storage.delete(CONTAINER, entry); - } - - return entries.size(); - } - }; - spaceId = UUID.randomUUID() - .toString(); - namespace = UUID.randomUUID() - .toString(); - } - - @Override - @AfterEach - public void tearDown() { - if (storage != null) { - try { - storage.close(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } - - @Override - protected void assertBlobDoesNotExist(String blobWithNoMetadataId) { - assertNull(storage.get(blobWithNoMetadataId)); - } - - @Override - public String addBlobWithNoMetadata() throws Exception { - Path path = Paths.get(TEST_FILE_LOCATION); - String id = UUID.randomUUID() - .toString(); - BlobInfo blobInfo = BlobInfo.newBuilder(BlobId.of(CONTAINER, id)) - .setContentDisposition(path.getFileName() - .toString()) - .setContentType(MediaType.APPLICATION_OCTET_STREAM_VALUE) - .build(); - storage.create(blobInfo, Files.newInputStream(path)); - return id; - } - - @Override - public void assertFileExists(boolean exceptedFileExist, FileEntry actualFile) { - Blob blob = storage.get(CONTAINER, actualFile.getId()); - boolean blobExists = blob != null; - - assertEquals(exceptedFileExist, blobExists); - } - -} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorageTest.java similarity index 88% rename from multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java rename to multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorageTest.java index 6356afd641..ae530c6b69 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorageTest.java @@ -1,5 +1,10 @@ package org.cloudfoundry.multiapps.controller.persistence.services; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -17,6 +22,7 @@ import java.util.UUID; import jakarta.xml.bind.DatatypeConverter; + import org.cloudfoundry.multiapps.common.util.DigestHelper; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; @@ -29,29 +35,24 @@ import org.junit.jupiter.api.Test; import org.springframework.http.MediaType; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; - -class JCloudsObjectStoreFileStorageTest { +class ObjectStoreFileStorageTest { - protected static final String TEST_FILE_LOCATION = "src/test/resources/pexels-photo-401794.jpeg"; - protected static final String SECOND_FILE_TEST_LOCATION = "src/test/resources/pexels-photo-463467.jpeg"; - protected static final String DIGEST_METHOD = "MD5"; - protected static final String CONTAINER = "container4e"; + private static final String TEST_FILE_LOCATION = "src/test/resources/pexels-photo-401794.jpeg"; + private static final String SECOND_FILE_TEST_LOCATION = "src/test/resources/pexels-photo-463467.jpeg"; + private static final String DIGEST_METHOD = "MD5"; + private static final String CONTAINER = "container4e"; - protected String spaceId; - protected String namespace; + private String spaceId; + private String namespace; - protected FileStorage fileStorage; + private FileStorage fileStorage; private BlobStoreContext blobStoreContext; @BeforeEach - protected void setUp() { + public void setUp() { createBlobStoreContext(); - fileStorage = new JCloudsObjectStoreFileStorage(blobStoreContext.getBlobStore(), CONTAINER) { + fileStorage = new ObjectStoreFileStorage(blobStoreContext.getBlobStore(), CONTAINER) { @Override protected long getRetryWaitTime() { return 1; @@ -71,7 +72,7 @@ private void createBlobStoreContext() { } @AfterEach - protected void tearDown() { + public void tearDown() { if (blobStoreContext != null) { blobStoreContext.close(); } @@ -162,10 +163,6 @@ void deleteFilesModifiedBefore() throws Exception { assertFileExists(true, fileEntryToRemain2); assertFileExists(false, fileEntryToDelete1); assertFileExists(false, fileEntryToDelete2); - assertBlobDoesNotExist(blobWithNoMetadataId); - } - - protected void assertBlobDoesNotExist(String blobWithNoMetadataId) { assertNull(blobStoreContext.getBlobStore() .getBlob(CONTAINER, blobWithNoMetadataId)); } @@ -179,10 +176,11 @@ void testConnection() { void testDeleteFilesByIds() throws Exception { FileEntry fileEntry = addFile(TEST_FILE_LOCATION); fileStorage.deleteFilesByIds(List.of(fileEntry.getId())); - assertBlobDoesNotExist(fileEntry.getId()); + assertNull(blobStoreContext.getBlobStore() + .getBlob(CONTAINER, fileEntry.getId())); } - protected String addBlobWithNoMetadata() throws Exception { + private String addBlobWithNoMetadata() throws Exception { BlobStore blobStore = blobStoreContext.getBlobStore(); Path path = Paths.get(TEST_FILE_LOCATION); long fileSize = Files.size(path); @@ -220,7 +218,7 @@ void testFileContentNotExisting() throws Exception { assertThrows(FileStorageException.class, () -> validateFileContent(dummyFileEntry, fileDigest)); } - protected void validateFileContent(FileEntry storedFile, final String expectedFileChecksum) throws FileStorageException { + private void validateFileContent(FileEntry storedFile, final String expectedFileChecksum) throws FileStorageException { fileStorage.processFileContent(storedFile.getSpace(), storedFile.getId(), contentStream -> { // make a digest out of the content and compare it to the original final byte[] digest = calculateFileDigest(contentStream); @@ -244,21 +242,21 @@ private byte[] calculateFileDigest(InputStream contentStream) throws IOException } } - protected void addBigAmountOfEntries() throws Exception { + private void addBigAmountOfEntries() throws Exception { for (int i = 0; i < 3001; i++) { addFileContent("test-file-" + i, "test".getBytes()); } } - protected FileEntry addFile(String pathString) throws Exception { + private FileEntry addFile(String pathString) throws Exception { return addFile(pathString, spaceId, namespace); } - protected FileEntry addFile(String pathString, String space, String namespace) throws Exception { + private FileEntry addFile(String pathString, String space, String namespace) throws Exception { return addFile(pathString, space, namespace, null); } - protected FileEntry addFile(String pathString, String space, String namespace, LocalDateTime date) throws Exception { + private FileEntry addFile(String pathString, String space, String namespace, LocalDateTime date) throws Exception { Path testFilePath = Paths.get(pathString) .toAbsolutePath(); FileEntry fileEntry = createFileEntry(space, namespace); @@ -313,12 +311,12 @@ private FileEntry enrichFileEntry(FileEntry fileEntry, Path path, LocalDateTime .build(); } - protected void assertFileExists(boolean expectedFileExist, FileEntry actualFile) { + private void assertFileExists(boolean exceptedFileExist, FileEntry actualFile) { Blob blob = blobStoreContext.getBlobStore() .getBlob(CONTAINER, actualFile.getId()); boolean blobExists = blob != null; - assertEquals(expectedFileExist, blobExists); + assertEquals(exceptedFileExist, blobExists); } } diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreFilterTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreFilterTest.java deleted file mode 100644 index a0b60a096b..0000000000 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreFilterTest.java +++ /dev/null @@ -1,79 +0,0 @@ -package org.cloudfoundry.multiapps.controller.persistence.util; - -import java.time.Instant; -import java.time.LocalDateTime; -import java.util.List; -import java.util.Map; - -import org.cloudfoundry.multiapps.controller.persistence.Constants; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class ObjectStoreFilterTest { - - private static final String SPACE_ID = "spaceId"; - private static final String BLOBNAME = "blobName"; - private static final String SPACE_ID_2 = "spaceId2"; - private static final String NAMESPACE = "namespace"; - private static final String NAMESPACE_2 = "namespace2"; - - @Test - void testFilterBySpaceIdsWhenMetadataIsEmpty() { - assertFalse(ObjectStoreFilter.filterBySpaceIds(Map.of(), List.of())); - } - - @Test - void testFilterBySpaceIdsWhenTheMetadataContainsTheSpaceId() { - assertTrue(ObjectStoreFilter.filterBySpaceIds(buildMetadata(), List.of(SPACE_ID))); - } - - @Test - void testFilterBySpaceIdsWhenTheMetadataDoesNotContainTheSpaceId() { - assertFalse(ObjectStoreFilter.filterBySpaceIds(buildMetadata(), List.of(SPACE_ID_2))); - } - - @Test - void testFilterBySpaceAndNamespaceWhenMetadataIsEmpty() { - assertFalse(ObjectStoreFilter.filterBySpaceAndNamespace(Map.of(), SPACE_ID, NAMESPACE)); - } - - @Test - void testFilterBySpaceAndNamespaceWhenTheSpacesAreDifferent() { - assertFalse(ObjectStoreFilter.filterBySpaceAndNamespace(buildMetadata(), SPACE_ID_2, NAMESPACE)); - } - - @Test - void testFilterBySpaceAndNamespaceWhenTheNamespacesAreDifferent() { - assertFalse(ObjectStoreFilter.filterBySpaceAndNamespace(buildMetadata(), SPACE_ID, NAMESPACE_2)); - } - - @Test - void testFilterBySpaceAndNamespaceWhenAllMatch() { - assertTrue(ObjectStoreFilter.filterBySpaceAndNamespace(buildMetadata(), SPACE_ID, NAMESPACE)); - } - - @Test - void testFilterByModificationTimeWhenMetadataIsEmpty() { - assertTrue(ObjectStoreFilter.filterByModificationTime(Map.of(), BLOBNAME, LocalDateTime.now())); - } - - @Test - void testFilterByModificationTimeWhenModificationTimeIsBeforeTheMetadataTime() { - assertFalse(ObjectStoreFilter.filterByModificationTime(buildMetadata(), BLOBNAME, LocalDateTime.now() - .minusMinutes(20))); - } - - @Test - void testFilterByModificationTimeWhenModificationTimeIsAfterTheMetadataTime() { - assertTrue(ObjectStoreFilter.filterByModificationTime(buildMetadata(), BLOBNAME, LocalDateTime.now() - .plusMinutes(20))); - } - - private Map buildMetadata() { - return Map.of(Constants.FILE_ENTRY_SPACE.toLowerCase(), SPACE_ID, Constants.FILE_ENTRY_NAMESPACE.toLowerCase(), NAMESPACE, - Constants.FILE_ENTRY_MODIFIED.toLowerCase(), String.valueOf(Instant.now() - .toEpochMilli())); - } -} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapperTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapperTest.java deleted file mode 100644 index d0cd9a23c3..0000000000 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapperTest.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.cloudfoundry.multiapps.controller.persistence.util; - -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.Map; -import java.util.UUID; - -import org.cloudfoundry.multiapps.controller.persistence.Constants; -import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; -import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -class ObjectStoreMapperTest { - - private static final String SPACE_ID = "spaceId"; - private static final String NAMESPACE = "namespace"; - - @Test - void testCreateFileEntryMetadata() { - LocalDateTime modifiedTime = LocalDateTime.now(); - String modifiedTimeInstantString = Long.toString(modifiedTime - .atZone( - ZoneId.systemDefault()) - .toInstant() - .toEpochMilli()); - Map metadata = ObjectStoreMapper.createFileEntryMetadata(buildCreateFileEntry(modifiedTime)); - - assertEquals(SPACE_ID, metadata.get(Constants.FILE_ENTRY_SPACE.toLowerCase())); - assertEquals(modifiedTimeInstantString, metadata.get(Constants.FILE_ENTRY_MODIFIED.toLowerCase())); - assertEquals(NAMESPACE, metadata.get(Constants.FILE_ENTRY_NAMESPACE.toLowerCase())); - } - - @Test - void testCreateFileEntry() { - String id = UUID.randomUUID() - .toString(); - FileEntry fileEntry = ObjectStoreMapper.createFileEntry(SPACE_ID, id); - - assertEquals(SPACE_ID, fileEntry.getSpace()); - assertEquals(id, fileEntry.getId()); - } - - private FileEntry buildCreateFileEntry(LocalDateTime modifiedTime) { - return ImmutableFileEntry.builder() - .space(SPACE_ID) - .modified(modifiedTime) - .namespace(NAMESPACE) - .build(); - } -} diff --git a/multiapps-controller-web/pom.xml b/multiapps-controller-web/pom.xml index 55d62ab602..c18efba006 100644 --- a/multiapps-controller-web/pom.xml +++ b/multiapps-controller-web/pom.xml @@ -199,13 +199,5 @@ javax.xml.bind jaxb-api - - com.google.cloud - google-cloud-storage - - - com.google.cloud - google-cloud-nio - diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/FileServiceFactoryBean.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/FileServiceFactoryBean.java index 40d7728836..68a076ba3d 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/FileServiceFactoryBean.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/FileServiceFactoryBean.java @@ -2,10 +2,12 @@ import jakarta.inject.Inject; import jakarta.inject.Named; + import org.cloudfoundry.multiapps.controller.persistence.DataSourceWithDialect; import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseFileService; import org.cloudfoundry.multiapps.controller.persistence.services.FileService; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; +import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.web.Messages; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,15 +23,15 @@ public class FileServiceFactoryBean implements FactoryBean, Initial @Inject private DataSourceWithDialect dataSourceWithDialect; @Autowired(required = false) - private FileStorage objectStoreFileStorage; - + private ObjectStoreFileStorage objectStoreFileStorage; private FileService fileService; @Override public void afterPropertiesSet() { - if (objectStoreFileStorage != null) { + FileStorage fileStorage = objectStoreFileStorage; + if (fileStorage != null) { LOGGER.info(Messages.OBJECTSTORE_FOR_BINARIES_STORAGE); - this.fileService = new FileService(dataSourceWithDialect, objectStoreFileStorage); + this.fileService = new FileService(dataSourceWithDialect, fileStorage); } else { LOGGER.info(Messages.DATABASE_FOR_BINARIES_STORAGE); this.fileService = new DatabaseFileService(dataSourceWithDialect); diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java index 0001e13be1..9d84a5507b 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java @@ -16,13 +16,10 @@ import org.apache.commons.lang3.StringUtils; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.util.UriUtil; -import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; -import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; -import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage; +import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder; import org.cloudfoundry.multiapps.controller.web.Constants; import org.cloudfoundry.multiapps.controller.web.Messages; -import org.cloudfoundry.multiapps.controller.web.configuration.service.ImmutableObjectStoreServiceInfo; import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfo; import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfoCreator; import org.jclouds.ContextBuilder; @@ -33,7 +30,7 @@ import org.springframework.beans.factory.FactoryBean; import org.springframework.beans.factory.InitializingBean; -public class ObjectStoreFileStorageFactoryBean implements FactoryBean, InitializingBean { +public class ObjectStoreFileStorageFactoryBean implements FactoryBean, InitializingBean { private static final Logger LOGGER = LoggerFactory.getLogger(ObjectStoreFileStorageFactoryBean.class); private static final Set CUSTOM_REGIONS = Set.of("eu-south-1"); @@ -42,7 +39,7 @@ public class ObjectStoreFileStorageFactoryBean implements FactoryBean providersServiceInfo = getProvidersServiceInfo(); if (providersServiceInfo.isEmpty()) { return null; @@ -67,46 +64,31 @@ private FileStorage createObjectStoreFileStorage() { return createObjectStoreFromFirstReachableProvider(exceptions, providersServiceInfo); } - Optional optionalFileStorage = createObjectStoreBasedOnProvider(objectStoreProviderName, providersServiceInfo, - exceptions); + Optional objectStoreServiceInfoOptional = getAppropriateProvider(objectStoreProviderName, + providersServiceInfo); - if (optionalFileStorage.isPresent()) { - return optionalFileStorage.get(); + if (objectStoreServiceInfoOptional.isPresent()) { + ObjectStoreServiceInfo objectStoreServiceInfo = objectStoreServiceInfoOptional.get(); + Optional createdObjectStore = tryToCreateObjectStore(objectStoreServiceInfo, exceptions); + if (createdObjectStore.isPresent()) { + return createdObjectStore.get(); + } } throw buildNoValidObjectStoreException(exceptions); } - public FileStorage createObjectStoreFromFirstReachableProvider(Map exceptions, - List providersServiceInfo) { + public ObjectStoreFileStorage createObjectStoreFromFirstReachableProvider(Map exceptions, + List providersServiceInfo) { for (ObjectStoreServiceInfo objectStoreServiceInfo : providersServiceInfo) { - Optional createdObjectStore = tryToCreateObjectStore(objectStoreServiceInfo, exceptions); - if (createdObjectStore.isPresent()) { - return createdObjectStore.get(); + Optional createdObjectStoreOptional = tryToCreateObjectStore(objectStoreServiceInfo, exceptions); + if (createdObjectStoreOptional.isPresent()) { + return createdObjectStoreOptional.get(); } } - Optional gcpObjectStoreOpt = tryToCreateGcpObjectStore(exceptions); - if (gcpObjectStoreOpt.isPresent()) { - return gcpObjectStoreOpt.get(); - } throw buildNoValidObjectStoreException(exceptions); } - private Optional createObjectStoreBasedOnProvider(String objectStoreProviderName, - List providersServiceInfo, - Map exceptions) { - Optional objectStoreServiceInfoOptional = getAppropriateProvider(objectStoreProviderName, - providersServiceInfo); - Optional createdObjectStore; - if (objectStoreServiceInfoOptional.isPresent()) { - ObjectStoreServiceInfo objectStoreServiceInfo = objectStoreServiceInfoOptional.get(); - createdObjectStore = tryToCreateObjectStore(objectStoreServiceInfo, exceptions); - } else { - createdObjectStore = tryToCreateGcpObjectStore(exceptions); - } - return createdObjectStore; - } - private Optional getAppropriateProvider(String objectStoreProviderName, List providersServiceInfo) { String appropriateProvider = Constants.ENV_TO_OS_PROVIDER.get(objectStoreProviderName); @@ -115,34 +97,6 @@ private Optional getAppropriateProvider(String objectSto .findFirst(); } - private Optional tryToCreateGcpObjectStore(Map exceptions) { - return tryToCreateObjectStore(ImmutableObjectStoreServiceInfo.builder() - .provider(Constants.GOOGLE_CLOUD_STORAGE) - .build(), exceptions); - } - - private Optional tryToCreateObjectStore(ObjectStoreServiceInfo objectStoreServiceInfo, - Map exceptions) { - try { - FileStorage fileStorage = getFileStorageBasedOnProvider(objectStoreServiceInfo); - fileStorage.testConnection(); - LOGGER.info(MessageFormat.format(Messages.OBJECT_STORE_WITH_PROVIDER_0_CREATED, objectStoreServiceInfo.getProvider())); - return Optional.of(fileStorage); - } catch (Exception e) { - exceptions.put(objectStoreServiceInfo.getProvider(), e); - return Optional.empty(); - } - } - - private FileStorage getFileStorageBasedOnProvider(ObjectStoreServiceInfo objectStoreServiceInfo) { - if (Constants.GOOGLE_CLOUD_STORAGE.equals(objectStoreServiceInfo.getProvider())) { - return createGcpFileStorage(); - } else { - BlobStoreContext context = getBlobStoreContext(objectStoreServiceInfo); - return createFileStorage(objectStoreServiceInfo, context); - } - } - private boolean isObjectStoreEnvValid(String objectStoreProviderName) { return objectStoreProviderName != null && !objectStoreProviderName.isEmpty() && Constants.ENV_TO_OS_PROVIDER.containsKey( objectStoreProviderName); @@ -155,21 +109,26 @@ private IllegalStateException buildNoValidObjectStoreException(Map getProvidersServiceInfo() { - Map credentials = getServiceCredentials(); - if (credentials.isEmpty()) { - return Collections.emptyList(); + private Optional tryToCreateObjectStore(ObjectStoreServiceInfo objectStoreServiceInfo, + Map exceptions) { + try { + BlobStoreContext context = getBlobStoreContext(objectStoreServiceInfo); + ObjectStoreFileStorage fileStorage = createFileStorage(objectStoreServiceInfo, context); + fileStorage.testConnection(); + LOGGER.info(MessageFormat.format(Messages.OBJECT_STORE_WITH_PROVIDER_0_CREATED, objectStoreServiceInfo.getProvider())); + return Optional.of(fileStorage); + } catch (Exception e) { + exceptions.put(objectStoreServiceInfo.getProvider(), e); + return Optional.empty(); } - return new ObjectStoreServiceInfoCreator().getAllProvidersServiceInfo(credentials); } - private Map getServiceCredentials() { + private List getProvidersServiceInfo() { CfService service = environmentServicesFinder.findService(serviceName); if (service == null) { - return Map.of(); + return Collections.emptyList(); } - return service.getCredentials() - .getMap(); + return new ObjectStoreServiceInfoCreator().getAllProvidersServiceInfo(service); } private BlobStoreContext getBlobStoreContext(ObjectStoreServiceInfo serviceInfo) { @@ -222,23 +181,18 @@ private void resolveContextEndpoint(ObjectStoreServiceInfo serviceInfo, ContextB } } - protected JCloudsObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) { - return new JCloudsObjectStoreFileStorage(context.getBlobStore(), objectStoreServiceInfo.getContainer()); - } - - protected GcpObjectStoreFileStorage createGcpFileStorage() { - Map credentials = getServiceCredentials(); - return new GcpObjectStoreFileStorage(credentials); + protected ObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) { + return new ObjectStoreFileStorage(context.getBlobStore(), objectStoreServiceInfo.getContainer()); } @Override - public FileStorage getObject() { + public ObjectStoreFileStorage getObject() { return objectStoreFileStorage; } @Override public Class getObjectType() { - return FileStorage.class; + return ObjectStoreFileStorage.class; } } \ No newline at end of file diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java index 86c445e306..bad4dbd804 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java @@ -30,4 +30,5 @@ public interface ObjectStoreServiceInfo { @Nullable String getHost(); + } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java index 57fd802b88..5b3f9cc4b1 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java @@ -2,17 +2,26 @@ import java.net.MalformedURLException; import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Base64; import java.util.List; import java.util.Map; +import com.google.common.base.Supplier; +import io.pivotal.cfenv.core.CfService; import org.cloudfoundry.multiapps.controller.web.Constants; import org.cloudfoundry.multiapps.controller.web.Messages; +import org.jclouds.domain.Credentials; +import org.jclouds.googlecloud.GoogleCredentialsFromJson; public class ObjectStoreServiceInfoCreator { - public List getAllProvidersServiceInfo(Map credentials) { + public List getAllProvidersServiceInfo(CfService service) { + Map credentials = service.getCredentials() + .getMap(); return List.of(createServiceInfoForAws(credentials), createServiceInfoForAliCloud(credentials), - createServiceInfoForAzure(credentials), createServiceInfoForCcee(credentials)); + createServiceInfoForAzure(credentials), createServiceInfoForGcpCloud(credentials), + createServiceInfoForCcee(credentials)); } private ObjectStoreServiceInfo createServiceInfoForAws(Map credentials) { @@ -86,4 +95,27 @@ private URL getContainerUriEndpoint(Map credentials) { throw new IllegalStateException(Messages.CANNOT_PARSE_CONTAINER_URI_OF_OBJECT_STORE, e); } } + + private ObjectStoreServiceInfo createServiceInfoForGcpCloud(Map credentials) { + String bucket = (String) credentials.get(Constants.BUCKET); + String region = (String) credentials.get(Constants.REGION); + Supplier credentialsSupplier = getGcpCredentialsSupplier(credentials); + return ImmutableObjectStoreServiceInfo.builder() + .provider(Constants.GOOGLE_CLOUD_STORAGE) + .credentialsSupplier(credentialsSupplier) + .container(bucket) + .region(region) + .build(); + } + + protected Supplier getGcpCredentialsSupplier(Map credentials) { + if (!credentials.containsKey(Constants.BASE_64_ENCODED_PRIVATE_KEY_DATA)) { + return () -> null; + } + byte[] decodedKey = Base64.getDecoder() + .decode((String) credentials.get(Constants.BASE_64_ENCODED_PRIVATE_KEY_DATA)); + String decodedCredential = new String(decodedKey, StandardCharsets.UTF_8); + return new GoogleCredentialsFromJson(decodedCredential); + } + } diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java index 5b5617b7a6..8cdb10fa93 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java @@ -1,21 +1,17 @@ package org.cloudfoundry.multiapps.controller.web.configuration.bean.factory; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Set; import io.pivotal.cfenv.core.CfCredentials; import io.pivotal.cfenv.core.CfService; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; -import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; -import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; -import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage; +import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder; import org.cloudfoundry.multiapps.controller.web.Constants; import org.cloudfoundry.multiapps.controller.web.Messages; import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfo; -import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfoCreator; import org.jclouds.blobstore.BlobStoreContext; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -49,10 +45,7 @@ class ObjectStoreFileStorageFactoryBeanTest { @Mock private ApplicationConfiguration applicationConfiguration; @Mock - private JCloudsObjectStoreFileStorage jCloudsObjectStoreFileStorage; - - @Mock - private GcpObjectStoreFileStorage gcpObjectStoreFileStorage; + private ObjectStoreFileStorage objectStoreFileStorage; @BeforeEach void setUp() throws Exception { @@ -66,7 +59,7 @@ void setUp() throws Exception { @Test void testObjectStoreCreationWithoutServiceInstance() { objectStoreFileStorageFactoryBean.afterPropertiesSet(); - FileStorage objectStoreFileStorage = objectStoreFileStorageFactoryBean.getObject(); + ObjectStoreFileStorage objectStoreFileStorage = objectStoreFileStorageFactoryBean.getObject(); assertNull(objectStoreFileStorage); } @@ -74,7 +67,7 @@ void testObjectStoreCreationWithoutServiceInstance() { void testObjectStoreCreationWithValidServiceInstance() { mockCfService(); objectStoreFileStorageFactoryBean.afterPropertiesSet(); - FileStorage objectStoreFileStorage = objectStoreFileStorageFactoryBean.getObject(); + ObjectStoreFileStorage objectStoreFileStorage = objectStoreFileStorageFactoryBean.getObject(); assertNotNull(objectStoreFileStorage); } @@ -85,12 +78,12 @@ void testObjectStoreCreationWhenEnvIsValid() { ObjectStoreFileStorageFactoryBean spy = spy(objectStoreFileStorageFactoryBean); spy.afterPropertiesSet(); - FileStorage createdObjectStoreFileStorage = spy.getObject(); + ObjectStoreFileStorage createdObjectStoreFileStorage = spy.getObject(); assertNotNull(createdObjectStoreFileStorage); verify(spy, never()) .createObjectStoreFromFirstReachableProvider(anyMap(), anyList()); - verify(jCloudsObjectStoreFileStorage, times(1)) + verify(objectStoreFileStorage, times(1)) .testConnection(); } @@ -102,7 +95,7 @@ void testObjectStoreCreationWhenEnvIsInvalid() { ObjectStoreFileStorageFactoryBean spy = spy(objectStoreFileStorageFactoryBean); spy.afterPropertiesSet(); - FileStorage createdObjectStoreFileStorage = spy.getObject(); + ObjectStoreFileStorage createdObjectStoreFileStorage = spy.getObject(); assertNotNull(createdObjectStoreFileStorage); verify(spy, times(1)) @@ -113,7 +106,7 @@ void testObjectStoreCreationWhenEnvIsInvalid() { void testObjectStoreCreationWhenEnvProviderFailsToConnect() { mockCfService(); when(applicationConfiguration.getObjectStoreClientType()).thenReturn(Constants.AWS); - doThrow(new IllegalStateException("Cannot create object store")).when(jCloudsObjectStoreFileStorage) + doThrow(new IllegalStateException("Cannot create object store")).when(objectStoreFileStorage) .testConnection(); Exception exception = assertThrows(IllegalStateException.class, () -> objectStoreFileStorageFactoryBean.afterPropertiesSet()); @@ -123,9 +116,7 @@ void testObjectStoreCreationWhenEnvProviderFailsToConnect() { @Test void testObjectStoreCreationWithoutValidServiceInstance() { mockCfService(); - doThrow(new IllegalStateException("Cannot create object store")).when(jCloudsObjectStoreFileStorage) - .testConnection(); - doThrow(new IllegalStateException("Cannot create object store")).when(gcpObjectStoreFileStorage) + doThrow(new IllegalStateException("Cannot create object store")).when(objectStoreFileStorage) .testConnection(); Exception exception = assertThrows(IllegalStateException.class, () -> objectStoreFileStorageFactoryBean.afterPropertiesSet()); assertEquals(Messages.NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND, exception.getMessage()); @@ -155,27 +146,8 @@ public ObjectStoreFileStorageFactoryBeanMock(String serviceName, EnvironmentServ } @Override - protected JCloudsObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) { - return ObjectStoreFileStorageFactoryBeanTest.this.jCloudsObjectStoreFileStorage; - } - - @Override - protected GcpObjectStoreFileStorage createGcpFileStorage() { - return ObjectStoreFileStorageFactoryBeanTest.this.gcpObjectStoreFileStorage; + protected ObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) { + return ObjectStoreFileStorageFactoryBeanTest.this.objectStoreFileStorage; } - - @Override - public List getProvidersServiceInfo() { - CfService service = environmentServicesFinder.findService("deploy-service-os"); - if (service != null) { - return new ObjectStoreServiceInfoCreatorMock().getAllProvidersServiceInfo(service.getCredentials() - .getMap()); - } else { - return List.of(); - } - } - } - - private class ObjectStoreServiceInfoCreatorMock extends ObjectStoreServiceInfoCreator { } } diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java index 6439250543..38e4216f7d 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java @@ -1,23 +1,29 @@ package org.cloudfoundry.multiapps.controller.web.configuration.service; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + import java.net.MalformedURLException; import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Base64; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Stream; -import io.pivotal.cfenv.core.CfCredentials; -import io.pivotal.cfenv.core.CfService; import org.cloudfoundry.multiapps.controller.web.Constants; +import org.jclouds.domain.Credentials; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mockito; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.when; +import com.google.common.base.Supplier; + +import io.pivotal.cfenv.core.CfCredentials; +import io.pivotal.cfenv.core.CfService; class ObjectStoreServiceInfoCreatorTest { @@ -30,6 +36,7 @@ class ObjectStoreServiceInfoCreatorTest { private static final String SAS_TOKEN_VALUE = "sas_token_value"; private static final String CONTAINER_NAME_VALUE = "container_name_value"; private static final String CONTAINER_URI_VALUE = "https://container.com:8080"; + private static final Supplier CREDENTIALS_SUPPLIER = () -> null; private ObjectStoreServiceInfoCreator objectStoreServiceInfoCreator; @@ -41,15 +48,14 @@ void setUp() { static Stream testDifferentProviders() throws MalformedURLException { return Stream.of(Arguments.of(buildCfService(buildAliCloudCredentials()), buildAliCloudObjectStoreServiceInfo()), Arguments.of(buildCfService(buildAwsCredentials()), buildAwsObjectStoreServiceInfo()), - Arguments.of(buildCfService(buildAzureCredentials()), buildAzureObjectStoreServiceInfo())); + Arguments.of(buildCfService(buildAzureCredentials()), buildAzureObjectStoreServiceInfo()), + Arguments.of(buildCfService(buildGcpCredentials()), buildGcpObjectStoreServiceInfo())); } @ParameterizedTest @MethodSource void testDifferentProviders(CfService cfService, ObjectStoreServiceInfo exprectedObjectStoreServiceInfo) { - List providersServiceInfo = objectStoreServiceInfoCreator.getAllProvidersServiceInfo( - cfService.getCredentials() - .getMap()); + List providersServiceInfo = objectStoreServiceInfoCreator.getAllProvidersServiceInfo(cfService); assertTrue(providersServiceInfo.contains(exprectedObjectStoreServiceInfo)); } @@ -118,6 +124,29 @@ private static ObjectStoreServiceInfo buildAzureObjectStoreServiceInfo() throws .build(); } + private static Map buildGcpCredentials() { + Map credentials = new HashMap<>(); + credentials.put(Constants.BUCKET, BUCKET_VALUE); + credentials.put(Constants.REGION, REGION_VALUE); + credentials.put(Constants.BASE_64_ENCODED_PRIVATE_KEY_DATA, Base64.getEncoder() + .encodeToString("encoded_data".getBytes(StandardCharsets.UTF_8))); + return credentials; + } + + private static ObjectStoreServiceInfo buildGcpObjectStoreServiceInfo() { + return ImmutableObjectStoreServiceInfo.builder() + .provider(Constants.GOOGLE_CLOUD_STORAGE) + .credentialsSupplier(CREDENTIALS_SUPPLIER) + .container(BUCKET_VALUE) + .region(REGION_VALUE) + .build(); + } + private static class ObjectStoreServiceInfoCreatorMock extends ObjectStoreServiceInfoCreator { + + @Override + protected Supplier getGcpCredentialsSupplier(Map credentials) { + return ObjectStoreServiceInfoCreatorTest.CREDENTIALS_SUPPLIER; + } } } diff --git a/pom.xml b/pom.xml index e8ece13a2a..2cd06f28ed 100644 --- a/pom.xml +++ b/pom.xml @@ -57,8 +57,6 @@ 1.28.0 3.2.2 1.3.1 - 2.59.0 - 0.128.7 multiapps-controller-client @@ -290,17 +288,6 @@ commons-compress ${apache.compress.version} - - com.google.cloud - google-cloud-storage - ${google-cloud.version} - - - com.google.cloud - google-cloud-nio - ${google-cloud-nio.version} - test - org.immutables