diff --git a/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculator.java b/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculator.java index 40272e7d90..70a85311c2 100644 --- a/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculator.java +++ b/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculator.java @@ -15,6 +15,7 @@ import java.util.concurrent.TimeoutException; import java.util.function.Consumer; import java.util.function.Supplier; + import jakarta.inject.Inject; import jakarta.inject.Named; import org.cloudfoundry.multiapps.common.SLException; @@ -29,7 +30,7 @@ import org.cloudfoundry.multiapps.controller.core.util.ApplicationInstanceNameUtil; import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseHealthService; import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseMonitoringService; -import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage; +import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -46,7 +47,7 @@ public class ApplicationHealthCalculator { // timeout private static final int TOTAL_TASK_TIMEOUT_IN_SECONDS = 3 * SINGLE_TASK_TIMEOUT_IN_SECONDS; - private final ObjectStoreFileStorage objectStoreFileStorage; + private final FileStorage objectStoreFileStorage; private final ApplicationConfiguration applicationConfiguration; private final DatabaseHealthService databaseHealthService; private final DatabaseMonitoringService databaseMonitoringService; @@ -74,7 +75,7 @@ public class ApplicationHealthCalculator { private final ResilientOperationExecutor resilientOperationExecutor = getResilienceExecutor(); @Inject - public ApplicationHealthCalculator(@Autowired(required = false) ObjectStoreFileStorage objectStoreFileStorage, + public ApplicationHealthCalculator(@Autowired(required = false) FileStorage objectStoreFileStorage, ApplicationConfiguration applicationConfiguration, DatabaseHealthService databaseHealthService, DatabaseMonitoringService databaseMonitoringService, DatabaseWaitingLocksAnalyzer databaseWaitingLocksAnalyzer) { diff --git a/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculatorTest.java b/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculatorTest.java index e067eb7e88..8a67a4bda3 100644 --- a/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculatorTest.java +++ b/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculatorTest.java @@ -1,9 +1,5 @@ package org.cloudfoundry.multiapps.controller.core.application.health; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - import org.cloudfoundry.multiapps.common.SLException; import org.cloudfoundry.multiapps.controller.client.util.ResilientOperationExecutor; import org.cloudfoundry.multiapps.controller.core.application.health.database.DatabaseWaitingLocksAnalyzer; @@ -11,7 +7,7 @@ import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseHealthService; import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseMonitoringService; -import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage; +import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; @@ -20,10 +16,14 @@ import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + class ApplicationHealthCalculatorTest { @Mock - private ObjectStoreFileStorage objectStoreFileStorage; + private JCloudsObjectStoreFileStorage jCloudsObjectStoreFileStorage; @Mock private ApplicationConfiguration applicationConfiguration; @Mock @@ -41,7 +41,7 @@ void setUp() throws Exception { .close(); Mockito.when(applicationConfiguration.isHealthCheckEnabled()) .thenReturn(true); - applicationHealthCalculator = new ApplicationHealthCalculatorMock(objectStoreFileStorage, + applicationHealthCalculator = new ApplicationHealthCalculatorMock(jCloudsObjectStoreFileStorage, applicationConfiguration, databaseHealthService, databaseMonitoringService, @@ -51,7 +51,7 @@ void setUp() throws Exception { @Test void testUpdateWithFailingObjectStore() { Mockito.doThrow(new SLException("Object store not working")) - .when(objectStoreFileStorage) + .when(jCloudsObjectStoreFileStorage) .testConnection(); applicationHealthCalculator.updateHealthStatus(); ResponseEntity applicationHealthResultResponseEntity = applicationHealthCalculator.calculateApplicationHealth(); @@ -117,12 +117,12 @@ void testSuccessfulUpdateWithMissingObjectStore() { } private static class ApplicationHealthCalculatorMock extends ApplicationHealthCalculator { - public ApplicationHealthCalculatorMock(ObjectStoreFileStorage objectStoreFileStorage, + public ApplicationHealthCalculatorMock(JCloudsObjectStoreFileStorage JCloudsObjectStoreFileStorage, ApplicationConfiguration applicationConfiguration, DatabaseHealthService databaseHealthService, DatabaseMonitoringService databaseMonitoringService, DatabaseWaitingLocksAnalyzer databaseWaitingLocksAnalyzer) { - super(objectStoreFileStorage, + super(JCloudsObjectStoreFileStorage, applicationConfiguration, databaseHealthService, databaseMonitoringService, diff --git a/multiapps-controller-persistence/pom.xml b/multiapps-controller-persistence/pom.xml index c3faa8923c..6cdee7d418 100644 --- a/multiapps-controller-persistence/pom.xml +++ b/multiapps-controller-persistence/pom.xml @@ -98,17 +98,19 @@ org.apache.jclouds.provider azureblob - - org.apache.jclouds.provider - google-cloud-storage - - - org.apache.jclouds.common - googlecloud - org.apache.jclouds jclouds-blobstore + + + org.apache.jclouds.provider + google-cloud-storage + + + + + com.google.cloud + google-cloud-storage com.aliyun.oss @@ -151,6 +153,11 @@ org.apache.logging.log4j log4j-core + + com.google.cloud + google-cloud-nio + compile + org.slf4j slf4j-api diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index a1ce012fad..875fb82e2d 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -30,6 +30,8 @@ requires aliyun.sdk.oss; requires com.fasterxml.jackson.annotation; requires com.fasterxml.jackson.databind; + requires com.google.auth; + requires com.google.auth.oauth2; requires com.google.auto.service; requires com.google.common; requires com.google.guice; @@ -37,6 +39,10 @@ requires flowable.engine; requires flowable.engine.common.api; requires flowable.variable.service.api; + requires gax; + requires google.cloud.core; + requires google.cloud.nio; + requires google.cloud.storage; requires jakarta.inject; requires org.apache.logging.log4j; requires org.apache.logging.log4j.core; @@ -53,5 +59,4 @@ requires static org.immutables.value; requires jakarta.xml.bind; requires org.bouncycastle.fips.pkix; - } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java new file mode 100644 index 0000000000..ab81b1750f --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java @@ -0,0 +1,237 @@ +package org.cloudfoundry.multiapps.controller.persistence.services; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.channels.Channels; +import java.text.MessageFormat; +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import com.google.api.gax.retrying.RetrySettings; +import com.google.auth.Credentials; +import com.google.auth.oauth2.GoogleCredentials; +import com.google.cloud.ReadChannel; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.StorageOptions; +import com.google.cloud.storage.StorageRetryStrategy; +import org.cloudfoundry.multiapps.common.util.MiscUtil; +import org.cloudfoundry.multiapps.controller.persistence.Messages; +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; +import org.springframework.http.MediaType; + +public class GcpObjectStoreFileStorage implements FileStorage { + + private final String bucketName; + private final Storage storage; + private static final String BUCKET = "bucket"; + private static final int OBJECT_STORE_MAX_ATTEMPTS_CONFIG = 6; + private static final double OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG = 2.0; + private static final Duration OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES = Duration.ofMinutes(10); + private static final Duration OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS = Duration.ofSeconds(10); + private static final Duration OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS = Duration.ofMillis(250); + private static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData"; + + public GcpObjectStoreFileStorage(Map credentials) { + this.bucketName = (String) credentials.get(BUCKET); + this.storage = createObjectStoreStorage(credentials); + } + + protected Storage createObjectStoreStorage(Map credentials) { + return StorageOptions.http() + .setCredentials(getGcpCredentialsSupplier(credentials)) + .setStorageRetryStrategy(StorageRetryStrategy.getDefaultStorageRetryStrategy()) + .setRetrySettings( + RetrySettings.newBuilder() + .setMaxAttempts(OBJECT_STORE_MAX_ATTEMPTS_CONFIG) + .setTotalTimeoutDuration(OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES) + .setMaxRetryDelayDuration(OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS) + .setInitialRetryDelayDuration(OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS) + .setRetryDelayMultiplier(OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG) + .build()) + .build() + .getService(); + } + + private Credentials getGcpCredentialsSupplier(Map credentials) { + if (!credentials.containsKey(BASE_64_ENCODED_PRIVATE_KEY_DATA)) { + return null; + } + byte[] decodedKey = Base64.getDecoder() + .decode((String) credentials.get(BASE_64_ENCODED_PRIVATE_KEY_DATA)); + try { + return GoogleCredentials.fromStream(new ByteArrayInputStream(decodedKey)); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + @Override + public void addFile(FileEntry fileEntry, InputStream content) throws FileStorageException { + BlobId blobId = BlobId.of(bucketName, fileEntry.getId()); + BlobInfo blobInfo = BlobInfo.newBuilder(blobId) + .setContentDisposition(fileEntry.getName()) + .setContentType(MediaType.APPLICATION_OCTET_STREAM_VALUE) + .setMetadata(ObjectStoreMapper.createFileEntryMetadata(fileEntry)) + .build(); + + putBlob(blobInfo, content); + } + + private void putBlob(BlobInfo blobInfo, InputStream content) throws FileStorageException { + try { + storage.createFrom(blobInfo, content); + } catch (IOException | StorageException e) { + throw new FileStorageException(e); + } + } + + @Override + public List getFileEntriesWithoutContent(List fileEntries) { + Set existingFiles = getAllEntries().stream() + .map(Blob::getName) + .collect(Collectors.toSet()); + return fileEntries.stream() + .filter(fileEntry -> !existingFiles.contains(fileEntry.getId())) + .toList(); + } + + @Override + public void deleteFile(String id, String space) { + storage.delete(bucketName, id); + } + + @Override + public void deleteFilesBySpaceIds(List spaceIds) { + removeBlobsByFilter(blob -> ObjectStoreFilter.filterBySpaceIds(blob.getMetadata(), spaceIds)); + } + + @Override + public void deleteFilesBySpaceAndNamespace(String space, String namespace) { + removeBlobsByFilter(blob -> ObjectStoreFilter.filterBySpaceAndNamespace(blob.getMetadata(), space, namespace)); + } + + @Override + public int deleteFilesModifiedBefore(LocalDateTime modificationTime) { + return removeBlobsByFilter( + blob -> ObjectStoreFilter.filterByModificationTime(blob.getMetadata(), blob.getName(), modificationTime)); + } + + @Override + public T processFileContent(String space, String id, + FileContentProcessor fileContentProcessor) throws FileStorageException { + FileEntry fileEntry = ObjectStoreMapper.createFileEntry(space, id); + try (InputStream inputStream = openBlobStream(fileEntry)) { + return fileContentProcessor.process(inputStream); + } catch (Exception e) { + throw new FileStorageException(e); + } + } + + private InputStream openBlobStream(FileEntry fileEntry) throws FileStorageException { + Blob blob = getBlob(fileEntry); + return Channels.newInputStream(blob.reader()); + } + + private Blob getBlob(FileEntry fileEntry) throws FileStorageException { + try { + Blob blob = storage.get(bucketName, fileEntry.getId()); + if (blob == null) { + throw new FileStorageException( + MessageFormat.format(Messages.FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST, + fileEntry.getId(), fileEntry.getSpace())); + } + return blob; + } catch (StorageException e) { + throw new FileStorageException(e); + } + } + + @Override + public InputStream openInputStream(String space, String id) throws FileStorageException { + FileEntry fileEntry = ObjectStoreMapper.createFileEntry(space, id); + return openBlobStream(fileEntry); + } + + @Override + public void testConnection() { + storage.get(bucketName, "test"); + } + + @Override + public void deleteFilesByIds(List fileIds) { + removeBlobsByFilter(blob -> fileIds.contains(blob.getName())); + } + + @Override + public T processArchiveEntryContent(FileContentToProcess fileContentToProcess, FileContentProcessor fileContentProcessor) + throws FileStorageException { + FileEntry fileEntry = ObjectStoreMapper.createFileEntry(fileContentToProcess.getSpaceGuid(), fileContentToProcess.getGuid()); + InputStream blobPayload = getBlobPayloadWithOffset(fileEntry, fileContentToProcess.getStartOffset(), + fileContentToProcess.getEndOffset()); + return processContent(fileContentProcessor, blobPayload); + } + + private T processContent(FileContentProcessor fileContentProcessor, InputStream inputStream) + throws FileStorageException { + try { + return fileContentProcessor.process(inputStream); + } catch (IOException e) { + throw new FileStorageException(e); + } + } + + public Set getAllEntries() { + return storage.list(bucketName) + .streamAll() + .collect(Collectors.toSet()); + } + + protected int removeBlobsByFilter(Predicate filter) { + List blobIds = getEntryNames(filter).stream() + .map(entry -> BlobId.of(bucketName, entry)) + .toList(); + List deletedBlobsResults = new ArrayList<>(); + if (!blobIds.isEmpty()) { + deletedBlobsResults = storage.delete(blobIds); + } + return MiscUtil.cast(deletedBlobsResults.stream() + .filter(Boolean::booleanValue) + .count()); + } + + protected Set getEntryNames(Predicate filter) { + return storage.list(bucketName) + .streamAll() + .filter(filter) + .map(Blob::getName) + .collect(Collectors.toSet()); + } + + private InputStream getBlobPayloadWithOffset(FileEntry fileEntry, long startOffset, long endOffset) + throws FileStorageException { + try { + Blob blob = getBlob(fileEntry); + ReadChannel reader = storage.reader(blob.getBlobId()); + reader.seek(startOffset); + reader.limit(endOffset + 1); + + return Channels.newInputStream(reader); + } catch (IOException | StorageException e) { + throw new FileStorageException(e); + } + } +} diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java similarity index 68% rename from multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java rename to multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java index 977b5968df..2e5ec53acb 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java @@ -3,23 +3,19 @@ import java.io.IOException; import java.io.InputStream; import java.text.MessageFormat; -import java.time.Instant; import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import org.cloudfoundry.multiapps.common.util.MiscUtil; -import org.cloudfoundry.multiapps.controller.persistence.Constants; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; -import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.ContainerNotFoundException; import org.jclouds.blobstore.domain.Blob; @@ -33,18 +29,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; -import org.springframework.util.CollectionUtils; -public class ObjectStoreFileStorage implements FileStorage { +public class JCloudsObjectStoreFileStorage implements FileStorage { - private static final Logger LOGGER = LoggerFactory.getLogger(ObjectStoreFileStorage.class); + private static final Logger LOGGER = LoggerFactory.getLogger(JCloudsObjectStoreFileStorage.class); private static final int MAX_RETRIES_COUNT = 3; private static final long RETRY_BASE_WAIT_TIME_IN_MILLIS = 5000L; private final BlobStore blobStore; private final String container; - public ObjectStoreFileStorage(BlobStore blobStore, String container) { + public JCloudsObjectStoreFileStorage(BlobStore blobStore, String container) { this.blobStore = blobStore; this.container = container; } @@ -59,10 +54,10 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage .contentDisposition(fileEntry.getName()) .contentType(MediaType.APPLICATION_OCTET_STREAM_VALUE) .contentLength(fileSize) - .userMetadata(createFileEntryMetadata(fileEntry)) + .userMetadata(ObjectStoreMapper.createFileEntryMetadata(fileEntry)) .build(); try { - putBlobWithRetries(blob, 3); + putBlobWithRetries(blob, MAX_RETRIES_COUNT); LOGGER.debug(MessageFormat.format(Messages.STORED_FILE_0_WITH_SIZE_1, fileEntry.getId(), fileSize)); } catch (ContainerNotFoundException e) { throw new FileStorageException(MessageFormat.format(Messages.FILE_UPLOAD_FAILED, fileEntry.getName(), @@ -87,22 +82,23 @@ public void deleteFile(String id, String space) { @Override public void deleteFilesBySpaceIds(List spaceIds) { - removeBlobsByFilter(blob -> filterBySpaceIds(blob, spaceIds)); + removeBlobsByFilter(blob -> ObjectStoreFilter.filterBySpaceIds(blob.getUserMetadata(), spaceIds)); } @Override public void deleteFilesBySpaceAndNamespace(String space, String namespace) { - removeBlobsByFilter(blob -> filterBySpaceAndNamespace(blob, space, namespace)); + removeBlobsByFilter(blob -> ObjectStoreFilter.filterBySpaceAndNamespace(blob.getUserMetadata(), space, namespace)); } @Override public int deleteFilesModifiedBefore(LocalDateTime modificationTime) { - return removeBlobsByFilter(blob -> filterByModificationTime(blob, modificationTime)); + return removeBlobsByFilter( + blob -> ObjectStoreFilter.filterByModificationTime(blob.getUserMetadata(), blob.getName(), modificationTime)); } @Override public T processFileContent(String space, String id, FileContentProcessor fileContentProcessor) throws FileStorageException { - FileEntry fileEntry = createFileEntry(space, id); + FileEntry fileEntry = ObjectStoreMapper.createFileEntry(space, id); try { Payload payload = getBlobPayload(fileEntry); return processContent(fileContentProcessor, payload); @@ -114,7 +110,7 @@ public T processFileContent(String space, String id, FileContentProcessor @Override public T processArchiveEntryContent(FileContentToProcess fileContentToProcess, FileContentProcessor fileContentProcessor) throws FileStorageException { - FileEntry fileEntry = createFileEntry(fileContentToProcess.getSpaceGuid(), fileContentToProcess.getGuid()); + FileEntry fileEntry = ObjectStoreMapper.createFileEntry(fileContentToProcess.getSpaceGuid(), fileContentToProcess.getGuid()); try { Payload payload = getBlobPayloadWithOffset(fileEntry, fileContentToProcess.getStartOffset(), fileContentToProcess.getEndOffset()); @@ -150,7 +146,7 @@ private Blob getBlobWithRetriesWithOffset(FileEntry fileEntry, int retries, long } private Payload getBlobPayload(FileEntry fileEntry) throws FileStorageException { - Blob blob = getBlobWithRetries(fileEntry, 3); + Blob blob = getBlobWithRetries(fileEntry, MAX_RETRIES_COUNT); if (blob == null) { throw new FileStorageException(MessageFormat.format(Messages.FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST, fileEntry.getId(), fileEntry.getSpace())); @@ -160,7 +156,7 @@ private Payload getBlobPayload(FileEntry fileEntry) throws FileStorageException @Override public InputStream openInputStream(String space, String id) throws FileStorageException { - FileEntry fileEntry = createFileEntry(space, id); + FileEntry fileEntry = ObjectStoreMapper.createFileEntry(space, id); Payload payload = getBlobPayload(fileEntry); return openPayloadInputStream(payload); } @@ -183,13 +179,6 @@ public void deleteFilesByIds(List fileIds) { removeBlobsByFilter(blob -> fileIds.contains(blob.getName())); } - private FileEntry createFileEntry(String space, String id) { - return ImmutableFileEntry.builder() - .space(space) - .id(id) - .build(); - } - private T processContent(FileContentProcessor fileContentProcessor, Payload payload) throws FileStorageException { try (InputStream fileContentStream = payload.openStream()) { return fileContentProcessor.process(fileContentStream); @@ -232,19 +221,6 @@ protected long getRetryWaitTime() { return RETRY_BASE_WAIT_TIME_IN_MILLIS; } - private Map createFileEntryMetadata(FileEntry fileEntry) { - Map metadata = new HashMap<>(); - metadata.put(Constants.FILE_ENTRY_SPACE.toLowerCase(), fileEntry.getSpace()); - metadata.put(Constants.FILE_ENTRY_MODIFIED.toLowerCase(), Long.toString(fileEntry.getModified() - .atZone(ZoneId.systemDefault()) - .toInstant() - .toEpochMilli())); - if (fileEntry.getNamespace() != null) { - metadata.put(Constants.FILE_ENTRY_NAMESPACE.toLowerCase(), fileEntry.getNamespace()); - } - return metadata; - } - private int removeBlobsByFilter(Predicate filter) { Set entries = getEntryNames(filter); if (!entries.isEmpty()) { @@ -271,45 +247,4 @@ private Set getAllEntries(ListContainerOptions options) { } return entries; } - - private boolean filterByModificationTime(StorageMetadata blobMetadata, LocalDateTime modificationTime) { - Map userMetadata = blobMetadata.getUserMetadata(); - // Clean up any blobStore entries that don't have any metadata as we can't check their creation date - if (CollectionUtils.isEmpty(userMetadata)) { - LOGGER.warn(MessageFormat.format(Messages.USER_METADATA_OF_BLOB_0_EMPTY_AND_WILL_BE_DELETED, blobMetadata.getName())); - return true; - } - String longString = userMetadata.get(Constants.FILE_ENTRY_MODIFIED.toLowerCase()); - try { - long dateLong = Long.parseLong(longString); - LocalDateTime date = LocalDateTime.ofInstant(Instant.ofEpochMilli(dateLong), ZoneId.systemDefault()); - return date.isBefore(modificationTime); - } catch (NumberFormatException e) { - // Clean up any blobStore entries that have invalid timestamp - LOGGER.warn(MessageFormat.format(Messages.DATE_METADATA_OF_BLOB_0_IS_NOT_IN_PROPER_FORMAT_AND_WILL_BE_DELETED, - blobMetadata.getName()), - e); - return true; - } - } - - private boolean filterBySpaceIds(StorageMetadata blobMetadata, List spaceIds) { - Map userMetadata = blobMetadata.getUserMetadata(); - if (CollectionUtils.isEmpty(userMetadata)) { - return false; - } - String spaceParameter = userMetadata.get(Constants.FILE_ENTRY_SPACE.toLowerCase()); - return spaceIds.contains(spaceParameter); - } - - private boolean filterBySpaceAndNamespace(StorageMetadata blobMetadata, String space, String namespace) { - Map userMetadata = blobMetadata.getUserMetadata(); - if (CollectionUtils.isEmpty(userMetadata)) { - return false; - } - String spaceParameter = userMetadata.get(Constants.FILE_ENTRY_SPACE.toLowerCase()); - String namespaceParameter = userMetadata.get(Constants.FILE_ENTRY_NAMESPACE.toLowerCase()); - return space.equals(spaceParameter) && namespace.equals(namespaceParameter); - } - } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreFilter.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreFilter.java new file mode 100644 index 0000000000..22ee92687e --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreFilter.java @@ -0,0 +1,58 @@ +package org.cloudfoundry.multiapps.controller.persistence.util; + +import java.text.MessageFormat; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.List; +import java.util.Map; + +import org.cloudfoundry.multiapps.controller.persistence.Constants; +import org.cloudfoundry.multiapps.controller.persistence.Messages; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.CollectionUtils; + +public class ObjectStoreFilter { + + private static final Logger LOGGER = LoggerFactory.getLogger(ObjectStoreFilter.class); + + public static boolean filterBySpaceIds(Map metadata, List spaceIds) { + if (CollectionUtils.isEmpty(metadata)) { + return false; + } + String spaceParameter = metadata.get(Constants.FILE_ENTRY_SPACE.toLowerCase()); + + return spaceIds.contains(spaceParameter); + } + + public static boolean filterBySpaceAndNamespace(Map metadata, String space, String namespace) { + if (CollectionUtils.isEmpty(metadata)) { + return false; + } + + String spaceParameter = metadata.get(Constants.FILE_ENTRY_SPACE.toLowerCase()); + String namespaceParameter = metadata.get(Constants.FILE_ENTRY_NAMESPACE.toLowerCase()); + return space.equals(spaceParameter) && namespace.equals(namespaceParameter); + } + + public static boolean filterByModificationTime(Map metadata, String blobName, LocalDateTime modificationTime) { + // Clean up any blobStore entries that don't have any metadata as we can't check their creation date + if (CollectionUtils.isEmpty(metadata)) { + LOGGER.warn(MessageFormat.format(Messages.USER_METADATA_OF_BLOB_0_EMPTY_AND_WILL_BE_DELETED, blobName)); + return true; + } + String longString = metadata.get(Constants.FILE_ENTRY_MODIFIED.toLowerCase()); + try { + long dateLong = Long.parseLong(longString); + LocalDateTime date = LocalDateTime.ofInstant(Instant.ofEpochMilli(dateLong), ZoneId.systemDefault()); + return date.isBefore(modificationTime); + } catch (NumberFormatException e) { + // Clean up any blobStore entries that have invalid timestamp + LOGGER.warn(MessageFormat.format(Messages.DATE_METADATA_OF_BLOB_0_IS_NOT_IN_PROPER_FORMAT_AND_WILL_BE_DELETED, + blobName), + e); + return true; + } + } +} diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapper.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapper.java new file mode 100644 index 0000000000..1875b88a50 --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapper.java @@ -0,0 +1,35 @@ +package org.cloudfoundry.multiapps.controller.persistence.util; + +import java.time.ZoneId; +import java.util.HashMap; +import java.util.Map; + +import org.cloudfoundry.multiapps.controller.persistence.Constants; +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; + +public class ObjectStoreMapper { + + public static Map createFileEntryMetadata(FileEntry fileEntry) { + Map metadata = new HashMap<>(); + metadata.put(Constants.FILE_ENTRY_SPACE.toLowerCase(), fileEntry.getSpace()); + metadata.put(Constants.FILE_ENTRY_MODIFIED.toLowerCase(), + Long.toString(fileEntry.getModified() + .atZone( + ZoneId.systemDefault()) + .toInstant() + .toEpochMilli())); + if (fileEntry.getNamespace() != null) { + metadata.put(Constants.FILE_ENTRY_NAMESPACE.toLowerCase(), fileEntry.getNamespace()); + } + return metadata; + } + + public static FileEntry createFileEntry(String space, String id) { + return ImmutableFileEntry.builder() + .space(space) + .id(id) + .build(); + } + +} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java new file mode 100644 index 0000000000..cc4b0aa773 --- /dev/null +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java @@ -0,0 +1,95 @@ +package org.cloudfoundry.multiapps.controller.persistence.services; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.Predicate; + +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.springframework.http.MediaType; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +class GcpObjectStoreFileStorageTest extends JCloudsObjectStoreFileStorageTest { + + private Storage storage; + + @Override + @BeforeEach + public void setUp() { + storage = LocalStorageHelper.getOptions() + .getService(); + fileStorage = new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER)) { + + @Override + protected Storage createObjectStoreStorage(Map credentials) { + return storage; + } + + @Override + protected int removeBlobsByFilter(Predicate filter) { + Set entries = getEntryNames(filter); + for (String entry : entries) { + storage.delete(CONTAINER, entry); + } + + return entries.size(); + } + }; + spaceId = UUID.randomUUID() + .toString(); + namespace = UUID.randomUUID() + .toString(); + } + + @Override + @AfterEach + public void tearDown() { + if (storage != null) { + try { + storage.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + @Override + protected void assertBlobDoesNotExist(String blobWithNoMetadataId) { + assertNull(storage.get(blobWithNoMetadataId)); + } + + @Override + public String addBlobWithNoMetadata() throws Exception { + Path path = Paths.get(TEST_FILE_LOCATION); + String id = UUID.randomUUID() + .toString(); + BlobInfo blobInfo = BlobInfo.newBuilder(BlobId.of(CONTAINER, id)) + .setContentDisposition(path.getFileName() + .toString()) + .setContentType(MediaType.APPLICATION_OCTET_STREAM_VALUE) + .build(); + storage.create(blobInfo, Files.newInputStream(path)); + return id; + } + + @Override + public void assertFileExists(boolean exceptedFileExist, FileEntry actualFile) { + Blob blob = storage.get(CONTAINER, actualFile.getId()); + boolean blobExists = blob != null; + + assertEquals(exceptedFileExist, blobExists); + } + +} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java similarity index 88% rename from multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorageTest.java rename to multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java index ae530c6b69..6356afd641 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java @@ -1,10 +1,5 @@ package org.cloudfoundry.multiapps.controller.persistence.services; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; - import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -22,7 +17,6 @@ import java.util.UUID; import jakarta.xml.bind.DatatypeConverter; - import org.cloudfoundry.multiapps.common.util.DigestHelper; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; @@ -35,24 +29,29 @@ import org.junit.jupiter.api.Test; import org.springframework.http.MediaType; -class ObjectStoreFileStorageTest { +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class JCloudsObjectStoreFileStorageTest { - private static final String TEST_FILE_LOCATION = "src/test/resources/pexels-photo-401794.jpeg"; - private static final String SECOND_FILE_TEST_LOCATION = "src/test/resources/pexels-photo-463467.jpeg"; - private static final String DIGEST_METHOD = "MD5"; - private static final String CONTAINER = "container4e"; + protected static final String TEST_FILE_LOCATION = "src/test/resources/pexels-photo-401794.jpeg"; + protected static final String SECOND_FILE_TEST_LOCATION = "src/test/resources/pexels-photo-463467.jpeg"; + protected static final String DIGEST_METHOD = "MD5"; + protected static final String CONTAINER = "container4e"; - private String spaceId; - private String namespace; + protected String spaceId; + protected String namespace; - private FileStorage fileStorage; + protected FileStorage fileStorage; private BlobStoreContext blobStoreContext; @BeforeEach - public void setUp() { + protected void setUp() { createBlobStoreContext(); - fileStorage = new ObjectStoreFileStorage(blobStoreContext.getBlobStore(), CONTAINER) { + fileStorage = new JCloudsObjectStoreFileStorage(blobStoreContext.getBlobStore(), CONTAINER) { @Override protected long getRetryWaitTime() { return 1; @@ -72,7 +71,7 @@ private void createBlobStoreContext() { } @AfterEach - public void tearDown() { + protected void tearDown() { if (blobStoreContext != null) { blobStoreContext.close(); } @@ -163,6 +162,10 @@ void deleteFilesModifiedBefore() throws Exception { assertFileExists(true, fileEntryToRemain2); assertFileExists(false, fileEntryToDelete1); assertFileExists(false, fileEntryToDelete2); + assertBlobDoesNotExist(blobWithNoMetadataId); + } + + protected void assertBlobDoesNotExist(String blobWithNoMetadataId) { assertNull(blobStoreContext.getBlobStore() .getBlob(CONTAINER, blobWithNoMetadataId)); } @@ -176,11 +179,10 @@ void testConnection() { void testDeleteFilesByIds() throws Exception { FileEntry fileEntry = addFile(TEST_FILE_LOCATION); fileStorage.deleteFilesByIds(List.of(fileEntry.getId())); - assertNull(blobStoreContext.getBlobStore() - .getBlob(CONTAINER, fileEntry.getId())); + assertBlobDoesNotExist(fileEntry.getId()); } - private String addBlobWithNoMetadata() throws Exception { + protected String addBlobWithNoMetadata() throws Exception { BlobStore blobStore = blobStoreContext.getBlobStore(); Path path = Paths.get(TEST_FILE_LOCATION); long fileSize = Files.size(path); @@ -218,7 +220,7 @@ void testFileContentNotExisting() throws Exception { assertThrows(FileStorageException.class, () -> validateFileContent(dummyFileEntry, fileDigest)); } - private void validateFileContent(FileEntry storedFile, final String expectedFileChecksum) throws FileStorageException { + protected void validateFileContent(FileEntry storedFile, final String expectedFileChecksum) throws FileStorageException { fileStorage.processFileContent(storedFile.getSpace(), storedFile.getId(), contentStream -> { // make a digest out of the content and compare it to the original final byte[] digest = calculateFileDigest(contentStream); @@ -242,21 +244,21 @@ private byte[] calculateFileDigest(InputStream contentStream) throws IOException } } - private void addBigAmountOfEntries() throws Exception { + protected void addBigAmountOfEntries() throws Exception { for (int i = 0; i < 3001; i++) { addFileContent("test-file-" + i, "test".getBytes()); } } - private FileEntry addFile(String pathString) throws Exception { + protected FileEntry addFile(String pathString) throws Exception { return addFile(pathString, spaceId, namespace); } - private FileEntry addFile(String pathString, String space, String namespace) throws Exception { + protected FileEntry addFile(String pathString, String space, String namespace) throws Exception { return addFile(pathString, space, namespace, null); } - private FileEntry addFile(String pathString, String space, String namespace, LocalDateTime date) throws Exception { + protected FileEntry addFile(String pathString, String space, String namespace, LocalDateTime date) throws Exception { Path testFilePath = Paths.get(pathString) .toAbsolutePath(); FileEntry fileEntry = createFileEntry(space, namespace); @@ -311,12 +313,12 @@ private FileEntry enrichFileEntry(FileEntry fileEntry, Path path, LocalDateTime .build(); } - private void assertFileExists(boolean exceptedFileExist, FileEntry actualFile) { + protected void assertFileExists(boolean expectedFileExist, FileEntry actualFile) { Blob blob = blobStoreContext.getBlobStore() .getBlob(CONTAINER, actualFile.getId()); boolean blobExists = blob != null; - assertEquals(exceptedFileExist, blobExists); + assertEquals(expectedFileExist, blobExists); } } diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreFilterTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreFilterTest.java new file mode 100644 index 0000000000..a0b60a096b --- /dev/null +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreFilterTest.java @@ -0,0 +1,79 @@ +package org.cloudfoundry.multiapps.controller.persistence.util; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; + +import org.cloudfoundry.multiapps.controller.persistence.Constants; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ObjectStoreFilterTest { + + private static final String SPACE_ID = "spaceId"; + private static final String BLOBNAME = "blobName"; + private static final String SPACE_ID_2 = "spaceId2"; + private static final String NAMESPACE = "namespace"; + private static final String NAMESPACE_2 = "namespace2"; + + @Test + void testFilterBySpaceIdsWhenMetadataIsEmpty() { + assertFalse(ObjectStoreFilter.filterBySpaceIds(Map.of(), List.of())); + } + + @Test + void testFilterBySpaceIdsWhenTheMetadataContainsTheSpaceId() { + assertTrue(ObjectStoreFilter.filterBySpaceIds(buildMetadata(), List.of(SPACE_ID))); + } + + @Test + void testFilterBySpaceIdsWhenTheMetadataDoesNotContainTheSpaceId() { + assertFalse(ObjectStoreFilter.filterBySpaceIds(buildMetadata(), List.of(SPACE_ID_2))); + } + + @Test + void testFilterBySpaceAndNamespaceWhenMetadataIsEmpty() { + assertFalse(ObjectStoreFilter.filterBySpaceAndNamespace(Map.of(), SPACE_ID, NAMESPACE)); + } + + @Test + void testFilterBySpaceAndNamespaceWhenTheSpacesAreDifferent() { + assertFalse(ObjectStoreFilter.filterBySpaceAndNamespace(buildMetadata(), SPACE_ID_2, NAMESPACE)); + } + + @Test + void testFilterBySpaceAndNamespaceWhenTheNamespacesAreDifferent() { + assertFalse(ObjectStoreFilter.filterBySpaceAndNamespace(buildMetadata(), SPACE_ID, NAMESPACE_2)); + } + + @Test + void testFilterBySpaceAndNamespaceWhenAllMatch() { + assertTrue(ObjectStoreFilter.filterBySpaceAndNamespace(buildMetadata(), SPACE_ID, NAMESPACE)); + } + + @Test + void testFilterByModificationTimeWhenMetadataIsEmpty() { + assertTrue(ObjectStoreFilter.filterByModificationTime(Map.of(), BLOBNAME, LocalDateTime.now())); + } + + @Test + void testFilterByModificationTimeWhenModificationTimeIsBeforeTheMetadataTime() { + assertFalse(ObjectStoreFilter.filterByModificationTime(buildMetadata(), BLOBNAME, LocalDateTime.now() + .minusMinutes(20))); + } + + @Test + void testFilterByModificationTimeWhenModificationTimeIsAfterTheMetadataTime() { + assertTrue(ObjectStoreFilter.filterByModificationTime(buildMetadata(), BLOBNAME, LocalDateTime.now() + .plusMinutes(20))); + } + + private Map buildMetadata() { + return Map.of(Constants.FILE_ENTRY_SPACE.toLowerCase(), SPACE_ID, Constants.FILE_ENTRY_NAMESPACE.toLowerCase(), NAMESPACE, + Constants.FILE_ENTRY_MODIFIED.toLowerCase(), String.valueOf(Instant.now() + .toEpochMilli())); + } +} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapperTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapperTest.java new file mode 100644 index 0000000000..d0cd9a23c3 --- /dev/null +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapperTest.java @@ -0,0 +1,52 @@ +package org.cloudfoundry.multiapps.controller.persistence.util; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Map; +import java.util.UUID; + +import org.cloudfoundry.multiapps.controller.persistence.Constants; +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class ObjectStoreMapperTest { + + private static final String SPACE_ID = "spaceId"; + private static final String NAMESPACE = "namespace"; + + @Test + void testCreateFileEntryMetadata() { + LocalDateTime modifiedTime = LocalDateTime.now(); + String modifiedTimeInstantString = Long.toString(modifiedTime + .atZone( + ZoneId.systemDefault()) + .toInstant() + .toEpochMilli()); + Map metadata = ObjectStoreMapper.createFileEntryMetadata(buildCreateFileEntry(modifiedTime)); + + assertEquals(SPACE_ID, metadata.get(Constants.FILE_ENTRY_SPACE.toLowerCase())); + assertEquals(modifiedTimeInstantString, metadata.get(Constants.FILE_ENTRY_MODIFIED.toLowerCase())); + assertEquals(NAMESPACE, metadata.get(Constants.FILE_ENTRY_NAMESPACE.toLowerCase())); + } + + @Test + void testCreateFileEntry() { + String id = UUID.randomUUID() + .toString(); + FileEntry fileEntry = ObjectStoreMapper.createFileEntry(SPACE_ID, id); + + assertEquals(SPACE_ID, fileEntry.getSpace()); + assertEquals(id, fileEntry.getId()); + } + + private FileEntry buildCreateFileEntry(LocalDateTime modifiedTime) { + return ImmutableFileEntry.builder() + .space(SPACE_ID) + .modified(modifiedTime) + .namespace(NAMESPACE) + .build(); + } +} diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java index e2c264a8d0..de999943a7 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java @@ -327,7 +327,7 @@ public class Messages { public static final String DELETED_FILE_ENTRIES_0 = "Deleted file entries: {0}"; public static final String APPLICATION_WITH_NAME_0_SAVED_TO_1 = "Application with name \"{0}\" saved to \"{1}\""; public static final String CLOSING_STREAM_FOR_PART_0 = "Closing stream for part: {0}"; - public static final String CLOSING_STREAM_FOR_PART_STREAM_FINISHED_0 = "Closing stream for part, stream finished: {0}"; + public static final String REACHED_THE_END_OF_THE_INPUTSTREAM = "Reached the end of the InputStream"; public static final String CLOSING_LAST_STREAM_FOR_PART_0 = "Closing the last stream, part: {0}"; public static final String SCALING_UP_OLD_APPLICATION = "Scaling up old application: \"{0}\" to {1} instances"; public static final String FILE_WITH_ID_0_WAS_DELETED = "File with id \"{0}\" was deleted"; diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/stream/LazyArchiveInputStream.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/stream/LazyArchiveInputStream.java index 89daad8fc3..ffc2f886ed 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/stream/LazyArchiveInputStream.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/stream/LazyArchiveInputStream.java @@ -28,7 +28,6 @@ public class LazyArchiveInputStream extends InputStream { private final long archiveSize; private final AtomicInteger totalBytesRead; private final AtomicInteger partIndex; - private InputStream currentInputStream; public LazyArchiveInputStream(FileService fileService, List archiveFileEntries, StepLogger stepLogger, long archiveSize) { @@ -52,8 +51,7 @@ public synchronized int read() throws IOException { currentInputStream = openBufferedInputStream(archiveFileEntries.get(partIndex.incrementAndGet())); c = currentInputStream.read(); } else if (c == -1) { - LOGGER.info(MessageFormat.format(Messages.CLOSING_STREAM_FOR_PART_STREAM_FINISHED_0, partIndex)); - IOUtils.closeQuietly(currentInputStream, e -> LOGGER.warn(e.getMessage(), e)); + LOGGER.info(MessageFormat.format(Messages.REACHED_THE_END_OF_THE_INPUTSTREAM, partIndex)); } if (c >= 0) { totalBytesRead.incrementAndGet(); @@ -73,8 +71,7 @@ public synchronized int read(byte[] b, int off, int len) throws IOException { currentInputStream = openBufferedInputStream(archiveFileEntries.get(partIndex.incrementAndGet())); bytesRead = currentInputStream.read(b, off, len); } else if (bytesRead == -1) { - LOGGER.info(MessageFormat.format(Messages.CLOSING_STREAM_FOR_PART_STREAM_FINISHED_0, partIndex)); - IOUtils.closeQuietly(currentInputStream, e -> LOGGER.warn(e.getMessage(), e)); + LOGGER.info(MessageFormat.format(Messages.REACHED_THE_END_OF_THE_INPUTSTREAM, partIndex)); } if (bytesRead > 0) { totalBytesRead.addAndGet(bytesRead); @@ -107,6 +104,8 @@ private BufferedInputStream openBufferedInputStream(FileEntry archiveFileEntry) try { stepLogger.debug(Messages.OPENING_A_NEW_INPUT_STREAM_FOR_FILE_WITH_ID_0_AND_NAME_1, archiveFileEntry.getId(), archiveFileEntry.getName()); + LOGGER.info(MessageFormat.format(Messages.OPENING_A_NEW_INPUT_STREAM_FOR_FILE_WITH_ID_0_AND_NAME_1, archiveFileEntry.getId(), + archiveFileEntry.getName())); InputStream inputStream = fileService.openInputStream(archiveFileEntry.getSpace(), archiveFileEntry.getId()); return new BufferedInputStream(inputStream, BUFFERED_SIZE); } catch (FileStorageException e) { diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/stream/LazyArchiveInputStreamTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/stream/LazyArchiveInputStreamTest.java index 8216949aa5..b366865068 100644 --- a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/stream/LazyArchiveInputStreamTest.java +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/stream/LazyArchiveInputStreamTest.java @@ -1,12 +1,5 @@ package org.cloudfoundry.multiapps.controller.process.stream; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; @@ -22,6 +15,13 @@ import org.mockito.Mock; import org.mockito.MockitoAnnotations; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + class LazyArchiveInputStreamTest { private static final String FILE_ID_1 = "file-1"; @@ -71,7 +71,6 @@ void readWhenEOFReached() throws IOException, FileStorageException { assertEquals(-1, read); assertEquals(0, lazyArchiveInputStream.available()); verify(firstPartInputStream).close(); - verify(secondPartInputStream).close(); } @Test @@ -104,7 +103,6 @@ void readBufferedWhenEOFReached() throws IOException, FileStorageException { assertEquals(-1, read); assertEquals(0, lazyArchiveInputStream.available()); verify(firstPartInputStream).close(); - verify(secondPartInputStream).close(); } private void prepareStream(int archiveSize) { diff --git a/multiapps-controller-web/pom.xml b/multiapps-controller-web/pom.xml index dd056e63f0..d31b2e25fd 100644 --- a/multiapps-controller-web/pom.xml +++ b/multiapps-controller-web/pom.xml @@ -199,5 +199,13 @@ javax.xml.bind jaxb-api + + com.google.cloud + google-cloud-storage + + + com.google.cloud + google-cloud-nio + diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/FileServiceFactoryBean.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/FileServiceFactoryBean.java index 68a076ba3d..40d7728836 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/FileServiceFactoryBean.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/FileServiceFactoryBean.java @@ -2,12 +2,10 @@ import jakarta.inject.Inject; import jakarta.inject.Named; - import org.cloudfoundry.multiapps.controller.persistence.DataSourceWithDialect; import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseFileService; import org.cloudfoundry.multiapps.controller.persistence.services.FileService; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; -import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.web.Messages; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,15 +21,15 @@ public class FileServiceFactoryBean implements FactoryBean, Initial @Inject private DataSourceWithDialect dataSourceWithDialect; @Autowired(required = false) - private ObjectStoreFileStorage objectStoreFileStorage; + private FileStorage objectStoreFileStorage; + private FileService fileService; @Override public void afterPropertiesSet() { - FileStorage fileStorage = objectStoreFileStorage; - if (fileStorage != null) { + if (objectStoreFileStorage != null) { LOGGER.info(Messages.OBJECTSTORE_FOR_BINARIES_STORAGE); - this.fileService = new FileService(dataSourceWithDialect, fileStorage); + this.fileService = new FileService(dataSourceWithDialect, objectStoreFileStorage); } else { LOGGER.info(Messages.DATABASE_FOR_BINARIES_STORAGE); this.fileService = new DatabaseFileService(dataSourceWithDialect); diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java index 9d84a5507b..0001e13be1 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java @@ -16,10 +16,13 @@ import org.apache.commons.lang3.StringUtils; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.util.UriUtil; -import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage; +import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; +import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; +import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder; import org.cloudfoundry.multiapps.controller.web.Constants; import org.cloudfoundry.multiapps.controller.web.Messages; +import org.cloudfoundry.multiapps.controller.web.configuration.service.ImmutableObjectStoreServiceInfo; import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfo; import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfoCreator; import org.jclouds.ContextBuilder; @@ -30,7 +33,7 @@ import org.springframework.beans.factory.FactoryBean; import org.springframework.beans.factory.InitializingBean; -public class ObjectStoreFileStorageFactoryBean implements FactoryBean, InitializingBean { +public class ObjectStoreFileStorageFactoryBean implements FactoryBean, InitializingBean { private static final Logger LOGGER = LoggerFactory.getLogger(ObjectStoreFileStorageFactoryBean.class); private static final Set CUSTOM_REGIONS = Set.of("eu-south-1"); @@ -39,7 +42,7 @@ public class ObjectStoreFileStorageFactoryBean implements FactoryBean providersServiceInfo = getProvidersServiceInfo(); if (providersServiceInfo.isEmpty()) { return null; @@ -64,31 +67,46 @@ private ObjectStoreFileStorage createObjectStoreFileStorage() { return createObjectStoreFromFirstReachableProvider(exceptions, providersServiceInfo); } - Optional objectStoreServiceInfoOptional = getAppropriateProvider(objectStoreProviderName, - providersServiceInfo); + Optional optionalFileStorage = createObjectStoreBasedOnProvider(objectStoreProviderName, providersServiceInfo, + exceptions); - if (objectStoreServiceInfoOptional.isPresent()) { - ObjectStoreServiceInfo objectStoreServiceInfo = objectStoreServiceInfoOptional.get(); - Optional createdObjectStore = tryToCreateObjectStore(objectStoreServiceInfo, exceptions); - if (createdObjectStore.isPresent()) { - return createdObjectStore.get(); - } + if (optionalFileStorage.isPresent()) { + return optionalFileStorage.get(); } throw buildNoValidObjectStoreException(exceptions); } - public ObjectStoreFileStorage createObjectStoreFromFirstReachableProvider(Map exceptions, - List providersServiceInfo) { + public FileStorage createObjectStoreFromFirstReachableProvider(Map exceptions, + List providersServiceInfo) { for (ObjectStoreServiceInfo objectStoreServiceInfo : providersServiceInfo) { - Optional createdObjectStoreOptional = tryToCreateObjectStore(objectStoreServiceInfo, exceptions); - if (createdObjectStoreOptional.isPresent()) { - return createdObjectStoreOptional.get(); + Optional createdObjectStore = tryToCreateObjectStore(objectStoreServiceInfo, exceptions); + if (createdObjectStore.isPresent()) { + return createdObjectStore.get(); } } + Optional gcpObjectStoreOpt = tryToCreateGcpObjectStore(exceptions); + if (gcpObjectStoreOpt.isPresent()) { + return gcpObjectStoreOpt.get(); + } throw buildNoValidObjectStoreException(exceptions); } + private Optional createObjectStoreBasedOnProvider(String objectStoreProviderName, + List providersServiceInfo, + Map exceptions) { + Optional objectStoreServiceInfoOptional = getAppropriateProvider(objectStoreProviderName, + providersServiceInfo); + Optional createdObjectStore; + if (objectStoreServiceInfoOptional.isPresent()) { + ObjectStoreServiceInfo objectStoreServiceInfo = objectStoreServiceInfoOptional.get(); + createdObjectStore = tryToCreateObjectStore(objectStoreServiceInfo, exceptions); + } else { + createdObjectStore = tryToCreateGcpObjectStore(exceptions); + } + return createdObjectStore; + } + private Optional getAppropriateProvider(String objectStoreProviderName, List providersServiceInfo) { String appropriateProvider = Constants.ENV_TO_OS_PROVIDER.get(objectStoreProviderName); @@ -97,6 +115,34 @@ private Optional getAppropriateProvider(String objectSto .findFirst(); } + private Optional tryToCreateGcpObjectStore(Map exceptions) { + return tryToCreateObjectStore(ImmutableObjectStoreServiceInfo.builder() + .provider(Constants.GOOGLE_CLOUD_STORAGE) + .build(), exceptions); + } + + private Optional tryToCreateObjectStore(ObjectStoreServiceInfo objectStoreServiceInfo, + Map exceptions) { + try { + FileStorage fileStorage = getFileStorageBasedOnProvider(objectStoreServiceInfo); + fileStorage.testConnection(); + LOGGER.info(MessageFormat.format(Messages.OBJECT_STORE_WITH_PROVIDER_0_CREATED, objectStoreServiceInfo.getProvider())); + return Optional.of(fileStorage); + } catch (Exception e) { + exceptions.put(objectStoreServiceInfo.getProvider(), e); + return Optional.empty(); + } + } + + private FileStorage getFileStorageBasedOnProvider(ObjectStoreServiceInfo objectStoreServiceInfo) { + if (Constants.GOOGLE_CLOUD_STORAGE.equals(objectStoreServiceInfo.getProvider())) { + return createGcpFileStorage(); + } else { + BlobStoreContext context = getBlobStoreContext(objectStoreServiceInfo); + return createFileStorage(objectStoreServiceInfo, context); + } + } + private boolean isObjectStoreEnvValid(String objectStoreProviderName) { return objectStoreProviderName != null && !objectStoreProviderName.isEmpty() && Constants.ENV_TO_OS_PROVIDER.containsKey( objectStoreProviderName); @@ -109,26 +155,21 @@ private IllegalStateException buildNoValidObjectStoreException(Map tryToCreateObjectStore(ObjectStoreServiceInfo objectStoreServiceInfo, - Map exceptions) { - try { - BlobStoreContext context = getBlobStoreContext(objectStoreServiceInfo); - ObjectStoreFileStorage fileStorage = createFileStorage(objectStoreServiceInfo, context); - fileStorage.testConnection(); - LOGGER.info(MessageFormat.format(Messages.OBJECT_STORE_WITH_PROVIDER_0_CREATED, objectStoreServiceInfo.getProvider())); - return Optional.of(fileStorage); - } catch (Exception e) { - exceptions.put(objectStoreServiceInfo.getProvider(), e); - return Optional.empty(); + public List getProvidersServiceInfo() { + Map credentials = getServiceCredentials(); + if (credentials.isEmpty()) { + return Collections.emptyList(); } + return new ObjectStoreServiceInfoCreator().getAllProvidersServiceInfo(credentials); } - private List getProvidersServiceInfo() { + private Map getServiceCredentials() { CfService service = environmentServicesFinder.findService(serviceName); if (service == null) { - return Collections.emptyList(); + return Map.of(); } - return new ObjectStoreServiceInfoCreator().getAllProvidersServiceInfo(service); + return service.getCredentials() + .getMap(); } private BlobStoreContext getBlobStoreContext(ObjectStoreServiceInfo serviceInfo) { @@ -181,18 +222,23 @@ private void resolveContextEndpoint(ObjectStoreServiceInfo serviceInfo, ContextB } } - protected ObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) { - return new ObjectStoreFileStorage(context.getBlobStore(), objectStoreServiceInfo.getContainer()); + protected JCloudsObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) { + return new JCloudsObjectStoreFileStorage(context.getBlobStore(), objectStoreServiceInfo.getContainer()); + } + + protected GcpObjectStoreFileStorage createGcpFileStorage() { + Map credentials = getServiceCredentials(); + return new GcpObjectStoreFileStorage(credentials); } @Override - public ObjectStoreFileStorage getObject() { + public FileStorage getObject() { return objectStoreFileStorage; } @Override public Class getObjectType() { - return ObjectStoreFileStorage.class; + return FileStorage.class; } } \ No newline at end of file diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java index bad4dbd804..86c445e306 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java @@ -30,5 +30,4 @@ public interface ObjectStoreServiceInfo { @Nullable String getHost(); - } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java index 5b3f9cc4b1..57fd802b88 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java @@ -2,26 +2,17 @@ import java.net.MalformedURLException; import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.Base64; import java.util.List; import java.util.Map; -import com.google.common.base.Supplier; -import io.pivotal.cfenv.core.CfService; import org.cloudfoundry.multiapps.controller.web.Constants; import org.cloudfoundry.multiapps.controller.web.Messages; -import org.jclouds.domain.Credentials; -import org.jclouds.googlecloud.GoogleCredentialsFromJson; public class ObjectStoreServiceInfoCreator { - public List getAllProvidersServiceInfo(CfService service) { - Map credentials = service.getCredentials() - .getMap(); + public List getAllProvidersServiceInfo(Map credentials) { return List.of(createServiceInfoForAws(credentials), createServiceInfoForAliCloud(credentials), - createServiceInfoForAzure(credentials), createServiceInfoForGcpCloud(credentials), - createServiceInfoForCcee(credentials)); + createServiceInfoForAzure(credentials), createServiceInfoForCcee(credentials)); } private ObjectStoreServiceInfo createServiceInfoForAws(Map credentials) { @@ -95,27 +86,4 @@ private URL getContainerUriEndpoint(Map credentials) { throw new IllegalStateException(Messages.CANNOT_PARSE_CONTAINER_URI_OF_OBJECT_STORE, e); } } - - private ObjectStoreServiceInfo createServiceInfoForGcpCloud(Map credentials) { - String bucket = (String) credentials.get(Constants.BUCKET); - String region = (String) credentials.get(Constants.REGION); - Supplier credentialsSupplier = getGcpCredentialsSupplier(credentials); - return ImmutableObjectStoreServiceInfo.builder() - .provider(Constants.GOOGLE_CLOUD_STORAGE) - .credentialsSupplier(credentialsSupplier) - .container(bucket) - .region(region) - .build(); - } - - protected Supplier getGcpCredentialsSupplier(Map credentials) { - if (!credentials.containsKey(Constants.BASE_64_ENCODED_PRIVATE_KEY_DATA)) { - return () -> null; - } - byte[] decodedKey = Base64.getDecoder() - .decode((String) credentials.get(Constants.BASE_64_ENCODED_PRIVATE_KEY_DATA)); - String decodedCredential = new String(decodedKey, StandardCharsets.UTF_8); - return new GoogleCredentialsFromJson(decodedCredential); - } - } diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java index 8cdb10fa93..5b5617b7a6 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java @@ -1,17 +1,21 @@ package org.cloudfoundry.multiapps.controller.web.configuration.bean.factory; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import io.pivotal.cfenv.core.CfCredentials; import io.pivotal.cfenv.core.CfService; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; -import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage; +import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; +import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; +import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder; import org.cloudfoundry.multiapps.controller.web.Constants; import org.cloudfoundry.multiapps.controller.web.Messages; import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfo; +import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfoCreator; import org.jclouds.blobstore.BlobStoreContext; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -45,7 +49,10 @@ class ObjectStoreFileStorageFactoryBeanTest { @Mock private ApplicationConfiguration applicationConfiguration; @Mock - private ObjectStoreFileStorage objectStoreFileStorage; + private JCloudsObjectStoreFileStorage jCloudsObjectStoreFileStorage; + + @Mock + private GcpObjectStoreFileStorage gcpObjectStoreFileStorage; @BeforeEach void setUp() throws Exception { @@ -59,7 +66,7 @@ void setUp() throws Exception { @Test void testObjectStoreCreationWithoutServiceInstance() { objectStoreFileStorageFactoryBean.afterPropertiesSet(); - ObjectStoreFileStorage objectStoreFileStorage = objectStoreFileStorageFactoryBean.getObject(); + FileStorage objectStoreFileStorage = objectStoreFileStorageFactoryBean.getObject(); assertNull(objectStoreFileStorage); } @@ -67,7 +74,7 @@ void testObjectStoreCreationWithoutServiceInstance() { void testObjectStoreCreationWithValidServiceInstance() { mockCfService(); objectStoreFileStorageFactoryBean.afterPropertiesSet(); - ObjectStoreFileStorage objectStoreFileStorage = objectStoreFileStorageFactoryBean.getObject(); + FileStorage objectStoreFileStorage = objectStoreFileStorageFactoryBean.getObject(); assertNotNull(objectStoreFileStorage); } @@ -78,12 +85,12 @@ void testObjectStoreCreationWhenEnvIsValid() { ObjectStoreFileStorageFactoryBean spy = spy(objectStoreFileStorageFactoryBean); spy.afterPropertiesSet(); - ObjectStoreFileStorage createdObjectStoreFileStorage = spy.getObject(); + FileStorage createdObjectStoreFileStorage = spy.getObject(); assertNotNull(createdObjectStoreFileStorage); verify(spy, never()) .createObjectStoreFromFirstReachableProvider(anyMap(), anyList()); - verify(objectStoreFileStorage, times(1)) + verify(jCloudsObjectStoreFileStorage, times(1)) .testConnection(); } @@ -95,7 +102,7 @@ void testObjectStoreCreationWhenEnvIsInvalid() { ObjectStoreFileStorageFactoryBean spy = spy(objectStoreFileStorageFactoryBean); spy.afterPropertiesSet(); - ObjectStoreFileStorage createdObjectStoreFileStorage = spy.getObject(); + FileStorage createdObjectStoreFileStorage = spy.getObject(); assertNotNull(createdObjectStoreFileStorage); verify(spy, times(1)) @@ -106,7 +113,7 @@ void testObjectStoreCreationWhenEnvIsInvalid() { void testObjectStoreCreationWhenEnvProviderFailsToConnect() { mockCfService(); when(applicationConfiguration.getObjectStoreClientType()).thenReturn(Constants.AWS); - doThrow(new IllegalStateException("Cannot create object store")).when(objectStoreFileStorage) + doThrow(new IllegalStateException("Cannot create object store")).when(jCloudsObjectStoreFileStorage) .testConnection(); Exception exception = assertThrows(IllegalStateException.class, () -> objectStoreFileStorageFactoryBean.afterPropertiesSet()); @@ -116,7 +123,9 @@ void testObjectStoreCreationWhenEnvProviderFailsToConnect() { @Test void testObjectStoreCreationWithoutValidServiceInstance() { mockCfService(); - doThrow(new IllegalStateException("Cannot create object store")).when(objectStoreFileStorage) + doThrow(new IllegalStateException("Cannot create object store")).when(jCloudsObjectStoreFileStorage) + .testConnection(); + doThrow(new IllegalStateException("Cannot create object store")).when(gcpObjectStoreFileStorage) .testConnection(); Exception exception = assertThrows(IllegalStateException.class, () -> objectStoreFileStorageFactoryBean.afterPropertiesSet()); assertEquals(Messages.NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND, exception.getMessage()); @@ -146,8 +155,27 @@ public ObjectStoreFileStorageFactoryBeanMock(String serviceName, EnvironmentServ } @Override - protected ObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) { - return ObjectStoreFileStorageFactoryBeanTest.this.objectStoreFileStorage; + protected JCloudsObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) { + return ObjectStoreFileStorageFactoryBeanTest.this.jCloudsObjectStoreFileStorage; + } + + @Override + protected GcpObjectStoreFileStorage createGcpFileStorage() { + return ObjectStoreFileStorageFactoryBeanTest.this.gcpObjectStoreFileStorage; } + + @Override + public List getProvidersServiceInfo() { + CfService service = environmentServicesFinder.findService("deploy-service-os"); + if (service != null) { + return new ObjectStoreServiceInfoCreatorMock().getAllProvidersServiceInfo(service.getCredentials() + .getMap()); + } else { + return List.of(); + } + } + } + + private class ObjectStoreServiceInfoCreatorMock extends ObjectStoreServiceInfoCreator { } } diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java index 38e4216f7d..6439250543 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java @@ -1,29 +1,23 @@ package org.cloudfoundry.multiapps.controller.web.configuration.service; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.when; - import java.net.MalformedURLException; import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.Base64; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Stream; +import io.pivotal.cfenv.core.CfCredentials; +import io.pivotal.cfenv.core.CfService; import org.cloudfoundry.multiapps.controller.web.Constants; -import org.jclouds.domain.Credentials; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mockito; -import com.google.common.base.Supplier; - -import io.pivotal.cfenv.core.CfCredentials; -import io.pivotal.cfenv.core.CfService; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; class ObjectStoreServiceInfoCreatorTest { @@ -36,7 +30,6 @@ class ObjectStoreServiceInfoCreatorTest { private static final String SAS_TOKEN_VALUE = "sas_token_value"; private static final String CONTAINER_NAME_VALUE = "container_name_value"; private static final String CONTAINER_URI_VALUE = "https://container.com:8080"; - private static final Supplier CREDENTIALS_SUPPLIER = () -> null; private ObjectStoreServiceInfoCreator objectStoreServiceInfoCreator; @@ -48,14 +41,15 @@ void setUp() { static Stream testDifferentProviders() throws MalformedURLException { return Stream.of(Arguments.of(buildCfService(buildAliCloudCredentials()), buildAliCloudObjectStoreServiceInfo()), Arguments.of(buildCfService(buildAwsCredentials()), buildAwsObjectStoreServiceInfo()), - Arguments.of(buildCfService(buildAzureCredentials()), buildAzureObjectStoreServiceInfo()), - Arguments.of(buildCfService(buildGcpCredentials()), buildGcpObjectStoreServiceInfo())); + Arguments.of(buildCfService(buildAzureCredentials()), buildAzureObjectStoreServiceInfo())); } @ParameterizedTest @MethodSource void testDifferentProviders(CfService cfService, ObjectStoreServiceInfo exprectedObjectStoreServiceInfo) { - List providersServiceInfo = objectStoreServiceInfoCreator.getAllProvidersServiceInfo(cfService); + List providersServiceInfo = objectStoreServiceInfoCreator.getAllProvidersServiceInfo( + cfService.getCredentials() + .getMap()); assertTrue(providersServiceInfo.contains(exprectedObjectStoreServiceInfo)); } @@ -124,29 +118,6 @@ private static ObjectStoreServiceInfo buildAzureObjectStoreServiceInfo() throws .build(); } - private static Map buildGcpCredentials() { - Map credentials = new HashMap<>(); - credentials.put(Constants.BUCKET, BUCKET_VALUE); - credentials.put(Constants.REGION, REGION_VALUE); - credentials.put(Constants.BASE_64_ENCODED_PRIVATE_KEY_DATA, Base64.getEncoder() - .encodeToString("encoded_data".getBytes(StandardCharsets.UTF_8))); - return credentials; - } - - private static ObjectStoreServiceInfo buildGcpObjectStoreServiceInfo() { - return ImmutableObjectStoreServiceInfo.builder() - .provider(Constants.GOOGLE_CLOUD_STORAGE) - .credentialsSupplier(CREDENTIALS_SUPPLIER) - .container(BUCKET_VALUE) - .region(REGION_VALUE) - .build(); - } - private static class ObjectStoreServiceInfoCreatorMock extends ObjectStoreServiceInfoCreator { - - @Override - protected Supplier getGcpCredentialsSupplier(Map credentials) { - return ObjectStoreServiceInfoCreatorTest.CREDENTIALS_SUPPLIER; - } } } diff --git a/pom.xml b/pom.xml index de1d1d3592..7ff0a2110a 100644 --- a/pom.xml +++ b/pom.xml @@ -57,6 +57,8 @@ 1.28.0 3.2.2 1.3.1 + 2.59.0 + 0.128.7 multiapps-controller-client @@ -288,6 +290,17 @@ commons-compress ${apache.compress.version} + + com.google.cloud + google-cloud-storage + ${google-cloud.version} + + + com.google.cloud + google-cloud-nio + ${google-cloud-nio.version} + test + org.immutables