From eecf35afd426d5a4cf1c7ea60c22eafca23bd69e Mon Sep 17 00:00:00 2001 From: Yavor16 Date: Wed, 15 Oct 2025 08:32:10 +0300 Subject: [PATCH 01/12] Add GCP Objectstore SDK instead of JClouds LMCROSSITXSADEPLOY-3313 --- .../health/ApplicationHealthCalculator.java | 7 +- multiapps-controller-persistence/pom.xml | 23 +- .../src/main/java/module-info.java | 5 +- .../services/GcpObjectStoreFileStorage.java | 234 +++++++++++++ .../services/ObjectStoreFileStorage.java | 84 +---- .../persistence/util/ObjectStoreUtil.java | 84 +++++ .../GcpObjectStoreFileStorageTest.java | 311 ++++++++++++++++++ .../persistence/util/ObjectStoreUtilTest.java | 117 +++++++ multiapps-controller-web/pom.xml | 4 + .../bean/factory/FileServiceFactoryBean.java | 10 +- .../ObjectStoreFileStorageFactoryBean.java | 41 ++- .../service/ObjectStoreServiceInfo.java | 4 + .../ObjectStoreServiceInfoCreator.java | 38 ++- ...ObjectStoreFileStorageFactoryBeanTest.java | 43 ++- .../ObjectStoreServiceInfoCreatorTest.java | 26 +- pom.xml | 12 + 16 files changed, 904 insertions(+), 139 deletions(-) create mode 100644 multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java create mode 100644 multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreUtil.java create mode 100644 multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java create mode 100644 multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreUtilTest.java diff --git a/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculator.java b/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculator.java index 40272e7d90..70a85311c2 100644 --- a/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculator.java +++ b/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculator.java @@ -15,6 +15,7 @@ import java.util.concurrent.TimeoutException; import java.util.function.Consumer; import java.util.function.Supplier; + import jakarta.inject.Inject; import jakarta.inject.Named; import org.cloudfoundry.multiapps.common.SLException; @@ -29,7 +30,7 @@ import org.cloudfoundry.multiapps.controller.core.util.ApplicationInstanceNameUtil; import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseHealthService; import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseMonitoringService; -import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage; +import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -46,7 +47,7 @@ public class ApplicationHealthCalculator { // timeout private static final int TOTAL_TASK_TIMEOUT_IN_SECONDS = 3 * SINGLE_TASK_TIMEOUT_IN_SECONDS; - private final ObjectStoreFileStorage objectStoreFileStorage; + private final FileStorage objectStoreFileStorage; private final ApplicationConfiguration applicationConfiguration; private final DatabaseHealthService databaseHealthService; private final DatabaseMonitoringService databaseMonitoringService; @@ -74,7 +75,7 @@ public class ApplicationHealthCalculator { private final ResilientOperationExecutor resilientOperationExecutor = getResilienceExecutor(); @Inject - public ApplicationHealthCalculator(@Autowired(required = false) ObjectStoreFileStorage objectStoreFileStorage, + public ApplicationHealthCalculator(@Autowired(required = false) FileStorage objectStoreFileStorage, ApplicationConfiguration applicationConfiguration, DatabaseHealthService databaseHealthService, DatabaseMonitoringService databaseMonitoringService, DatabaseWaitingLocksAnalyzer databaseWaitingLocksAnalyzer) { diff --git a/multiapps-controller-persistence/pom.xml b/multiapps-controller-persistence/pom.xml index c3faa8923c..6cdee7d418 100644 --- a/multiapps-controller-persistence/pom.xml +++ b/multiapps-controller-persistence/pom.xml @@ -98,17 +98,19 @@ org.apache.jclouds.provider azureblob - - org.apache.jclouds.provider - google-cloud-storage - - - org.apache.jclouds.common - googlecloud - org.apache.jclouds jclouds-blobstore + + + org.apache.jclouds.provider + google-cloud-storage + + + + + com.google.cloud + google-cloud-storage com.aliyun.oss @@ -151,6 +153,11 @@ org.apache.logging.log4j log4j-core + + com.google.cloud + google-cloud-nio + compile + org.slf4j slf4j-api diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index a1ce012fad..c3bb201136 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -37,6 +37,10 @@ requires flowable.engine; requires flowable.engine.common.api; requires flowable.variable.service.api; + requires google.cloud.storage; + requires google.cloud.core; + requires google.cloud.nio; + requires gax; requires jakarta.inject; requires org.apache.logging.log4j; requires org.apache.logging.log4j.core; @@ -53,5 +57,4 @@ requires static org.immutables.value; requires jakarta.xml.bind; requires org.bouncycastle.fips.pkix; - } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java new file mode 100644 index 0000000000..fa5cc3d0c5 --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java @@ -0,0 +1,234 @@ +package org.cloudfoundry.multiapps.controller.persistence.services; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.channels.Channels; +import java.text.MessageFormat; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import com.google.cloud.ReadChannel; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import org.cloudfoundry.multiapps.common.util.MiscUtil; +import org.cloudfoundry.multiapps.controller.persistence.Messages; +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreUtil; +import org.jclouds.http.HttpResponseException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.MediaType; + +public class GcpObjectStoreFileStorage implements FileStorage { + + private static final Logger LOGGER = LoggerFactory.getLogger(GcpObjectStoreFileStorage.class); + private final String bucketName; + private final Storage storage; + + public GcpObjectStoreFileStorage(String bucketName, Storage storage) { + this.bucketName = bucketName; + this.storage = storage; + } + + @Override + public void addFile(FileEntry fileEntry, InputStream content) throws FileStorageException { + BlobId blobId = BlobId.of(bucketName, fileEntry.getId()); + BlobInfo blobInfo = BlobInfo.newBuilder(blobId) + .setContentDisposition(fileEntry.getName()) + .setContentType(MediaType.APPLICATION_OCTET_STREAM_VALUE) + .setMetadata(ObjectStoreUtil.createFileEntryMetadata(fileEntry)) + .build(); + + putBlobWithRetries(blobInfo, content, 3); + } + + private void putBlobWithRetries(BlobInfo blobInfo, InputStream content, int retries) throws FileStorageException { + for (int i = 1; i <= retries; i++) { + try { + storage.createFrom(blobInfo, content); + return; + } catch (HttpResponseException e) { + LOGGER.warn(MessageFormat.format(Messages.ATTEMPT_TO_UPLOAD_BLOB_FAILED, i, retries, e.getMessage()), e); + if (i == retries) { + throw e; + } + } catch (IOException e) { + throw new FileStorageException(e); + } + MiscUtil.sleep(i * getRetryWaitTime()); + } + } + + @Override + public List getFileEntriesWithoutContent(List fileEntries) throws FileStorageException { + Set existingFiles = getAllEntries().stream() + .map(Blob::getName) + .collect(Collectors.toSet()); + return fileEntries.stream() + .filter(fileEntry -> !existingFiles.contains(fileEntry.getId())) + .toList(); + } + + @Override + public void deleteFile(String id, String space) { + storage.delete(bucketName, id); + } + + @Override + public void deleteFilesBySpaceIds(List spaceIds) { + removeBlobsByFilter(blob -> ObjectStoreUtil.filterBySpaceIds(blob.getMetadata(), spaceIds)); + } + + @Override + public void deleteFilesBySpaceAndNamespace(String space, String namespace) { + removeBlobsByFilter(blob -> ObjectStoreUtil.filterBySpaceAndNamespace(blob.getMetadata(), space, namespace)); + } + + @Override + public int deleteFilesModifiedBefore(LocalDateTime modificationTime) { + return removeBlobsByFilter( + blob -> ObjectStoreUtil.filterByModificationTime(blob.getMetadata(), blob.getName(), modificationTime)); + } + + private InputStream getBlobPayloadWithOffset(FileEntry fileEntry, long startOffset, long endOffset) + throws FileStorageException { + try { + return getBlobWithRetriesWithOffset(fileEntry, 3, startOffset, endOffset); + } catch (IOException e) { + throw new FileStorageException(e); + } + } + + @Override + public T processFileContent(String space, String id, + FileContentProcessor fileContentProcessor) throws FileStorageException { + FileEntry fileEntry = ObjectStoreUtil.createFileEntry(space, id); + try (InputStream inputStream = openBlobStreamWithRetries(fileEntry, 3)) { + if (inputStream == null) { + throw new FileStorageException( + MessageFormat.format(Messages.FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST, + fileEntry.getId(), fileEntry.getSpace())); + } + return fileContentProcessor.process(inputStream); + } catch (Exception e) { + throw new FileStorageException(e); + } + } + + private InputStream openBlobStreamWithRetries(FileEntry fileEntry, int maxAttempts) { + Blob blob = getBlobWithRetries(fileEntry, maxAttempts); + if (blob == null) { + return null; + } + return Channels.newInputStream(blob.reader()); + } + + private Blob getBlobWithRetries(FileEntry fileEntry, int retries) { + for (int i = 1; i <= retries; i++) { + Blob blob = storage.get(bucketName, fileEntry.getId()); + if (blob != null) { + return blob; + } + LOGGER.warn(MessageFormat.format(Messages.ATTEMPT_TO_DOWNLOAD_MISSING_BLOB, i, retries, fileEntry.getId())); + if (i == retries) { + break; + } + MiscUtil.sleep(i * getRetryWaitTime()); + } + return null; + } + + protected long getRetryWaitTime() { + return 5000L; + } + + @Override + public InputStream openInputStream(String space, String id) throws FileStorageException { + FileEntry fileEntry = ObjectStoreUtil.createFileEntry(space, id); + return getBlobStream(fileEntry); + } + + private InputStream getBlobStream(FileEntry fileEntry) throws FileStorageException { + Blob blob = getBlobWithRetries(fileEntry, 3); + if (blob == null) { + throw new FileStorageException( + MessageFormat.format(Messages.FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST, + fileEntry.getId(), fileEntry.getSpace())); + } + return Channels.newInputStream(blob.reader()); + } + + @Override + public void testConnection() { + storage.get(bucketName, "test"); + } + + @Override + public void deleteFilesByIds(List fileIds) throws FileStorageException { + removeBlobsByFilter(blob -> fileIds.contains(blob.getName())); + } + + @Override + public T processArchiveEntryContent(FileContentToProcess fileContentToProcess, FileContentProcessor fileContentProcessor) + throws FileStorageException { + FileEntry fileEntry = ObjectStoreUtil.createFileEntry(fileContentToProcess.getSpaceGuid(), fileContentToProcess.getGuid()); + InputStream res = getBlobPayloadWithOffset(fileEntry, fileContentToProcess.getStartOffset(), + fileContentToProcess.getEndOffset()); + return processContent(fileContentProcessor, res); + } + + private T processContent(FileContentProcessor fileContentProcessor, InputStream res) + throws FileStorageException { + try { + return fileContentProcessor.process(res); + } catch (IOException e) { + throw new FileStorageException(e); + } + } + + public Set getAllEntries() { + return storage.list(bucketName) + .streamAll() + .collect(Collectors.toSet()); + } + + private int removeBlobsByFilter(Predicate filter) { + Set entries = getEntryNames(filter); + + if (!entries.isEmpty()) { + for (String name : entries) { + storage.delete(bucketName, name); + } + } + return entries.size(); + } + + private Set getEntryNames(Predicate filter) { + return storage.list(bucketName) + .streamAll() + .filter(filter) + .map(Blob::getName) + .collect(Collectors.toSet()); + } + + private InputStream getBlobWithRetriesWithOffset(FileEntry fileEntry, int retries, long startOffset, long endOffset) + throws IOException, FileStorageException { + Blob blob = getBlobWithRetries(fileEntry, retries); + if (blob == null) { + throw new FileStorageException( + MessageFormat.format(Messages.FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST, + fileEntry.getId(), fileEntry.getSpace())); + } + BlobId blobId = BlobId.of(bucketName, fileEntry.getId()); + ReadChannel reader = storage.reader(blobId); + reader.seek(startOffset); + reader.limit(endOffset + 1); + + return Channels.newInputStream(reader); + } +} diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java index 977b5968df..150ae02fd2 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java @@ -3,23 +3,18 @@ import java.io.IOException; import java.io.InputStream; import java.text.MessageFormat; -import java.time.Instant; import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import org.cloudfoundry.multiapps.common.util.MiscUtil; -import org.cloudfoundry.multiapps.controller.persistence.Constants; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; -import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreUtil; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.ContainerNotFoundException; import org.jclouds.blobstore.domain.Blob; @@ -33,7 +28,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; -import org.springframework.util.CollectionUtils; public class ObjectStoreFileStorage implements FileStorage { @@ -59,7 +53,7 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage .contentDisposition(fileEntry.getName()) .contentType(MediaType.APPLICATION_OCTET_STREAM_VALUE) .contentLength(fileSize) - .userMetadata(createFileEntryMetadata(fileEntry)) + .userMetadata(ObjectStoreUtil.createFileEntryMetadata(fileEntry)) .build(); try { putBlobWithRetries(blob, 3); @@ -87,22 +81,23 @@ public void deleteFile(String id, String space) { @Override public void deleteFilesBySpaceIds(List spaceIds) { - removeBlobsByFilter(blob -> filterBySpaceIds(blob, spaceIds)); + removeBlobsByFilter(blob -> ObjectStoreUtil.filterBySpaceIds(blob.getUserMetadata(), spaceIds)); } @Override public void deleteFilesBySpaceAndNamespace(String space, String namespace) { - removeBlobsByFilter(blob -> filterBySpaceAndNamespace(blob, space, namespace)); + removeBlobsByFilter(blob -> ObjectStoreUtil.filterBySpaceAndNamespace(blob.getUserMetadata(), space, namespace)); } @Override public int deleteFilesModifiedBefore(LocalDateTime modificationTime) { - return removeBlobsByFilter(blob -> filterByModificationTime(blob, modificationTime)); + return removeBlobsByFilter( + blob -> ObjectStoreUtil.filterByModificationTime(blob.getUserMetadata(), blob.getName(), modificationTime)); } @Override public T processFileContent(String space, String id, FileContentProcessor fileContentProcessor) throws FileStorageException { - FileEntry fileEntry = createFileEntry(space, id); + FileEntry fileEntry = ObjectStoreUtil.createFileEntry(space, id); try { Payload payload = getBlobPayload(fileEntry); return processContent(fileContentProcessor, payload); @@ -114,7 +109,7 @@ public T processFileContent(String space, String id, FileContentProcessor @Override public T processArchiveEntryContent(FileContentToProcess fileContentToProcess, FileContentProcessor fileContentProcessor) throws FileStorageException { - FileEntry fileEntry = createFileEntry(fileContentToProcess.getSpaceGuid(), fileContentToProcess.getGuid()); + FileEntry fileEntry = ObjectStoreUtil.createFileEntry(fileContentToProcess.getSpaceGuid(), fileContentToProcess.getGuid()); try { Payload payload = getBlobPayloadWithOffset(fileEntry, fileContentToProcess.getStartOffset(), fileContentToProcess.getEndOffset()); @@ -160,7 +155,7 @@ private Payload getBlobPayload(FileEntry fileEntry) throws FileStorageException @Override public InputStream openInputStream(String space, String id) throws FileStorageException { - FileEntry fileEntry = createFileEntry(space, id); + FileEntry fileEntry = ObjectStoreUtil.createFileEntry(space, id); Payload payload = getBlobPayload(fileEntry); return openPayloadInputStream(payload); } @@ -183,13 +178,6 @@ public void deleteFilesByIds(List fileIds) { removeBlobsByFilter(blob -> fileIds.contains(blob.getName())); } - private FileEntry createFileEntry(String space, String id) { - return ImmutableFileEntry.builder() - .space(space) - .id(id) - .build(); - } - private T processContent(FileContentProcessor fileContentProcessor, Payload payload) throws FileStorageException { try (InputStream fileContentStream = payload.openStream()) { return fileContentProcessor.process(fileContentStream); @@ -232,19 +220,6 @@ protected long getRetryWaitTime() { return RETRY_BASE_WAIT_TIME_IN_MILLIS; } - private Map createFileEntryMetadata(FileEntry fileEntry) { - Map metadata = new HashMap<>(); - metadata.put(Constants.FILE_ENTRY_SPACE.toLowerCase(), fileEntry.getSpace()); - metadata.put(Constants.FILE_ENTRY_MODIFIED.toLowerCase(), Long.toString(fileEntry.getModified() - .atZone(ZoneId.systemDefault()) - .toInstant() - .toEpochMilli())); - if (fileEntry.getNamespace() != null) { - metadata.put(Constants.FILE_ENTRY_NAMESPACE.toLowerCase(), fileEntry.getNamespace()); - } - return metadata; - } - private int removeBlobsByFilter(Predicate filter) { Set entries = getEntryNames(filter); if (!entries.isEmpty()) { @@ -271,45 +246,4 @@ private Set getAllEntries(ListContainerOptions options) { } return entries; } - - private boolean filterByModificationTime(StorageMetadata blobMetadata, LocalDateTime modificationTime) { - Map userMetadata = blobMetadata.getUserMetadata(); - // Clean up any blobStore entries that don't have any metadata as we can't check their creation date - if (CollectionUtils.isEmpty(userMetadata)) { - LOGGER.warn(MessageFormat.format(Messages.USER_METADATA_OF_BLOB_0_EMPTY_AND_WILL_BE_DELETED, blobMetadata.getName())); - return true; - } - String longString = userMetadata.get(Constants.FILE_ENTRY_MODIFIED.toLowerCase()); - try { - long dateLong = Long.parseLong(longString); - LocalDateTime date = LocalDateTime.ofInstant(Instant.ofEpochMilli(dateLong), ZoneId.systemDefault()); - return date.isBefore(modificationTime); - } catch (NumberFormatException e) { - // Clean up any blobStore entries that have invalid timestamp - LOGGER.warn(MessageFormat.format(Messages.DATE_METADATA_OF_BLOB_0_IS_NOT_IN_PROPER_FORMAT_AND_WILL_BE_DELETED, - blobMetadata.getName()), - e); - return true; - } - } - - private boolean filterBySpaceIds(StorageMetadata blobMetadata, List spaceIds) { - Map userMetadata = blobMetadata.getUserMetadata(); - if (CollectionUtils.isEmpty(userMetadata)) { - return false; - } - String spaceParameter = userMetadata.get(Constants.FILE_ENTRY_SPACE.toLowerCase()); - return spaceIds.contains(spaceParameter); - } - - private boolean filterBySpaceAndNamespace(StorageMetadata blobMetadata, String space, String namespace) { - Map userMetadata = blobMetadata.getUserMetadata(); - if (CollectionUtils.isEmpty(userMetadata)) { - return false; - } - String spaceParameter = userMetadata.get(Constants.FILE_ENTRY_SPACE.toLowerCase()); - String namespaceParameter = userMetadata.get(Constants.FILE_ENTRY_NAMESPACE.toLowerCase()); - return space.equals(spaceParameter) && namespace.equals(namespaceParameter); - } - } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreUtil.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreUtil.java new file mode 100644 index 0000000000..ec41f7fbe0 --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreUtil.java @@ -0,0 +1,84 @@ +package org.cloudfoundry.multiapps.controller.persistence.util; + +import java.text.MessageFormat; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.cloudfoundry.multiapps.controller.persistence.Constants; +import org.cloudfoundry.multiapps.controller.persistence.Messages; +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.CollectionUtils; + +public class ObjectStoreUtil { + + private static final Logger LOGGER = LoggerFactory.getLogger(ObjectStoreUtil.class); + + public static boolean filterBySpaceIds(Map metadata, List spaceIds) { + if (CollectionUtils.isEmpty(metadata)) { + return false; + } + String spaceParameter = metadata.get(Constants.FILE_ENTRY_SPACE.toLowerCase()); + + return spaceIds.contains(spaceParameter); + } + + public static boolean filterBySpaceAndNamespace(Map metadata, String space, String namespace) { + if (CollectionUtils.isEmpty(metadata)) { + return false; + } + + String spaceParameter = metadata.get(Constants.FILE_ENTRY_SPACE.toLowerCase()); + String namespaceParameter = metadata.get(Constants.FILE_ENTRY_NAMESPACE.toLowerCase()); + return space.equals(spaceParameter) && namespace.equals(namespaceParameter); + } + + public static boolean filterByModificationTime(Map metadata, String blobName, LocalDateTime modificationTime) { + // Clean up any blobStore entries that don't have any metadata as we can't check their creation date + if (CollectionUtils.isEmpty(metadata)) { + LOGGER.warn(MessageFormat.format(Messages.USER_METADATA_OF_BLOB_0_EMPTY_AND_WILL_BE_DELETED, blobName)); + return true; + } + String longString = metadata.get(Constants.FILE_ENTRY_MODIFIED.toLowerCase()); + try { + long dateLong = Long.parseLong(longString); + LocalDateTime date = LocalDateTime.ofInstant(Instant.ofEpochMilli(dateLong), ZoneId.systemDefault()); + return date.isBefore(modificationTime); + } catch (NumberFormatException e) { + // Clean up any blobStore entries that have invalid timestamp + LOGGER.warn(MessageFormat.format(Messages.DATE_METADATA_OF_BLOB_0_IS_NOT_IN_PROPER_FORMAT_AND_WILL_BE_DELETED, + blobName), + e); + return true; + } + } + + public static Map createFileEntryMetadata(FileEntry fileEntry) { + Map metadata = new HashMap<>(); + metadata.put(Constants.FILE_ENTRY_SPACE.toLowerCase(), fileEntry.getSpace()); + metadata.put(Constants.FILE_ENTRY_MODIFIED.toLowerCase(), + Long.toString(fileEntry.getModified() + .atZone( + ZoneId.systemDefault()) + .toInstant() + .toEpochMilli())); + if (fileEntry.getNamespace() != null) { + metadata.put(Constants.FILE_ENTRY_NAMESPACE.toLowerCase(), fileEntry.getNamespace()); + } + return metadata; + } + + public static FileEntry createFileEntry(String space, String id) { + return ImmutableFileEntry.builder() + .space(space) + .id(id) + .build(); + } + +} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java new file mode 100644 index 0000000000..ca91722097 --- /dev/null +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java @@ -0,0 +1,311 @@ +package org.cloudfoundry.multiapps.controller.persistence.services; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; +import jakarta.xml.bind.DatatypeConverter; +import org.cloudfoundry.multiapps.common.util.DigestHelper; +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.http.MediaType; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class GcpObjectStoreFileStorageTest { + + private static final String TEST_FILE_LOCATION = "src/test/resources/pexels-photo-401794.jpeg"; + private static final String SECOND_FILE_TEST_LOCATION = "src/test/resources/pexels-photo-463467.jpeg"; + private static final String DIGEST_METHOD = "MD5"; + private static final String CONTAINER = "container4e"; + + private String spaceId; + private String namespace; + + private FileStorage fileStorage; + + private Storage storage; + + @BeforeEach + public void setUp() throws Exception { + storage = LocalStorageHelper.getOptions() + .getService(); + fileStorage = new GcpObjectStoreFileStorage(CONTAINER, storage) { + @Override + protected long getRetryWaitTime() { + return 1; + } + }; + spaceId = UUID.randomUUID() + .toString(); + namespace = UUID.randomUUID() + .toString(); + } + + @AfterEach + public void tearDown() throws Exception { + if (storage != null) { + storage.close(); + } + } + + @Test + void addFileTest() throws Exception { + FileEntry fileEntry = addFile(TEST_FILE_LOCATION); + assertFileExists(true, fileEntry); + } + + @Test + void getFileEntriesWithoutContent() throws Exception { + List fileEntries = new ArrayList<>(); + FileEntry existingFile = addFile(TEST_FILE_LOCATION); + fileEntries.add(existingFile); + FileEntry existingFile2 = addFile(SECOND_FILE_TEST_LOCATION); + fileEntries.add(existingFile2); + FileEntry nonExistingFile = createFileEntry(); + fileEntries.add(nonExistingFile); + + addBigAmountOfEntries(); + + List withoutContent = fileStorage.getFileEntriesWithoutContent(fileEntries); + assertEquals(1, withoutContent.size()); + assertEquals(nonExistingFile.getId(), withoutContent.get(0) + .getId()); + } + + @Test + void deleteFile() throws Exception { + FileEntry fileThatWillBeDeleted = addFile(TEST_FILE_LOCATION); + FileEntry fileThatStays = addFile(SECOND_FILE_TEST_LOCATION); + + fileStorage.deleteFile(fileThatWillBeDeleted.getId(), fileThatWillBeDeleted.getSpace()); + assertFileExists(false, fileThatWillBeDeleted); + assertFileExists(true, fileThatStays); + } + + @Test + void deleteFilesBySpace() throws Exception { + FileEntry firstFile = addFile(TEST_FILE_LOCATION); + FileEntry secondFile = addFile(SECOND_FILE_TEST_LOCATION); + FileEntry fileInOtherSpace = addFile(TEST_FILE_LOCATION, "otherspace", namespace); + + fileStorage.deleteFilesBySpaceIds(List.of(spaceId)); + assertFileExists(false, firstFile); + assertFileExists(false, secondFile); + assertFileExists(true, fileInOtherSpace); + } + + @Test + void deleteFilesBySpaceAndNamespace() throws Exception { + FileEntry firstFile = addFile(TEST_FILE_LOCATION); + FileEntry secondFile = addFile(SECOND_FILE_TEST_LOCATION); + FileEntry fileInOtherSpace = addFile(TEST_FILE_LOCATION, "otherspace", namespace); + FileEntry fileInOtherNamespace = addFile(TEST_FILE_LOCATION, spaceId, "othernamespace"); + + fileStorage.deleteFilesBySpaceAndNamespace(spaceId, namespace); + assertFileExists(true, fileInOtherNamespace); + assertFileExists(true, fileInOtherSpace); + assertFileExists(false, firstFile); + assertFileExists(false, secondFile); + } + + @Test + void deleteFilesModifiedBefore() throws Exception { + long currentMillis = System.currentTimeMillis(); + final long oldFilesTtl = 1000 * 60 * 10; // 10min + final long pastMoment = currentMillis - 1000 * 60 * 15; // before 15min + + addBigAmountOfEntries(); + + FileEntry fileEntryToRemain1 = addFile(TEST_FILE_LOCATION); + FileEntry fileEntryToRemain2 = addFile(SECOND_FILE_TEST_LOCATION); + FileEntry fileEntryToDelete1 = addFile(TEST_FILE_LOCATION, spaceId, namespace, + LocalDateTime.ofInstant(Instant.ofEpochMilli(pastMoment), ZoneId.systemDefault())); + FileEntry fileEntryToDelete2 = addFile(SECOND_FILE_TEST_LOCATION, spaceId, null, + LocalDateTime.ofInstant(Instant.ofEpochMilli(pastMoment), ZoneId.systemDefault())); + + String blobWithNoMetadataId = addBlobWithNoMetadata(); + + int deletedFiles = fileStorage.deleteFilesModifiedBefore(LocalDateTime.ofInstant(Instant.ofEpochMilli(currentMillis - oldFilesTtl), + ZoneId.systemDefault())); + + assertEquals(3, deletedFiles); + assertFileExists(true, fileEntryToRemain1); + assertFileExists(true, fileEntryToRemain2); + assertFileExists(false, fileEntryToDelete1); + assertFileExists(false, fileEntryToDelete2); + + assertNull(storage.get(blobWithNoMetadataId)); + } + + @Test + void testConnection() { + assertDoesNotThrow(() -> fileStorage.testConnection()); + } + + @Test + void testDeleteFilesByIds() throws Exception { + FileEntry fileEntry = addFile(TEST_FILE_LOCATION); + fileStorage.deleteFilesByIds(List.of(fileEntry.getId())); + assertNull(storage.get(fileEntry.getId())); + } + + private String addBlobWithNoMetadata() throws Exception { + Path path = Paths.get(TEST_FILE_LOCATION); + long fileSize = Files.size(path); + String id = UUID.randomUUID() + .toString(); + BlobInfo blobInfo = BlobInfo.newBuilder(BlobId.of(CONTAINER, id)) + .setContentDisposition(path.getFileName() + .toString()) + .setContentType(MediaType.APPLICATION_OCTET_STREAM_VALUE) + .build(); + storage.create(blobInfo, Files.newInputStream(path)); + return id; + } + + @Test + void processFileContent() throws Exception { + FileEntry fileEntry = addFile(TEST_FILE_LOCATION); + String testFileDigest = DigestHelper.computeFileChecksum(Paths.get(TEST_FILE_LOCATION), DIGEST_METHOD) + .toLowerCase(); + validateFileContent(fileEntry, testFileDigest); + } + + @Test + void testFileContentNotExisting() throws Exception { + String fileId = "not-existing-file-id"; + String fileSpace = "not-existing-space-id"; + String fileDigest = DigestHelper.computeFileChecksum(Paths.get(TEST_FILE_LOCATION), DIGEST_METHOD) + .toLowerCase(); + FileEntry dummyFileEntry = ImmutableFileEntry.builder() + .id(fileId) + .space(fileSpace) + .build(); + assertThrows(FileStorageException.class, () -> validateFileContent(dummyFileEntry, fileDigest)); + } + + private void validateFileContent(FileEntry storedFile, final String expectedFileChecksum) throws FileStorageException { + fileStorage.processFileContent(storedFile.getSpace(), storedFile.getId(), contentStream -> { + // make a digest out of the content and compare it to the original + final byte[] digest = calculateFileDigest(contentStream); + assertEquals(expectedFileChecksum, DatatypeConverter.printHexBinary(digest) + .toLowerCase()); + return null; + }); + } + + private byte[] calculateFileDigest(InputStream contentStream) throws IOException { + try { + MessageDigest md = MessageDigest.getInstance(DIGEST_METHOD); + int read = 0; + byte[] buffer = new byte[4 * 1024]; + while ((read = contentStream.read(buffer)) > -1) { + md.update(buffer, 0, read); + } + return md.digest(); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + private void addBigAmountOfEntries() throws Exception { + for (int i = 0; i < 3001; i++) { + addFileContent("test-file-" + i, "test".getBytes()); + } + } + + private FileEntry addFile(String pathString) throws Exception { + return addFile(pathString, spaceId, namespace); + } + + private FileEntry addFile(String pathString, String space, String namespace) throws Exception { + return addFile(pathString, space, namespace, null); + } + + private FileEntry addFile(String pathString, String space, String namespace, LocalDateTime date) throws Exception { + Path testFilePath = Paths.get(pathString) + .toAbsolutePath(); + FileEntry fileEntry = createFileEntry(space, namespace); + fileEntry = enrichFileEntry(fileEntry, testFilePath, date); + try (InputStream content = Files.newInputStream(testFilePath)) { + fileStorage.addFile(fileEntry, content); + } + return fileEntry; + } + + private FileEntry addFileContent(String entryName, byte[] content) throws Exception { + FileEntry fileEntry = createFileEntry(entryName, content); + try (InputStream contentStream = new ByteArrayInputStream(content)) { + fileStorage.addFile(fileEntry, contentStream); + } + return fileEntry; + } + + private FileEntry createFileEntry() { + return createFileEntry(spaceId, namespace); + } + + private FileEntry createFileEntry(String entryName, byte[] content) { + return ImmutableFileEntry.builder() + .id(UUID.randomUUID() + .toString()) + .space(spaceId) + .size(BigInteger.valueOf(content.length)) + .modified(LocalDateTime.now()) + .name(entryName) + .build(); + } + + private FileEntry createFileEntry(String space, String namespace) { + return ImmutableFileEntry.builder() + .id(UUID.randomUUID() + .toString()) + .space(space) + .namespace(namespace) + .build(); + } + + private FileEntry enrichFileEntry(FileEntry fileEntry, Path path, LocalDateTime date) throws IOException { + long sizeOfFile = Files.size(path); + BigInteger bigInteger = BigInteger.valueOf(sizeOfFile); + return ImmutableFileEntry.builder() + .from(fileEntry) + .size(bigInteger) + .modified(date != null ? date : LocalDateTime.now()) + .name(path.getFileName() + .toString()) + .build(); + } + + private void assertFileExists(boolean exceptedFileExist, FileEntry actualFile) { + Blob blob = storage.get(CONTAINER, actualFile.getId()); + boolean blobExists = blob != null; + + assertEquals(exceptedFileExist, blobExists); + } + +} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreUtilTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreUtilTest.java new file mode 100644 index 0000000000..b94ca600ca --- /dev/null +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreUtilTest.java @@ -0,0 +1,117 @@ +package org.cloudfoundry.multiapps.controller.persistence.util; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.cloudfoundry.multiapps.controller.persistence.Constants; +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ObjectStoreUtilTest { + + private static final String SPACE_ID = "spaceId"; + private static final String SPACE_ID_2 = "spaceId2"; + private static final String NAMESPACE = "namespace"; + private static final String BLOBNAME = "blobName"; + private static final String NAMESPACE_2 = "namespace2"; + + @Test + void testFilterBySpaceIdsWhenMetadataIsEmpty() { + assertFalse(ObjectStoreUtil.filterBySpaceIds(Map.of(), List.of())); + } + + @Test + void testFilterBySpaceIdsWhenTheMetadataContainsTheSpaceId() { + assertTrue(ObjectStoreUtil.filterBySpaceIds(buildMetadata(), List.of(SPACE_ID))); + } + + @Test + void testFilterBySpaceIdsWhenTheMetadataDoesNotContainTheSpaceId() { + assertFalse(ObjectStoreUtil.filterBySpaceIds(buildMetadata(), List.of(SPACE_ID_2))); + } + + @Test + void testFilterBySpaceAndNamespaceWhenMetadataIsEmpty() { + assertFalse(ObjectStoreUtil.filterBySpaceAndNamespace(Map.of(), SPACE_ID, NAMESPACE)); + } + + @Test + void testFilterBySpaceAndNamespaceWhenTheSpacesAreDifferent() { + assertFalse(ObjectStoreUtil.filterBySpaceAndNamespace(buildMetadata(), SPACE_ID_2, NAMESPACE)); + } + + @Test + void testFilterBySpaceAndNamespaceWhenTheNamespacesAreDifferent() { + assertFalse(ObjectStoreUtil.filterBySpaceAndNamespace(buildMetadata(), SPACE_ID, NAMESPACE_2)); + } + + @Test + void testFilterBySpaceAndNamespaceWhenAllMatch() { + assertTrue(ObjectStoreUtil.filterBySpaceAndNamespace(buildMetadata(), SPACE_ID, NAMESPACE)); + } + + @Test + void testFilterByModificationTimeWhenMetadataIsEmpty() { + assertTrue(ObjectStoreUtil.filterByModificationTime(Map.of(), BLOBNAME, LocalDateTime.now())); + } + + @Test + void testFilterByModificationTimeWhenModificationTimeIsBeforeTheMetadataTime() { + assertFalse(ObjectStoreUtil.filterByModificationTime(buildMetadata(), BLOBNAME, LocalDateTime.now() + .minusMinutes(20))); + } + + @Test + void testFilterByModificationTimeWhenModificationTimeIsAfterTheMetadataTime() { + assertTrue(ObjectStoreUtil.filterByModificationTime(buildMetadata(), BLOBNAME, LocalDateTime.now() + .plusMinutes(20))); + } + + @Test + void testCreateFileEntryMetadata() { + LocalDateTime modifiedTime = LocalDateTime.now(); + String modifiedTimeInstantString = Long.toString(modifiedTime + .atZone( + ZoneId.systemDefault()) + .toInstant() + .toEpochMilli()); + Map metadata = ObjectStoreUtil.createFileEntryMetadata(buildCreateFileEntry(modifiedTime)); + + assertEquals(SPACE_ID, metadata.get(Constants.FILE_ENTRY_SPACE.toLowerCase())); + assertEquals(modifiedTimeInstantString, metadata.get(Constants.FILE_ENTRY_MODIFIED.toLowerCase())); + assertEquals(NAMESPACE, metadata.get(Constants.FILE_ENTRY_NAMESPACE.toLowerCase())); + } + + @Test + void testCreateFileEntry() { + String id = UUID.randomUUID() + .toString(); + FileEntry fileEntry = ObjectStoreUtil.createFileEntry(SPACE_ID, id); + + assertEquals(SPACE_ID, fileEntry.getSpace()); + assertEquals(id, fileEntry.getId()); + } + + private FileEntry buildCreateFileEntry(LocalDateTime modifiedTime) { + return ImmutableFileEntry.builder() + .space(SPACE_ID) + .modified(modifiedTime) + .namespace(NAMESPACE) + .build(); + } + + private Map buildMetadata() { + return Map.of(Constants.FILE_ENTRY_SPACE.toLowerCase(), SPACE_ID, Constants.FILE_ENTRY_NAMESPACE.toLowerCase(), NAMESPACE, + Constants.FILE_ENTRY_MODIFIED.toLowerCase(), String.valueOf(Instant.now() + .toEpochMilli())); + } +} diff --git a/multiapps-controller-web/pom.xml b/multiapps-controller-web/pom.xml index dd056e63f0..3049aea29f 100644 --- a/multiapps-controller-web/pom.xml +++ b/multiapps-controller-web/pom.xml @@ -199,5 +199,9 @@ javax.xml.bind jaxb-api + + com.google.cloud + google-cloud-storage + diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/FileServiceFactoryBean.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/FileServiceFactoryBean.java index 68a076ba3d..40d7728836 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/FileServiceFactoryBean.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/FileServiceFactoryBean.java @@ -2,12 +2,10 @@ import jakarta.inject.Inject; import jakarta.inject.Named; - import org.cloudfoundry.multiapps.controller.persistence.DataSourceWithDialect; import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseFileService; import org.cloudfoundry.multiapps.controller.persistence.services.FileService; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; -import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.web.Messages; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,15 +21,15 @@ public class FileServiceFactoryBean implements FactoryBean, Initial @Inject private DataSourceWithDialect dataSourceWithDialect; @Autowired(required = false) - private ObjectStoreFileStorage objectStoreFileStorage; + private FileStorage objectStoreFileStorage; + private FileService fileService; @Override public void afterPropertiesSet() { - FileStorage fileStorage = objectStoreFileStorage; - if (fileStorage != null) { + if (objectStoreFileStorage != null) { LOGGER.info(Messages.OBJECTSTORE_FOR_BINARIES_STORAGE); - this.fileService = new FileService(dataSourceWithDialect, fileStorage); + this.fileService = new FileService(dataSourceWithDialect, objectStoreFileStorage); } else { LOGGER.info(Messages.DATABASE_FOR_BINARIES_STORAGE); this.fileService = new DatabaseFileService(dataSourceWithDialect); diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java index 9d84a5507b..f227fd8cf2 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java @@ -16,6 +16,8 @@ import org.apache.commons.lang3.StringUtils; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.util.UriUtil; +import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; +import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder; import org.cloudfoundry.multiapps.controller.web.Constants; @@ -30,7 +32,7 @@ import org.springframework.beans.factory.FactoryBean; import org.springframework.beans.factory.InitializingBean; -public class ObjectStoreFileStorageFactoryBean implements FactoryBean, InitializingBean { +public class ObjectStoreFileStorageFactoryBean implements FactoryBean, InitializingBean { private static final Logger LOGGER = LoggerFactory.getLogger(ObjectStoreFileStorageFactoryBean.class); private static final Set CUSTOM_REGIONS = Set.of("eu-south-1"); @@ -39,7 +41,7 @@ public class ObjectStoreFileStorageFactoryBean implements FactoryBean providersServiceInfo = getProvidersServiceInfo(); if (providersServiceInfo.isEmpty()) { return null; @@ -69,7 +71,7 @@ private ObjectStoreFileStorage createObjectStoreFileStorage() { if (objectStoreServiceInfoOptional.isPresent()) { ObjectStoreServiceInfo objectStoreServiceInfo = objectStoreServiceInfoOptional.get(); - Optional createdObjectStore = tryToCreateObjectStore(objectStoreServiceInfo, exceptions); + Optional createdObjectStore = tryToCreateObjectStore(objectStoreServiceInfo, exceptions); if (createdObjectStore.isPresent()) { return createdObjectStore.get(); } @@ -78,14 +80,15 @@ private ObjectStoreFileStorage createObjectStoreFileStorage() { throw buildNoValidObjectStoreException(exceptions); } - public ObjectStoreFileStorage createObjectStoreFromFirstReachableProvider(Map exceptions, - List providersServiceInfo) { + public FileStorage createObjectStoreFromFirstReachableProvider(Map exceptions, + List providersServiceInfo) { for (ObjectStoreServiceInfo objectStoreServiceInfo : providersServiceInfo) { - Optional createdObjectStoreOptional = tryToCreateObjectStore(objectStoreServiceInfo, exceptions); + Optional createdObjectStoreOptional = tryToCreateObjectStore(objectStoreServiceInfo, exceptions); if (createdObjectStoreOptional.isPresent()) { return createdObjectStoreOptional.get(); } } + throw buildNoValidObjectStoreException(exceptions); } @@ -109,11 +112,10 @@ private IllegalStateException buildNoValidObjectStoreException(Map tryToCreateObjectStore(ObjectStoreServiceInfo objectStoreServiceInfo, - Map exceptions) { + private Optional tryToCreateObjectStore(ObjectStoreServiceInfo objectStoreServiceInfo, + Map exceptions) { try { - BlobStoreContext context = getBlobStoreContext(objectStoreServiceInfo); - ObjectStoreFileStorage fileStorage = createFileStorage(objectStoreServiceInfo, context); + FileStorage fileStorage = getFileStorageBasedOnProvider(objectStoreServiceInfo); fileStorage.testConnection(); LOGGER.info(MessageFormat.format(Messages.OBJECT_STORE_WITH_PROVIDER_0_CREATED, objectStoreServiceInfo.getProvider())); return Optional.of(fileStorage); @@ -123,7 +125,16 @@ private Optional tryToCreateObjectStore(ObjectStoreServi } } - private List getProvidersServiceInfo() { + private FileStorage getFileStorageBasedOnProvider(ObjectStoreServiceInfo objectStoreServiceInfo) { + if (Constants.GOOGLE_CLOUD_STORAGE.equals(objectStoreServiceInfo.getProvider())) { + return createFileStorage(objectStoreServiceInfo); + } else { + BlobStoreContext context = getBlobStoreContext(objectStoreServiceInfo); + return createFileStorage(objectStoreServiceInfo, context); + } + } + + public List getProvidersServiceInfo() { CfService service = environmentServicesFinder.findService(serviceName); if (service == null) { return Collections.emptyList(); @@ -185,8 +196,12 @@ protected ObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo object return new ObjectStoreFileStorage(context.getBlobStore(), objectStoreServiceInfo.getContainer()); } + protected GcpObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { + return new GcpObjectStoreFileStorage(objectStoreServiceInfo.getContainer(), objectStoreServiceInfo.getGcpStorage()); + } + @Override - public ObjectStoreFileStorage getObject() { + public FileStorage getObject() { return objectStoreFileStorage; } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java index bad4dbd804..5bd108715e 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java @@ -1,5 +1,6 @@ package org.cloudfoundry.multiapps.controller.web.configuration.service; +import com.google.cloud.storage.Storage; import com.google.common.base.Supplier; import org.cloudfoundry.multiapps.common.Nullable; import org.immutables.value.Value; @@ -31,4 +32,7 @@ public interface ObjectStoreServiceInfo { @Nullable String getHost(); + @Nullable + Storage getGcpStorage(); + } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java index 5b3f9cc4b1..37da7ccbc8 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java @@ -1,18 +1,20 @@ package org.cloudfoundry.multiapps.controller.web.configuration.service; +import java.io.ByteArrayInputStream; +import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; -import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.List; import java.util.Map; -import com.google.common.base.Supplier; +import com.google.auth.Credentials; +import com.google.auth.oauth2.GoogleCredentials; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; import io.pivotal.cfenv.core.CfService; import org.cloudfoundry.multiapps.controller.web.Constants; import org.cloudfoundry.multiapps.controller.web.Messages; -import org.jclouds.domain.Credentials; -import org.jclouds.googlecloud.GoogleCredentialsFromJson; public class ObjectStoreServiceInfoCreator { @@ -97,25 +99,33 @@ private URL getContainerUriEndpoint(Map credentials) { } private ObjectStoreServiceInfo createServiceInfoForGcpCloud(Map credentials) { - String bucket = (String) credentials.get(Constants.BUCKET); - String region = (String) credentials.get(Constants.REGION); - Supplier credentialsSupplier = getGcpCredentialsSupplier(credentials); + String bucketName = (String) credentials.get(Constants.BUCKET); + Storage storage = createObjectStoreStorage(credentials); return ImmutableObjectStoreServiceInfo.builder() .provider(Constants.GOOGLE_CLOUD_STORAGE) - .credentialsSupplier(credentialsSupplier) - .container(bucket) - .region(region) + .container(bucketName) + .gcpStorage(storage) .build(); } - protected Supplier getGcpCredentialsSupplier(Map credentials) { + public Storage createObjectStoreStorage(Map credentials) { + return StorageOptions.newBuilder() + .setCredentials(getGcpCredentialsSupplier(credentials)) + .build() + .getService(); + } + + private Credentials getGcpCredentialsSupplier(Map credentials) { if (!credentials.containsKey(Constants.BASE_64_ENCODED_PRIVATE_KEY_DATA)) { - return () -> null; + return null; } byte[] decodedKey = Base64.getDecoder() .decode((String) credentials.get(Constants.BASE_64_ENCODED_PRIVATE_KEY_DATA)); - String decodedCredential = new String(decodedKey, StandardCharsets.UTF_8); - return new GoogleCredentialsFromJson(decodedCredential); + try { + return GoogleCredentials.fromStream(new ByteArrayInputStream(decodedKey)); + } catch (IOException e) { + throw new IllegalStateException(e); + } } } diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java index 8cdb10fa93..950eb5917f 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java @@ -1,17 +1,23 @@ package org.cloudfoundry.multiapps.controller.web.configuration.bean.factory; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; import io.pivotal.cfenv.core.CfCredentials; import io.pivotal.cfenv.core.CfService; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; +import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; +import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder; import org.cloudfoundry.multiapps.controller.web.Constants; import org.cloudfoundry.multiapps.controller.web.Messages; import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfo; +import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfoCreator; import org.jclouds.blobstore.BlobStoreContext; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -47,6 +53,9 @@ class ObjectStoreFileStorageFactoryBeanTest { @Mock private ObjectStoreFileStorage objectStoreFileStorage; + @Mock + private GcpObjectStoreFileStorage gcpObjectStoreFileStorage; + @BeforeEach void setUp() throws Exception { MockitoAnnotations.openMocks(this) @@ -59,7 +68,7 @@ void setUp() throws Exception { @Test void testObjectStoreCreationWithoutServiceInstance() { objectStoreFileStorageFactoryBean.afterPropertiesSet(); - ObjectStoreFileStorage objectStoreFileStorage = objectStoreFileStorageFactoryBean.getObject(); + FileStorage objectStoreFileStorage = objectStoreFileStorageFactoryBean.getObject(); assertNull(objectStoreFileStorage); } @@ -67,7 +76,7 @@ void testObjectStoreCreationWithoutServiceInstance() { void testObjectStoreCreationWithValidServiceInstance() { mockCfService(); objectStoreFileStorageFactoryBean.afterPropertiesSet(); - ObjectStoreFileStorage objectStoreFileStorage = objectStoreFileStorageFactoryBean.getObject(); + FileStorage objectStoreFileStorage = objectStoreFileStorageFactoryBean.getObject(); assertNotNull(objectStoreFileStorage); } @@ -78,7 +87,7 @@ void testObjectStoreCreationWhenEnvIsValid() { ObjectStoreFileStorageFactoryBean spy = spy(objectStoreFileStorageFactoryBean); spy.afterPropertiesSet(); - ObjectStoreFileStorage createdObjectStoreFileStorage = spy.getObject(); + FileStorage createdObjectStoreFileStorage = spy.getObject(); assertNotNull(createdObjectStoreFileStorage); verify(spy, never()) @@ -95,7 +104,7 @@ void testObjectStoreCreationWhenEnvIsInvalid() { ObjectStoreFileStorageFactoryBean spy = spy(objectStoreFileStorageFactoryBean); spy.afterPropertiesSet(); - ObjectStoreFileStorage createdObjectStoreFileStorage = spy.getObject(); + FileStorage createdObjectStoreFileStorage = spy.getObject(); assertNotNull(createdObjectStoreFileStorage); verify(spy, times(1)) @@ -118,6 +127,8 @@ void testObjectStoreCreationWithoutValidServiceInstance() { mockCfService(); doThrow(new IllegalStateException("Cannot create object store")).when(objectStoreFileStorage) .testConnection(); + doThrow(new IllegalStateException("Cannot create object store")).when(gcpObjectStoreFileStorage) + .testConnection(); Exception exception = assertThrows(IllegalStateException.class, () -> objectStoreFileStorageFactoryBean.afterPropertiesSet()); assertEquals(Messages.NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND, exception.getMessage()); } @@ -149,5 +160,29 @@ public ObjectStoreFileStorageFactoryBeanMock(String serviceName, EnvironmentServ protected ObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) { return ObjectStoreFileStorageFactoryBeanTest.this.objectStoreFileStorage; } + + @Override + protected GcpObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { + return ObjectStoreFileStorageFactoryBeanTest.this.gcpObjectStoreFileStorage; + } + + @Override + public List getProvidersServiceInfo() { + CfService service = environmentServicesFinder.findService("deploy-service-os"); + if (service != null) { + return new ObjectStoreServiceInfoCreatorMock().getAllProvidersServiceInfo(service); + } else { + return List.of(); + } + } + } + + private class ObjectStoreServiceInfoCreatorMock extends ObjectStoreServiceInfoCreator { + + @Override + public Storage createObjectStoreStorage(Map credentials) { + return LocalStorageHelper.getOptions() + .getService(); + } } } diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java index 38e4216f7d..87fecb25f5 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java @@ -1,8 +1,5 @@ package org.cloudfoundry.multiapps.controller.web.configuration.service; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.when; - import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.StandardCharsets; @@ -12,18 +9,19 @@ import java.util.Map; import java.util.stream.Stream; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; +import io.pivotal.cfenv.core.CfCredentials; +import io.pivotal.cfenv.core.CfService; import org.cloudfoundry.multiapps.controller.web.Constants; -import org.jclouds.domain.Credentials; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mockito; -import com.google.common.base.Supplier; - -import io.pivotal.cfenv.core.CfCredentials; -import io.pivotal.cfenv.core.CfService; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; class ObjectStoreServiceInfoCreatorTest { @@ -36,7 +34,8 @@ class ObjectStoreServiceInfoCreatorTest { private static final String SAS_TOKEN_VALUE = "sas_token_value"; private static final String CONTAINER_NAME_VALUE = "container_name_value"; private static final String CONTAINER_URI_VALUE = "https://container.com:8080"; - private static final Supplier CREDENTIALS_SUPPLIER = () -> null; + private static final Storage STORAGE = LocalStorageHelper.getOptions() + .getService(); private ObjectStoreServiceInfoCreator objectStoreServiceInfoCreator; @@ -127,7 +126,6 @@ private static ObjectStoreServiceInfo buildAzureObjectStoreServiceInfo() throws private static Map buildGcpCredentials() { Map credentials = new HashMap<>(); credentials.put(Constants.BUCKET, BUCKET_VALUE); - credentials.put(Constants.REGION, REGION_VALUE); credentials.put(Constants.BASE_64_ENCODED_PRIVATE_KEY_DATA, Base64.getEncoder() .encodeToString("encoded_data".getBytes(StandardCharsets.UTF_8))); return credentials; @@ -136,17 +134,15 @@ private static Map buildGcpCredentials() { private static ObjectStoreServiceInfo buildGcpObjectStoreServiceInfo() { return ImmutableObjectStoreServiceInfo.builder() .provider(Constants.GOOGLE_CLOUD_STORAGE) - .credentialsSupplier(CREDENTIALS_SUPPLIER) + .gcpStorage(STORAGE) .container(BUCKET_VALUE) - .region(REGION_VALUE) .build(); } private static class ObjectStoreServiceInfoCreatorMock extends ObjectStoreServiceInfoCreator { - @Override - protected Supplier getGcpCredentialsSupplier(Map credentials) { - return ObjectStoreServiceInfoCreatorTest.CREDENTIALS_SUPPLIER; + public Storage createObjectStoreStorage(Map credentials) { + return STORAGE; } } } diff --git a/pom.xml b/pom.xml index de1d1d3592..8d05555f72 100644 --- a/pom.xml +++ b/pom.xml @@ -57,6 +57,7 @@ 1.28.0 3.2.2 1.3.1 + 2.7.4 multiapps-controller-client @@ -288,6 +289,17 @@ commons-compress ${apache.compress.version} + + com.google.cloud + google-cloud-storage + 2.56.0 + + + com.google.cloud + google-cloud-nio + 0.128.5 + test + org.immutables From 1345c251824b1b2c324487ec91496e5d77a9c442 Mon Sep 17 00:00:00 2001 From: Yavor16 Date: Fri, 17 Oct 2025 11:30:16 +0300 Subject: [PATCH 02/12] Fix comments from Ivan --- .../src/main/java/module-info.java | 4 +- .../services/GcpObjectStoreFileStorage.java | 54 ++-- .../GcpObjectStoreFileStorageTest.java | 276 ++---------------- .../services/ObjectStoreFileStorageTest.java | 50 ++-- multiapps-controller-web/pom.xml | 4 + .../ObjectStoreFileStorageFactoryBean.java | 4 +- ...ObjectStoreFileStorageFactoryBeanTest.java | 2 +- pom.xml | 7 +- 8 files changed, 93 insertions(+), 308 deletions(-) diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index c3bb201136..013753e4d9 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -37,10 +37,10 @@ requires flowable.engine; requires flowable.engine.common.api; requires flowable.variable.service.api; - requires google.cloud.storage; + requires gax; requires google.cloud.core; requires google.cloud.nio; - requires gax; + requires google.cloud.storage; requires jakarta.inject; requires org.apache.logging.log4j; requires org.apache.logging.log4j.core; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java index fa5cc3d0c5..a91bde5c3c 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java @@ -15,11 +15,11 @@ import com.google.cloud.storage.BlobId; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageException; import org.cloudfoundry.multiapps.common.util.MiscUtil; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreUtil; -import org.jclouds.http.HttpResponseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; @@ -29,6 +29,7 @@ public class GcpObjectStoreFileStorage implements FileStorage { private static final Logger LOGGER = LoggerFactory.getLogger(GcpObjectStoreFileStorage.class); private final String bucketName; private final Storage storage; + private static final long RETRY_WAIT_TIME = 5000L; public GcpObjectStoreFileStorage(String bucketName, Storage storage) { this.bucketName = bucketName; @@ -52,13 +53,11 @@ private void putBlobWithRetries(BlobInfo blobInfo, InputStream content, int retr try { storage.createFrom(blobInfo, content); return; - } catch (HttpResponseException e) { + } catch (IOException e) { LOGGER.warn(MessageFormat.format(Messages.ATTEMPT_TO_UPLOAD_BLOB_FAILED, i, retries, e.getMessage()), e); if (i == retries) { - throw e; + throw new FileStorageException(e); } - } catch (IOException e) { - throw new FileStorageException(e); } MiscUtil.sleep(i * getRetryWaitTime()); } @@ -109,42 +108,39 @@ public T processFileContent(String space, String id, FileContentProcessor fileContentProcessor) throws FileStorageException { FileEntry fileEntry = ObjectStoreUtil.createFileEntry(space, id); try (InputStream inputStream = openBlobStreamWithRetries(fileEntry, 3)) { - if (inputStream == null) { - throw new FileStorageException( - MessageFormat.format(Messages.FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST, - fileEntry.getId(), fileEntry.getSpace())); - } return fileContentProcessor.process(inputStream); } catch (Exception e) { throw new FileStorageException(e); } } - private InputStream openBlobStreamWithRetries(FileEntry fileEntry, int maxAttempts) { + private InputStream openBlobStreamWithRetries(FileEntry fileEntry, int maxAttempts) throws FileStorageException { Blob blob = getBlobWithRetries(fileEntry, maxAttempts); if (blob == null) { - return null; + throw new FileStorageException( + MessageFormat.format(Messages.FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST, + fileEntry.getId(), fileEntry.getSpace())); } return Channels.newInputStream(blob.reader()); } private Blob getBlobWithRetries(FileEntry fileEntry, int retries) { for (int i = 1; i <= retries; i++) { - Blob blob = storage.get(bucketName, fileEntry.getId()); - if (blob != null) { - return blob; - } - LOGGER.warn(MessageFormat.format(Messages.ATTEMPT_TO_DOWNLOAD_MISSING_BLOB, i, retries, fileEntry.getId())); - if (i == retries) { - break; + try { + return storage.get(bucketName, fileEntry.getId()); + } catch (StorageException e) { + LOGGER.warn(MessageFormat.format(Messages.ATTEMPT_TO_DOWNLOAD_MISSING_BLOB, i, retries, fileEntry.getId())); + if (i == retries) { + break; + } + MiscUtil.sleep(i * getRetryWaitTime()); } - MiscUtil.sleep(i * getRetryWaitTime()); } return null; } protected long getRetryWaitTime() { - return 5000L; + return RETRY_WAIT_TIME; } @Override @@ -197,18 +193,18 @@ public Set getAllEntries() { .collect(Collectors.toSet()); } - private int removeBlobsByFilter(Predicate filter) { - Set entries = getEntryNames(filter); + protected int removeBlobsByFilter(Predicate filter) { + List blobIds = getEntryNames(filter).stream() + .map(entry -> BlobId.of(bucketName, entry)) + .toList(); - if (!entries.isEmpty()) { - for (String name : entries) { - storage.delete(bucketName, name); - } + if (!blobIds.isEmpty()) { + storage.delete(blobIds); } - return entries.size(); + return blobIds.size(); } - private Set getEntryNames(Predicate filter) { + protected Set getEntryNames(Predicate filter) { return storage.list(bucketName) .streamAll() .filter(filter) diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java index ca91722097..8170cd351b 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java @@ -1,62 +1,48 @@ package org.cloudfoundry.multiapps.controller.persistence.services; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.math.BigInteger; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.ArrayList; -import java.util.List; +import java.util.Set; import java.util.UUID; +import java.util.function.Predicate; import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobId; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.Storage; import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; -import jakarta.xml.bind.DatatypeConverter; -import org.cloudfoundry.multiapps.common.util.DigestHelper; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; -import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; import org.springframework.http.MediaType; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -class GcpObjectStoreFileStorageTest { - - private static final String TEST_FILE_LOCATION = "src/test/resources/pexels-photo-401794.jpeg"; - private static final String SECOND_FILE_TEST_LOCATION = "src/test/resources/pexels-photo-463467.jpeg"; - private static final String DIGEST_METHOD = "MD5"; - private static final String CONTAINER = "container4e"; - - private String spaceId; - private String namespace; - - private FileStorage fileStorage; +class GcpObjectStoreFileStorageTest extends ObjectStoreFileStorageTest { private Storage storage; + @Override @BeforeEach - public void setUp() throws Exception { + public void setUp() { storage = LocalStorageHelper.getOptions() .getService(); fileStorage = new GcpObjectStoreFileStorage(CONTAINER, storage) { @Override protected long getRetryWaitTime() { - return 1; + return 0; + } + + @Override + protected int removeBlobsByFilter(Predicate filter) { + Set entries = getEntryNames(filter); + for (String entry : entries) { + storage.delete(CONTAINER, entry); + } + + return entries.size(); } }; spaceId = UUID.randomUUID() @@ -65,117 +51,26 @@ protected long getRetryWaitTime() { .toString(); } + @Override @AfterEach - public void tearDown() throws Exception { + public void tearDown() { if (storage != null) { - storage.close(); + try { + storage.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } } } - @Test - void addFileTest() throws Exception { - FileEntry fileEntry = addFile(TEST_FILE_LOCATION); - assertFileExists(true, fileEntry); - } - - @Test - void getFileEntriesWithoutContent() throws Exception { - List fileEntries = new ArrayList<>(); - FileEntry existingFile = addFile(TEST_FILE_LOCATION); - fileEntries.add(existingFile); - FileEntry existingFile2 = addFile(SECOND_FILE_TEST_LOCATION); - fileEntries.add(existingFile2); - FileEntry nonExistingFile = createFileEntry(); - fileEntries.add(nonExistingFile); - - addBigAmountOfEntries(); - - List withoutContent = fileStorage.getFileEntriesWithoutContent(fileEntries); - assertEquals(1, withoutContent.size()); - assertEquals(nonExistingFile.getId(), withoutContent.get(0) - .getId()); - } - - @Test - void deleteFile() throws Exception { - FileEntry fileThatWillBeDeleted = addFile(TEST_FILE_LOCATION); - FileEntry fileThatStays = addFile(SECOND_FILE_TEST_LOCATION); - - fileStorage.deleteFile(fileThatWillBeDeleted.getId(), fileThatWillBeDeleted.getSpace()); - assertFileExists(false, fileThatWillBeDeleted); - assertFileExists(true, fileThatStays); - } - - @Test - void deleteFilesBySpace() throws Exception { - FileEntry firstFile = addFile(TEST_FILE_LOCATION); - FileEntry secondFile = addFile(SECOND_FILE_TEST_LOCATION); - FileEntry fileInOtherSpace = addFile(TEST_FILE_LOCATION, "otherspace", namespace); - - fileStorage.deleteFilesBySpaceIds(List.of(spaceId)); - assertFileExists(false, firstFile); - assertFileExists(false, secondFile); - assertFileExists(true, fileInOtherSpace); - } - - @Test - void deleteFilesBySpaceAndNamespace() throws Exception { - FileEntry firstFile = addFile(TEST_FILE_LOCATION); - FileEntry secondFile = addFile(SECOND_FILE_TEST_LOCATION); - FileEntry fileInOtherSpace = addFile(TEST_FILE_LOCATION, "otherspace", namespace); - FileEntry fileInOtherNamespace = addFile(TEST_FILE_LOCATION, spaceId, "othernamespace"); - - fileStorage.deleteFilesBySpaceAndNamespace(spaceId, namespace); - assertFileExists(true, fileInOtherNamespace); - assertFileExists(true, fileInOtherSpace); - assertFileExists(false, firstFile); - assertFileExists(false, secondFile); - } - - @Test - void deleteFilesModifiedBefore() throws Exception { - long currentMillis = System.currentTimeMillis(); - final long oldFilesTtl = 1000 * 60 * 10; // 10min - final long pastMoment = currentMillis - 1000 * 60 * 15; // before 15min - - addBigAmountOfEntries(); - - FileEntry fileEntryToRemain1 = addFile(TEST_FILE_LOCATION); - FileEntry fileEntryToRemain2 = addFile(SECOND_FILE_TEST_LOCATION); - FileEntry fileEntryToDelete1 = addFile(TEST_FILE_LOCATION, spaceId, namespace, - LocalDateTime.ofInstant(Instant.ofEpochMilli(pastMoment), ZoneId.systemDefault())); - FileEntry fileEntryToDelete2 = addFile(SECOND_FILE_TEST_LOCATION, spaceId, null, - LocalDateTime.ofInstant(Instant.ofEpochMilli(pastMoment), ZoneId.systemDefault())); - - String blobWithNoMetadataId = addBlobWithNoMetadata(); - - int deletedFiles = fileStorage.deleteFilesModifiedBefore(LocalDateTime.ofInstant(Instant.ofEpochMilli(currentMillis - oldFilesTtl), - ZoneId.systemDefault())); - - assertEquals(3, deletedFiles); - assertFileExists(true, fileEntryToRemain1); - assertFileExists(true, fileEntryToRemain2); - assertFileExists(false, fileEntryToDelete1); - assertFileExists(false, fileEntryToDelete2); - + @Override + protected void assertBlobInNull(String blobWithNoMetadataId) { assertNull(storage.get(blobWithNoMetadataId)); } - @Test - void testConnection() { - assertDoesNotThrow(() -> fileStorage.testConnection()); - } - - @Test - void testDeleteFilesByIds() throws Exception { - FileEntry fileEntry = addFile(TEST_FILE_LOCATION); - fileStorage.deleteFilesByIds(List.of(fileEntry.getId())); - assertNull(storage.get(fileEntry.getId())); - } - - private String addBlobWithNoMetadata() throws Exception { + @Override + public String addBlobWithNoMetadata() throws Exception { Path path = Paths.get(TEST_FILE_LOCATION); - long fileSize = Files.size(path); String id = UUID.randomUUID() .toString(); BlobInfo blobInfo = BlobInfo.newBuilder(BlobId.of(CONTAINER, id)) @@ -187,121 +82,8 @@ private String addBlobWithNoMetadata() throws Exception { return id; } - @Test - void processFileContent() throws Exception { - FileEntry fileEntry = addFile(TEST_FILE_LOCATION); - String testFileDigest = DigestHelper.computeFileChecksum(Paths.get(TEST_FILE_LOCATION), DIGEST_METHOD) - .toLowerCase(); - validateFileContent(fileEntry, testFileDigest); - } - - @Test - void testFileContentNotExisting() throws Exception { - String fileId = "not-existing-file-id"; - String fileSpace = "not-existing-space-id"; - String fileDigest = DigestHelper.computeFileChecksum(Paths.get(TEST_FILE_LOCATION), DIGEST_METHOD) - .toLowerCase(); - FileEntry dummyFileEntry = ImmutableFileEntry.builder() - .id(fileId) - .space(fileSpace) - .build(); - assertThrows(FileStorageException.class, () -> validateFileContent(dummyFileEntry, fileDigest)); - } - - private void validateFileContent(FileEntry storedFile, final String expectedFileChecksum) throws FileStorageException { - fileStorage.processFileContent(storedFile.getSpace(), storedFile.getId(), contentStream -> { - // make a digest out of the content and compare it to the original - final byte[] digest = calculateFileDigest(contentStream); - assertEquals(expectedFileChecksum, DatatypeConverter.printHexBinary(digest) - .toLowerCase()); - return null; - }); - } - - private byte[] calculateFileDigest(InputStream contentStream) throws IOException { - try { - MessageDigest md = MessageDigest.getInstance(DIGEST_METHOD); - int read = 0; - byte[] buffer = new byte[4 * 1024]; - while ((read = contentStream.read(buffer)) > -1) { - md.update(buffer, 0, read); - } - return md.digest(); - } catch (NoSuchAlgorithmException e) { - throw new IllegalStateException(e); - } - } - - private void addBigAmountOfEntries() throws Exception { - for (int i = 0; i < 3001; i++) { - addFileContent("test-file-" + i, "test".getBytes()); - } - } - - private FileEntry addFile(String pathString) throws Exception { - return addFile(pathString, spaceId, namespace); - } - - private FileEntry addFile(String pathString, String space, String namespace) throws Exception { - return addFile(pathString, space, namespace, null); - } - - private FileEntry addFile(String pathString, String space, String namespace, LocalDateTime date) throws Exception { - Path testFilePath = Paths.get(pathString) - .toAbsolutePath(); - FileEntry fileEntry = createFileEntry(space, namespace); - fileEntry = enrichFileEntry(fileEntry, testFilePath, date); - try (InputStream content = Files.newInputStream(testFilePath)) { - fileStorage.addFile(fileEntry, content); - } - return fileEntry; - } - - private FileEntry addFileContent(String entryName, byte[] content) throws Exception { - FileEntry fileEntry = createFileEntry(entryName, content); - try (InputStream contentStream = new ByteArrayInputStream(content)) { - fileStorage.addFile(fileEntry, contentStream); - } - return fileEntry; - } - - private FileEntry createFileEntry() { - return createFileEntry(spaceId, namespace); - } - - private FileEntry createFileEntry(String entryName, byte[] content) { - return ImmutableFileEntry.builder() - .id(UUID.randomUUID() - .toString()) - .space(spaceId) - .size(BigInteger.valueOf(content.length)) - .modified(LocalDateTime.now()) - .name(entryName) - .build(); - } - - private FileEntry createFileEntry(String space, String namespace) { - return ImmutableFileEntry.builder() - .id(UUID.randomUUID() - .toString()) - .space(space) - .namespace(namespace) - .build(); - } - - private FileEntry enrichFileEntry(FileEntry fileEntry, Path path, LocalDateTime date) throws IOException { - long sizeOfFile = Files.size(path); - BigInteger bigInteger = BigInteger.valueOf(sizeOfFile); - return ImmutableFileEntry.builder() - .from(fileEntry) - .size(bigInteger) - .modified(date != null ? date : LocalDateTime.now()) - .name(path.getFileName() - .toString()) - .build(); - } - - private void assertFileExists(boolean exceptedFileExist, FileEntry actualFile) { + @Override + public void assertFileExists(boolean exceptedFileExist, FileEntry actualFile) { Blob blob = storage.get(CONTAINER, actualFile.getId()); boolean blobExists = blob != null; diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorageTest.java index ae530c6b69..55147da04a 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorageTest.java @@ -1,10 +1,5 @@ package org.cloudfoundry.multiapps.controller.persistence.services; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; - import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -22,7 +17,6 @@ import java.util.UUID; import jakarta.xml.bind.DatatypeConverter; - import org.cloudfoundry.multiapps.common.util.DigestHelper; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; @@ -35,22 +29,27 @@ import org.junit.jupiter.api.Test; import org.springframework.http.MediaType; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + class ObjectStoreFileStorageTest { - private static final String TEST_FILE_LOCATION = "src/test/resources/pexels-photo-401794.jpeg"; - private static final String SECOND_FILE_TEST_LOCATION = "src/test/resources/pexels-photo-463467.jpeg"; - private static final String DIGEST_METHOD = "MD5"; - private static final String CONTAINER = "container4e"; + protected static final String TEST_FILE_LOCATION = "src/test/resources/pexels-photo-401794.jpeg"; + protected static final String SECOND_FILE_TEST_LOCATION = "src/test/resources/pexels-photo-463467.jpeg"; + protected static final String DIGEST_METHOD = "MD5"; + protected static final String CONTAINER = "container4e"; - private String spaceId; - private String namespace; + protected String spaceId; + protected String namespace; - private FileStorage fileStorage; + protected FileStorage fileStorage; private BlobStoreContext blobStoreContext; @BeforeEach - public void setUp() { + protected void setUp() { createBlobStoreContext(); fileStorage = new ObjectStoreFileStorage(blobStoreContext.getBlobStore(), CONTAINER) { @Override @@ -72,7 +71,7 @@ private void createBlobStoreContext() { } @AfterEach - public void tearDown() { + protected void tearDown() { if (blobStoreContext != null) { blobStoreContext.close(); } @@ -163,6 +162,10 @@ void deleteFilesModifiedBefore() throws Exception { assertFileExists(true, fileEntryToRemain2); assertFileExists(false, fileEntryToDelete1); assertFileExists(false, fileEntryToDelete2); + assertBlobInNull(blobWithNoMetadataId); + } + + protected void assertBlobInNull(String blobWithNoMetadataId) { assertNull(blobStoreContext.getBlobStore() .getBlob(CONTAINER, blobWithNoMetadataId)); } @@ -176,11 +179,10 @@ void testConnection() { void testDeleteFilesByIds() throws Exception { FileEntry fileEntry = addFile(TEST_FILE_LOCATION); fileStorage.deleteFilesByIds(List.of(fileEntry.getId())); - assertNull(blobStoreContext.getBlobStore() - .getBlob(CONTAINER, fileEntry.getId())); + assertBlobInNull(fileEntry.getId()); } - private String addBlobWithNoMetadata() throws Exception { + protected String addBlobWithNoMetadata() throws Exception { BlobStore blobStore = blobStoreContext.getBlobStore(); Path path = Paths.get(TEST_FILE_LOCATION); long fileSize = Files.size(path); @@ -218,7 +220,7 @@ void testFileContentNotExisting() throws Exception { assertThrows(FileStorageException.class, () -> validateFileContent(dummyFileEntry, fileDigest)); } - private void validateFileContent(FileEntry storedFile, final String expectedFileChecksum) throws FileStorageException { + protected void validateFileContent(FileEntry storedFile, final String expectedFileChecksum) throws FileStorageException { fileStorage.processFileContent(storedFile.getSpace(), storedFile.getId(), contentStream -> { // make a digest out of the content and compare it to the original final byte[] digest = calculateFileDigest(contentStream); @@ -242,21 +244,21 @@ private byte[] calculateFileDigest(InputStream contentStream) throws IOException } } - private void addBigAmountOfEntries() throws Exception { + protected void addBigAmountOfEntries() throws Exception { for (int i = 0; i < 3001; i++) { addFileContent("test-file-" + i, "test".getBytes()); } } - private FileEntry addFile(String pathString) throws Exception { + protected FileEntry addFile(String pathString) throws Exception { return addFile(pathString, spaceId, namespace); } - private FileEntry addFile(String pathString, String space, String namespace) throws Exception { + protected FileEntry addFile(String pathString, String space, String namespace) throws Exception { return addFile(pathString, space, namespace, null); } - private FileEntry addFile(String pathString, String space, String namespace, LocalDateTime date) throws Exception { + protected FileEntry addFile(String pathString, String space, String namespace, LocalDateTime date) throws Exception { Path testFilePath = Paths.get(pathString) .toAbsolutePath(); FileEntry fileEntry = createFileEntry(space, namespace); @@ -311,7 +313,7 @@ private FileEntry enrichFileEntry(FileEntry fileEntry, Path path, LocalDateTime .build(); } - private void assertFileExists(boolean exceptedFileExist, FileEntry actualFile) { + protected void assertFileExists(boolean exceptedFileExist, FileEntry actualFile) { Blob blob = blobStoreContext.getBlobStore() .getBlob(CONTAINER, actualFile.getId()); boolean blobExists = blob != null; diff --git a/multiapps-controller-web/pom.xml b/multiapps-controller-web/pom.xml index 3049aea29f..d31b2e25fd 100644 --- a/multiapps-controller-web/pom.xml +++ b/multiapps-controller-web/pom.xml @@ -203,5 +203,9 @@ com.google.cloud google-cloud-storage + + com.google.cloud + google-cloud-nio + diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java index f227fd8cf2..0f3d0a0887 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java @@ -127,7 +127,7 @@ private Optional tryToCreateObjectStore(ObjectStoreServiceInfo obje private FileStorage getFileStorageBasedOnProvider(ObjectStoreServiceInfo objectStoreServiceInfo) { if (Constants.GOOGLE_CLOUD_STORAGE.equals(objectStoreServiceInfo.getProvider())) { - return createFileStorage(objectStoreServiceInfo); + return createGcpFileStorage(objectStoreServiceInfo); } else { BlobStoreContext context = getBlobStoreContext(objectStoreServiceInfo); return createFileStorage(objectStoreServiceInfo, context); @@ -196,7 +196,7 @@ protected ObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo object return new ObjectStoreFileStorage(context.getBlobStore(), objectStoreServiceInfo.getContainer()); } - protected GcpObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { + protected GcpObjectStoreFileStorage createGcpFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { return new GcpObjectStoreFileStorage(objectStoreServiceInfo.getContainer(), objectStoreServiceInfo.getGcpStorage()); } diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java index 950eb5917f..820887d619 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java @@ -162,7 +162,7 @@ protected ObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo object } @Override - protected GcpObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { + protected GcpObjectStoreFileStorage createGcpFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { return ObjectStoreFileStorageFactoryBeanTest.this.gcpObjectStoreFileStorage; } diff --git a/pom.xml b/pom.xml index 8d05555f72..3d29899d6c 100644 --- a/pom.xml +++ b/pom.xml @@ -57,7 +57,8 @@ 1.28.0 3.2.2 1.3.1 - 2.7.4 + 2.56.0 + 0.128.5 multiapps-controller-client @@ -292,12 +293,12 @@ com.google.cloud google-cloud-storage - 2.56.0 + ${google-cloud.version} com.google.cloud google-cloud-nio - 0.128.5 + ${google-cloud-nio.version} test From 1c8050e064deec74bb7a0b56150bb3d90d1873f3 Mon Sep 17 00:00:00 2001 From: Yavor16 Date: Mon, 20 Oct 2025 14:10:11 +0300 Subject: [PATCH 03/12] Fix comments --- .../src/main/java/module-info.java | 3 + .../services/GcpObjectStoreFileStorage.java | 166 +++++++++--------- .../GcpObjectStoreFileStorageTest.java | 8 +- .../ObjectStoreFileStorageFactoryBean.java | 85 ++++++--- .../ObjectStoreServiceInfoCreator.java | 46 +---- ...ObjectStoreFileStorageFactoryBeanTest.java | 13 +- .../ObjectStoreServiceInfoCreatorTest.java | 29 +-- 7 files changed, 158 insertions(+), 192 deletions(-) diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index 013753e4d9..557d727bc3 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -57,4 +57,7 @@ requires static org.immutables.value; requires jakarta.xml.bind; requires org.bouncycastle.fips.pkix; + requires com.google.auth.oauth2; + requires com.google.auth; + requires org.threeten.bp; } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java index a91bde5c3c..46ea00d17b 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java @@ -1,39 +1,77 @@ package org.cloudfoundry.multiapps.controller.persistence.services; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.channels.Channels; import java.text.MessageFormat; import java.time.LocalDateTime; +import java.util.Base64; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; +import com.google.api.gax.retrying.RetrySettings; +import com.google.auth.Credentials; +import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.ReadChannel; import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobId; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageException; -import org.cloudfoundry.multiapps.common.util.MiscUtil; +import com.google.cloud.storage.StorageOptions; +import com.google.cloud.storage.StorageRetryStrategy; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; +import org.threeten.bp.Duration; public class GcpObjectStoreFileStorage implements FileStorage { - private static final Logger LOGGER = LoggerFactory.getLogger(GcpObjectStoreFileStorage.class); private final String bucketName; private final Storage storage; - private static final long RETRY_WAIT_TIME = 5000L; - - public GcpObjectStoreFileStorage(String bucketName, Storage storage) { - this.bucketName = bucketName; - this.storage = storage; + private static final String BUCKET = "bucket"; + private static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData"; + + public GcpObjectStoreFileStorage(Map credentials) { + this.bucketName = (String) credentials.get(BUCKET); + this.storage = createObjectStoreStorage(credentials); + } + + protected Storage createObjectStoreStorage(Map credentials) { + return StorageOptions.newBuilder() + .setCredentials(getGcpCredentialsSupplier(credentials)) + .setStorageRetryStrategy(StorageRetryStrategy.getDefaultStorageRetryStrategy()) + .setRetrySettings( + RetrySettings.newBuilder() + .setMaxAttempts(6) + .setInitialRetryDelay(Duration.ofMillis(250)) + .setRetryDelayMultiplier(2.0) + .setMaxRetryDelay(Duration.ofSeconds(10)) + .setInitialRpcTimeout(Duration.ofSeconds(60)) + .setRpcTimeoutMultiplier(1.0) + .setMaxRpcTimeout(Duration.ofSeconds(60)) + .setTotalTimeout(Duration.ofMinutes(10)) + .build()) + .build() + .getService(); + } + + private Credentials getGcpCredentialsSupplier(Map credentials) { + if (!credentials.containsKey(BASE_64_ENCODED_PRIVATE_KEY_DATA)) { + return null; + } + byte[] decodedKey = Base64.getDecoder() + .decode((String) credentials.get(BASE_64_ENCODED_PRIVATE_KEY_DATA)); + try { + return GoogleCredentials.fromStream(new ByteArrayInputStream(decodedKey)); + } catch (IOException e) { + throw new IllegalStateException(e); + } } @Override @@ -45,21 +83,14 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage .setMetadata(ObjectStoreUtil.createFileEntryMetadata(fileEntry)) .build(); - putBlobWithRetries(blobInfo, content, 3); + putBlob(blobInfo, content); } - private void putBlobWithRetries(BlobInfo blobInfo, InputStream content, int retries) throws FileStorageException { - for (int i = 1; i <= retries; i++) { - try { - storage.createFrom(blobInfo, content); - return; - } catch (IOException e) { - LOGGER.warn(MessageFormat.format(Messages.ATTEMPT_TO_UPLOAD_BLOB_FAILED, i, retries, e.getMessage()), e); - if (i == retries) { - throw new FileStorageException(e); - } - } - MiscUtil.sleep(i * getRetryWaitTime()); + private void putBlob(BlobInfo blobInfo, InputStream content) throws FileStorageException { + try { + storage.createFrom(blobInfo, content); + } catch (IOException | StorageException e) { + throw new FileStorageException(e); } } @@ -94,69 +125,40 @@ public int deleteFilesModifiedBefore(LocalDateTime modificationTime) { blob -> ObjectStoreUtil.filterByModificationTime(blob.getMetadata(), blob.getName(), modificationTime)); } - private InputStream getBlobPayloadWithOffset(FileEntry fileEntry, long startOffset, long endOffset) - throws FileStorageException { - try { - return getBlobWithRetriesWithOffset(fileEntry, 3, startOffset, endOffset); - } catch (IOException e) { - throw new FileStorageException(e); - } - } - @Override public T processFileContent(String space, String id, FileContentProcessor fileContentProcessor) throws FileStorageException { FileEntry fileEntry = ObjectStoreUtil.createFileEntry(space, id); - try (InputStream inputStream = openBlobStreamWithRetries(fileEntry, 3)) { + try (InputStream inputStream = openBlobStream(fileEntry)) { return fileContentProcessor.process(inputStream); } catch (Exception e) { throw new FileStorageException(e); } } - private InputStream openBlobStreamWithRetries(FileEntry fileEntry, int maxAttempts) throws FileStorageException { - Blob blob = getBlobWithRetries(fileEntry, maxAttempts); - if (blob == null) { - throw new FileStorageException( - MessageFormat.format(Messages.FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST, - fileEntry.getId(), fileEntry.getSpace())); - } + private InputStream openBlobStream(FileEntry fileEntry) throws FileStorageException { + Blob blob = getBlob(fileEntry); return Channels.newInputStream(blob.reader()); } - private Blob getBlobWithRetries(FileEntry fileEntry, int retries) { - for (int i = 1; i <= retries; i++) { - try { - return storage.get(bucketName, fileEntry.getId()); - } catch (StorageException e) { - LOGGER.warn(MessageFormat.format(Messages.ATTEMPT_TO_DOWNLOAD_MISSING_BLOB, i, retries, fileEntry.getId())); - if (i == retries) { - break; - } - MiscUtil.sleep(i * getRetryWaitTime()); + private Blob getBlob(FileEntry fileEntry) throws FileStorageException { + try { + Blob blob = storage.get(bucketName, fileEntry.getId()); + if (blob == null) { + throw new FileStorageException( + MessageFormat.format(Messages.FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST, + fileEntry.getId(), fileEntry.getSpace())); } + return blob; + } catch (StorageException e) { + throw new FileStorageException(e); } - return null; - } - - protected long getRetryWaitTime() { - return RETRY_WAIT_TIME; } @Override public InputStream openInputStream(String space, String id) throws FileStorageException { FileEntry fileEntry = ObjectStoreUtil.createFileEntry(space, id); - return getBlobStream(fileEntry); - } - - private InputStream getBlobStream(FileEntry fileEntry) throws FileStorageException { - Blob blob = getBlobWithRetries(fileEntry, 3); - if (blob == null) { - throw new FileStorageException( - MessageFormat.format(Messages.FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST, - fileEntry.getId(), fileEntry.getSpace())); - } - return Channels.newInputStream(blob.reader()); + return openBlobStream(fileEntry); } @Override @@ -173,15 +175,15 @@ public void deleteFilesByIds(List fileIds) throws FileStorageException { public T processArchiveEntryContent(FileContentToProcess fileContentToProcess, FileContentProcessor fileContentProcessor) throws FileStorageException { FileEntry fileEntry = ObjectStoreUtil.createFileEntry(fileContentToProcess.getSpaceGuid(), fileContentToProcess.getGuid()); - InputStream res = getBlobPayloadWithOffset(fileEntry, fileContentToProcess.getStartOffset(), - fileContentToProcess.getEndOffset()); - return processContent(fileContentProcessor, res); + InputStream blobPayload = getBlobPayloadWithOffset(fileEntry, fileContentToProcess.getStartOffset(), + fileContentToProcess.getEndOffset()); + return processContent(fileContentProcessor, blobPayload); } - private T processContent(FileContentProcessor fileContentProcessor, InputStream res) + private T processContent(FileContentProcessor fileContentProcessor, InputStream inputStream) throws FileStorageException { try { - return fileContentProcessor.process(res); + return fileContentProcessor.process(inputStream); } catch (IOException e) { throw new FileStorageException(e); } @@ -212,19 +214,17 @@ protected Set getEntryNames(Predicate filter) { .collect(Collectors.toSet()); } - private InputStream getBlobWithRetriesWithOffset(FileEntry fileEntry, int retries, long startOffset, long endOffset) - throws IOException, FileStorageException { - Blob blob = getBlobWithRetries(fileEntry, retries); - if (blob == null) { - throw new FileStorageException( - MessageFormat.format(Messages.FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST, - fileEntry.getId(), fileEntry.getSpace())); - } - BlobId blobId = BlobId.of(bucketName, fileEntry.getId()); - ReadChannel reader = storage.reader(blobId); - reader.seek(startOffset); - reader.limit(endOffset + 1); + private InputStream getBlobPayloadWithOffset(FileEntry fileEntry, long startOffset, long endOffset) + throws FileStorageException { + try { + Blob blob = getBlob(fileEntry); + ReadChannel reader = storage.reader(blob.getBlobId()); + reader.seek(startOffset); + reader.limit(endOffset + 1); - return Channels.newInputStream(reader); + return Channels.newInputStream(reader); + } catch (IOException | StorageException e) { + throw new FileStorageException(e); + } } } diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java index 8170cd351b..0e7fa581c7 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java @@ -3,6 +3,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.function.Predicate; @@ -29,10 +30,11 @@ class GcpObjectStoreFileStorageTest extends ObjectStoreFileStorageTest { public void setUp() { storage = LocalStorageHelper.getOptions() .getService(); - fileStorage = new GcpObjectStoreFileStorage(CONTAINER, storage) { + fileStorage = new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER)) { + @Override - protected long getRetryWaitTime() { - return 0; + protected Storage createObjectStoreStorage(Map credentials) { + return storage; } @Override diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java index 0f3d0a0887..45ca4b71f0 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java @@ -22,6 +22,7 @@ import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder; import org.cloudfoundry.multiapps.controller.web.Constants; import org.cloudfoundry.multiapps.controller.web.Messages; +import org.cloudfoundry.multiapps.controller.web.configuration.service.ImmutableObjectStoreServiceInfo; import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfo; import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfoCreator; import org.jclouds.ContextBuilder; @@ -66,15 +67,11 @@ private FileStorage createObjectStoreFileStorage() { return createObjectStoreFromFirstReachableProvider(exceptions, providersServiceInfo); } - Optional objectStoreServiceInfoOptional = getAppropriateProvider(objectStoreProviderName, - providersServiceInfo); + Optional optionalFileStorage = createObjectStoreBasedOnProvider(objectStoreProviderName, providersServiceInfo, + exceptions); - if (objectStoreServiceInfoOptional.isPresent()) { - ObjectStoreServiceInfo objectStoreServiceInfo = objectStoreServiceInfoOptional.get(); - Optional createdObjectStore = tryToCreateObjectStore(objectStoreServiceInfo, exceptions); - if (createdObjectStore.isPresent()) { - return createdObjectStore.get(); - } + if (optionalFileStorage.isPresent()) { + return optionalFileStorage.get(); } throw buildNoValidObjectStoreException(exceptions); @@ -83,15 +80,33 @@ private FileStorage createObjectStoreFileStorage() { public FileStorage createObjectStoreFromFirstReachableProvider(Map exceptions, List providersServiceInfo) { for (ObjectStoreServiceInfo objectStoreServiceInfo : providersServiceInfo) { - Optional createdObjectStoreOptional = tryToCreateObjectStore(objectStoreServiceInfo, exceptions); - if (createdObjectStoreOptional.isPresent()) { - return createdObjectStoreOptional.get(); + Optional createdObjectStore = tryToCreateObjectStore(objectStoreServiceInfo, exceptions); + if (createdObjectStore.isPresent()) { + return createdObjectStore.get(); } } - + Optional gcpObjectStoreOpt = tryToCreateGcpObjectStore(exceptions); + if (gcpObjectStoreOpt.isPresent()) { + return gcpObjectStoreOpt.get(); + } throw buildNoValidObjectStoreException(exceptions); } + private Optional createObjectStoreBasedOnProvider(String objectStoreProviderName, + List providersServiceInfo, + Map exceptions) { + Optional objectStoreServiceInfoOptional = getAppropriateProvider(objectStoreProviderName, + providersServiceInfo); + Optional createdObjectStore; + if (objectStoreServiceInfoOptional.isPresent()) { + ObjectStoreServiceInfo objectStoreServiceInfo = objectStoreServiceInfoOptional.get(); + createdObjectStore = tryToCreateObjectStore(objectStoreServiceInfo, exceptions); + } else { + createdObjectStore = tryToCreateGcpObjectStore(exceptions); + } + return createdObjectStore; + } + private Optional getAppropriateProvider(String objectStoreProviderName, List providersServiceInfo) { String appropriateProvider = Constants.ENV_TO_OS_PROVIDER.get(objectStoreProviderName); @@ -100,16 +115,10 @@ private Optional getAppropriateProvider(String objectSto .findFirst(); } - private boolean isObjectStoreEnvValid(String objectStoreProviderName) { - return objectStoreProviderName != null && !objectStoreProviderName.isEmpty() && Constants.ENV_TO_OS_PROVIDER.containsKey( - objectStoreProviderName); - } - - private IllegalStateException buildNoValidObjectStoreException(Map exceptions) { - exceptions.forEach((provider, exception) -> LOGGER.error( - MessageFormat.format(Messages.CANNOT_CREATE_OBJECT_STORE_CLIENT_WITH_PROVIDER_0, provider), - exception)); - return new IllegalStateException(Messages.NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND); + private Optional tryToCreateGcpObjectStore(Map exceptions) { + return tryToCreateObjectStore(ImmutableObjectStoreServiceInfo.builder() + .provider(Constants.GOOGLE_CLOUD_STORAGE) + .build(), exceptions); } private Optional tryToCreateObjectStore(ObjectStoreServiceInfo objectStoreServiceInfo, @@ -127,19 +136,40 @@ private Optional tryToCreateObjectStore(ObjectStoreServiceInfo obje private FileStorage getFileStorageBasedOnProvider(ObjectStoreServiceInfo objectStoreServiceInfo) { if (Constants.GOOGLE_CLOUD_STORAGE.equals(objectStoreServiceInfo.getProvider())) { - return createGcpFileStorage(objectStoreServiceInfo); + return createGcpFileStorage(); } else { BlobStoreContext context = getBlobStoreContext(objectStoreServiceInfo); return createFileStorage(objectStoreServiceInfo, context); } } + private boolean isObjectStoreEnvValid(String objectStoreProviderName) { + return objectStoreProviderName != null && !objectStoreProviderName.isEmpty() && Constants.ENV_TO_OS_PROVIDER.containsKey( + objectStoreProviderName); + } + + private IllegalStateException buildNoValidObjectStoreException(Map exceptions) { + exceptions.forEach((provider, exception) -> LOGGER.error( + MessageFormat.format(Messages.CANNOT_CREATE_OBJECT_STORE_CLIENT_WITH_PROVIDER_0, provider), + exception)); + return new IllegalStateException(Messages.NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND); + } + public List getProvidersServiceInfo() { + Map credentials = getServiceCredentials(); + if (credentials.isEmpty()) { + return Collections.emptyList(); + } + return new ObjectStoreServiceInfoCreator().getAllProvidersServiceInfo(credentials); + } + + private Map getServiceCredentials() { CfService service = environmentServicesFinder.findService(serviceName); if (service == null) { - return Collections.emptyList(); + return Map.of(); } - return new ObjectStoreServiceInfoCreator().getAllProvidersServiceInfo(service); + return service.getCredentials() + .getMap(); } private BlobStoreContext getBlobStoreContext(ObjectStoreServiceInfo serviceInfo) { @@ -196,8 +226,9 @@ protected ObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo object return new ObjectStoreFileStorage(context.getBlobStore(), objectStoreServiceInfo.getContainer()); } - protected GcpObjectStoreFileStorage createGcpFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { - return new GcpObjectStoreFileStorage(objectStoreServiceInfo.getContainer(), objectStoreServiceInfo.getGcpStorage()); + protected GcpObjectStoreFileStorage createGcpFileStorage() { + Map credentials = getServiceCredentials(); + return new GcpObjectStoreFileStorage(credentials); } @Override diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java index 37da7ccbc8..57fd802b88 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java @@ -1,29 +1,18 @@ package org.cloudfoundry.multiapps.controller.web.configuration.service; -import java.io.ByteArrayInputStream; -import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; -import java.util.Base64; import java.util.List; import java.util.Map; -import com.google.auth.Credentials; -import com.google.auth.oauth2.GoogleCredentials; -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.StorageOptions; -import io.pivotal.cfenv.core.CfService; import org.cloudfoundry.multiapps.controller.web.Constants; import org.cloudfoundry.multiapps.controller.web.Messages; public class ObjectStoreServiceInfoCreator { - public List getAllProvidersServiceInfo(CfService service) { - Map credentials = service.getCredentials() - .getMap(); + public List getAllProvidersServiceInfo(Map credentials) { return List.of(createServiceInfoForAws(credentials), createServiceInfoForAliCloud(credentials), - createServiceInfoForAzure(credentials), createServiceInfoForGcpCloud(credentials), - createServiceInfoForCcee(credentials)); + createServiceInfoForAzure(credentials), createServiceInfoForCcee(credentials)); } private ObjectStoreServiceInfo createServiceInfoForAws(Map credentials) { @@ -97,35 +86,4 @@ private URL getContainerUriEndpoint(Map credentials) { throw new IllegalStateException(Messages.CANNOT_PARSE_CONTAINER_URI_OF_OBJECT_STORE, e); } } - - private ObjectStoreServiceInfo createServiceInfoForGcpCloud(Map credentials) { - String bucketName = (String) credentials.get(Constants.BUCKET); - Storage storage = createObjectStoreStorage(credentials); - return ImmutableObjectStoreServiceInfo.builder() - .provider(Constants.GOOGLE_CLOUD_STORAGE) - .container(bucketName) - .gcpStorage(storage) - .build(); - } - - public Storage createObjectStoreStorage(Map credentials) { - return StorageOptions.newBuilder() - .setCredentials(getGcpCredentialsSupplier(credentials)) - .build() - .getService(); - } - - private Credentials getGcpCredentialsSupplier(Map credentials) { - if (!credentials.containsKey(Constants.BASE_64_ENCODED_PRIVATE_KEY_DATA)) { - return null; - } - byte[] decodedKey = Base64.getDecoder() - .decode((String) credentials.get(Constants.BASE_64_ENCODED_PRIVATE_KEY_DATA)); - try { - return GoogleCredentials.fromStream(new ByteArrayInputStream(decodedKey)); - } catch (IOException e) { - throw new IllegalStateException(e); - } - } - } diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java index 820887d619..a1b8b4d809 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java @@ -5,8 +5,6 @@ import java.util.Map; import java.util.Set; -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; import io.pivotal.cfenv.core.CfCredentials; import io.pivotal.cfenv.core.CfService; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; @@ -162,7 +160,7 @@ protected ObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo object } @Override - protected GcpObjectStoreFileStorage createGcpFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { + protected GcpObjectStoreFileStorage createGcpFileStorage() { return ObjectStoreFileStorageFactoryBeanTest.this.gcpObjectStoreFileStorage; } @@ -170,7 +168,8 @@ protected GcpObjectStoreFileStorage createGcpFileStorage(ObjectStoreServiceInfo public List getProvidersServiceInfo() { CfService service = environmentServicesFinder.findService("deploy-service-os"); if (service != null) { - return new ObjectStoreServiceInfoCreatorMock().getAllProvidersServiceInfo(service); + return new ObjectStoreServiceInfoCreatorMock().getAllProvidersServiceInfo(service.getCredentials() + .getMap()); } else { return List.of(); } @@ -178,11 +177,5 @@ public List getProvidersServiceInfo() { } private class ObjectStoreServiceInfoCreatorMock extends ObjectStoreServiceInfoCreator { - - @Override - public Storage createObjectStoreStorage(Map credentials) { - return LocalStorageHelper.getOptions() - .getService(); - } } } diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java index 87fecb25f5..c233caac67 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java @@ -2,8 +2,6 @@ import java.net.MalformedURLException; import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.Base64; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -47,14 +45,15 @@ void setUp() { static Stream testDifferentProviders() throws MalformedURLException { return Stream.of(Arguments.of(buildCfService(buildAliCloudCredentials()), buildAliCloudObjectStoreServiceInfo()), Arguments.of(buildCfService(buildAwsCredentials()), buildAwsObjectStoreServiceInfo()), - Arguments.of(buildCfService(buildAzureCredentials()), buildAzureObjectStoreServiceInfo()), - Arguments.of(buildCfService(buildGcpCredentials()), buildGcpObjectStoreServiceInfo())); + Arguments.of(buildCfService(buildAzureCredentials()), buildAzureObjectStoreServiceInfo())); } @ParameterizedTest @MethodSource void testDifferentProviders(CfService cfService, ObjectStoreServiceInfo exprectedObjectStoreServiceInfo) { - List providersServiceInfo = objectStoreServiceInfoCreator.getAllProvidersServiceInfo(cfService); + List providersServiceInfo = objectStoreServiceInfoCreator.getAllProvidersServiceInfo( + cfService.getCredentials() + .getMap()); assertTrue(providersServiceInfo.contains(exprectedObjectStoreServiceInfo)); } @@ -123,26 +122,6 @@ private static ObjectStoreServiceInfo buildAzureObjectStoreServiceInfo() throws .build(); } - private static Map buildGcpCredentials() { - Map credentials = new HashMap<>(); - credentials.put(Constants.BUCKET, BUCKET_VALUE); - credentials.put(Constants.BASE_64_ENCODED_PRIVATE_KEY_DATA, Base64.getEncoder() - .encodeToString("encoded_data".getBytes(StandardCharsets.UTF_8))); - return credentials; - } - - private static ObjectStoreServiceInfo buildGcpObjectStoreServiceInfo() { - return ImmutableObjectStoreServiceInfo.builder() - .provider(Constants.GOOGLE_CLOUD_STORAGE) - .gcpStorage(STORAGE) - .container(BUCKET_VALUE) - .build(); - } - private static class ObjectStoreServiceInfoCreatorMock extends ObjectStoreServiceInfoCreator { - @Override - public Storage createObjectStoreStorage(Map credentials) { - return STORAGE; - } } } From ea5f5bffff9c572d9270f3ad4d04d416e143f2da Mon Sep 17 00:00:00 2001 From: Yavor16 Date: Wed, 29 Oct 2025 13:27:33 +0200 Subject: [PATCH 04/12] Fix comments --- .../services/GcpObjectStoreFileStorage.java | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java index 46ea00d17b..907461f222 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java @@ -6,6 +6,7 @@ import java.nio.channels.Channels; import java.text.MessageFormat; import java.time.LocalDateTime; +import java.util.ArrayList; import java.util.Base64; import java.util.List; import java.util.Map; @@ -27,6 +28,8 @@ import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; import org.threeten.bp.Duration; @@ -35,7 +38,13 @@ public class GcpObjectStoreFileStorage implements FileStorage { private final String bucketName; private final Storage storage; private static final String BUCKET = "bucket"; + private static final int OBJECTSTORE_MAX_ATTEMPTS_CONFIG = 6; + private static final double OBJECTSTORE_RETRY_DELAY_MULTIPLIER_CONFIG = 2.0; + private static final Duration OBJECTSTORE_TOTAL_TIMEOUT_CONFIG = Duration.ofMinutes(10); + private static final Duration OBJECTSTORE_MAX_RETRY_DELAY_CONFIG = Duration.ofSeconds(10); + private static final Duration OBJECTSTORE_INITIAL_RETRY_DELAY_CONFIG = Duration.ofMillis(250); private static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData"; + private static final Logger LOGGER = LoggerFactory.getLogger(GcpObjectStoreFileStorage.class); public GcpObjectStoreFileStorage(Map credentials) { this.bucketName = (String) credentials.get(BUCKET); @@ -43,19 +52,16 @@ public GcpObjectStoreFileStorage(Map credentials) { } protected Storage createObjectStoreStorage(Map credentials) { - return StorageOptions.newBuilder() + return StorageOptions.http() .setCredentials(getGcpCredentialsSupplier(credentials)) .setStorageRetryStrategy(StorageRetryStrategy.getDefaultStorageRetryStrategy()) .setRetrySettings( RetrySettings.newBuilder() - .setMaxAttempts(6) - .setInitialRetryDelay(Duration.ofMillis(250)) - .setRetryDelayMultiplier(2.0) - .setMaxRetryDelay(Duration.ofSeconds(10)) - .setInitialRpcTimeout(Duration.ofSeconds(60)) - .setRpcTimeoutMultiplier(1.0) - .setMaxRpcTimeout(Duration.ofSeconds(60)) - .setTotalTimeout(Duration.ofMinutes(10)) + .setMaxAttempts(OBJECTSTORE_MAX_ATTEMPTS_CONFIG) + .setTotalTimeout(OBJECTSTORE_TOTAL_TIMEOUT_CONFIG) + .setMaxRetryDelay(OBJECTSTORE_MAX_RETRY_DELAY_CONFIG) + .setInitialRetryDelay(OBJECTSTORE_INITIAL_RETRY_DELAY_CONFIG) + .setRetryDelayMultiplier(OBJECTSTORE_RETRY_DELAY_MULTIPLIER_CONFIG) .build()) .build() .getService(); @@ -163,6 +169,8 @@ public InputStream openInputStream(String space, String id) throws FileStorageEx @Override public void testConnection() { + LOGGER.error("Test: " + storage.getClass() + .getName()); storage.get(bucketName, "test"); } @@ -199,13 +207,16 @@ protected int removeBlobsByFilter(Predicate filter) { List blobIds = getEntryNames(filter).stream() .map(entry -> BlobId.of(bucketName, entry)) .toList(); - + List deletedBlobsResults = new ArrayList<>(); if (!blobIds.isEmpty()) { - storage.delete(blobIds); + deletedBlobsResults = storage.delete(blobIds); } - return blobIds.size(); + return deletedBlobsResults.stream() + .filter(Boolean::booleanValue) + .toList() + .size(); } - + protected Set getEntryNames(Predicate filter) { return storage.list(bucketName) .streamAll() From 7e0471ef0afae5239e41937bb8cd2ddfb87766c2 Mon Sep 17 00:00:00 2001 From: Yavor16 Date: Wed, 29 Oct 2025 13:51:48 +0200 Subject: [PATCH 05/12] Fix comments --- .../services/GcpObjectStoreFileStorage.java | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java index 907461f222..122a412079 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java @@ -25,11 +25,10 @@ import com.google.cloud.storage.StorageException; import com.google.cloud.storage.StorageOptions; import com.google.cloud.storage.StorageRetryStrategy; +import org.cloudfoundry.multiapps.common.util.MiscUtil; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; import org.threeten.bp.Duration; @@ -44,7 +43,6 @@ public class GcpObjectStoreFileStorage implements FileStorage { private static final Duration OBJECTSTORE_MAX_RETRY_DELAY_CONFIG = Duration.ofSeconds(10); private static final Duration OBJECTSTORE_INITIAL_RETRY_DELAY_CONFIG = Duration.ofMillis(250); private static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData"; - private static final Logger LOGGER = LoggerFactory.getLogger(GcpObjectStoreFileStorage.class); public GcpObjectStoreFileStorage(Map credentials) { this.bucketName = (String) credentials.get(BUCKET); @@ -169,8 +167,6 @@ public InputStream openInputStream(String space, String id) throws FileStorageEx @Override public void testConnection() { - LOGGER.error("Test: " + storage.getClass() - .getName()); storage.get(bucketName, "test"); } @@ -211,12 +207,10 @@ protected int removeBlobsByFilter(Predicate filter) { if (!blobIds.isEmpty()) { deletedBlobsResults = storage.delete(blobIds); } - return deletedBlobsResults.stream() - .filter(Boolean::booleanValue) - .toList() - .size(); + return MiscUtil.cast(deletedBlobsResults.stream() + .count()); } - + protected Set getEntryNames(Predicate filter) { return storage.list(bucketName) .streamAll() From cb1a52856b236cfd4e795aca6e1e5c5596b04ab6 Mon Sep 17 00:00:00 2001 From: Yavor16 Date: Wed, 29 Oct 2025 13:58:12 +0200 Subject: [PATCH 06/12] Fix comments --- .../persistence/services/GcpObjectStoreFileStorage.java | 1 + 1 file changed, 1 insertion(+) diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java index 122a412079..a43d601cfe 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java @@ -208,6 +208,7 @@ protected int removeBlobsByFilter(Predicate filter) { deletedBlobsResults = storage.delete(blobIds); } return MiscUtil.cast(deletedBlobsResults.stream() + .filter(Boolean::booleanValue) .count()); } From b588376053452ddfcaefcdedf5b48584228a2f0f Mon Sep 17 00:00:00 2001 From: Yavor16 Date: Tue, 4 Nov 2025 11:11:15 +0200 Subject: [PATCH 07/12] Fix stef comments --- .../services/GcpObjectStoreFileStorage.java | 21 ++-- .../services/ObjectStoreFileStorage.java | 17 +-- ...tStoreUtil.java => ObjectStoreFilter.java} | 30 +---- .../persistence/util/ObjectStoreMapper.java | 35 ++++++ .../GcpObjectStoreFileStorageTest.java | 2 +- .../services/ObjectStoreFileStorageTest.java | 10 +- .../util/ObjectStoreFilterTest.java | 79 ++++++++++++ .../util/ObjectStoreMapperTest.java | 52 ++++++++ .../persistence/util/ObjectStoreUtilTest.java | 117 ------------------ .../ObjectStoreFileStorageFactoryBean.java | 2 +- .../service/ObjectStoreServiceInfo.java | 5 - .../ObjectStoreServiceInfoCreatorTest.java | 4 - pom.xml | 2 +- 13 files changed, 196 insertions(+), 180 deletions(-) rename multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/{ObjectStoreUtil.java => ObjectStoreFilter.java} (65%) create mode 100644 multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapper.java create mode 100644 multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreFilterTest.java create mode 100644 multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapperTest.java delete mode 100644 multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreUtilTest.java diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java index a43d601cfe..92006d87c2 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java @@ -28,7 +28,8 @@ import org.cloudfoundry.multiapps.common.util.MiscUtil; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; -import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreUtil; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; import org.springframework.http.MediaType; import org.threeten.bp.Duration; @@ -84,7 +85,7 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage BlobInfo blobInfo = BlobInfo.newBuilder(blobId) .setContentDisposition(fileEntry.getName()) .setContentType(MediaType.APPLICATION_OCTET_STREAM_VALUE) - .setMetadata(ObjectStoreUtil.createFileEntryMetadata(fileEntry)) + .setMetadata(ObjectStoreMapper.createFileEntryMetadata(fileEntry)) .build(); putBlob(blobInfo, content); @@ -99,7 +100,7 @@ private void putBlob(BlobInfo blobInfo, InputStream content) throws FileStorageE } @Override - public List getFileEntriesWithoutContent(List fileEntries) throws FileStorageException { + public List getFileEntriesWithoutContent(List fileEntries) { Set existingFiles = getAllEntries().stream() .map(Blob::getName) .collect(Collectors.toSet()); @@ -115,24 +116,24 @@ public void deleteFile(String id, String space) { @Override public void deleteFilesBySpaceIds(List spaceIds) { - removeBlobsByFilter(blob -> ObjectStoreUtil.filterBySpaceIds(blob.getMetadata(), spaceIds)); + removeBlobsByFilter(blob -> ObjectStoreFilter.filterBySpaceIds(blob.getMetadata(), spaceIds)); } @Override public void deleteFilesBySpaceAndNamespace(String space, String namespace) { - removeBlobsByFilter(blob -> ObjectStoreUtil.filterBySpaceAndNamespace(blob.getMetadata(), space, namespace)); + removeBlobsByFilter(blob -> ObjectStoreFilter.filterBySpaceAndNamespace(blob.getMetadata(), space, namespace)); } @Override public int deleteFilesModifiedBefore(LocalDateTime modificationTime) { return removeBlobsByFilter( - blob -> ObjectStoreUtil.filterByModificationTime(blob.getMetadata(), blob.getName(), modificationTime)); + blob -> ObjectStoreFilter.filterByModificationTime(blob.getMetadata(), blob.getName(), modificationTime)); } @Override public T processFileContent(String space, String id, FileContentProcessor fileContentProcessor) throws FileStorageException { - FileEntry fileEntry = ObjectStoreUtil.createFileEntry(space, id); + FileEntry fileEntry = ObjectStoreMapper.createFileEntry(space, id); try (InputStream inputStream = openBlobStream(fileEntry)) { return fileContentProcessor.process(inputStream); } catch (Exception e) { @@ -161,7 +162,7 @@ private Blob getBlob(FileEntry fileEntry) throws FileStorageException { @Override public InputStream openInputStream(String space, String id) throws FileStorageException { - FileEntry fileEntry = ObjectStoreUtil.createFileEntry(space, id); + FileEntry fileEntry = ObjectStoreMapper.createFileEntry(space, id); return openBlobStream(fileEntry); } @@ -171,14 +172,14 @@ public void testConnection() { } @Override - public void deleteFilesByIds(List fileIds) throws FileStorageException { + public void deleteFilesByIds(List fileIds) { removeBlobsByFilter(blob -> fileIds.contains(blob.getName())); } @Override public T processArchiveEntryContent(FileContentToProcess fileContentToProcess, FileContentProcessor fileContentProcessor) throws FileStorageException { - FileEntry fileEntry = ObjectStoreUtil.createFileEntry(fileContentToProcess.getSpaceGuid(), fileContentToProcess.getGuid()); + FileEntry fileEntry = ObjectStoreMapper.createFileEntry(fileContentToProcess.getSpaceGuid(), fileContentToProcess.getGuid()); InputStream blobPayload = getBlobPayloadWithOffset(fileEntry, fileContentToProcess.getStartOffset(), fileContentToProcess.getEndOffset()); return processContent(fileContentProcessor, blobPayload); diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java index 150ae02fd2..b402694ab0 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java @@ -14,7 +14,8 @@ import org.cloudfoundry.multiapps.common.util.MiscUtil; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; -import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreUtil; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.ContainerNotFoundException; import org.jclouds.blobstore.domain.Blob; @@ -53,7 +54,7 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage .contentDisposition(fileEntry.getName()) .contentType(MediaType.APPLICATION_OCTET_STREAM_VALUE) .contentLength(fileSize) - .userMetadata(ObjectStoreUtil.createFileEntryMetadata(fileEntry)) + .userMetadata(ObjectStoreMapper.createFileEntryMetadata(fileEntry)) .build(); try { putBlobWithRetries(blob, 3); @@ -81,23 +82,23 @@ public void deleteFile(String id, String space) { @Override public void deleteFilesBySpaceIds(List spaceIds) { - removeBlobsByFilter(blob -> ObjectStoreUtil.filterBySpaceIds(blob.getUserMetadata(), spaceIds)); + removeBlobsByFilter(blob -> ObjectStoreFilter.filterBySpaceIds(blob.getUserMetadata(), spaceIds)); } @Override public void deleteFilesBySpaceAndNamespace(String space, String namespace) { - removeBlobsByFilter(blob -> ObjectStoreUtil.filterBySpaceAndNamespace(blob.getUserMetadata(), space, namespace)); + removeBlobsByFilter(blob -> ObjectStoreFilter.filterBySpaceAndNamespace(blob.getUserMetadata(), space, namespace)); } @Override public int deleteFilesModifiedBefore(LocalDateTime modificationTime) { return removeBlobsByFilter( - blob -> ObjectStoreUtil.filterByModificationTime(blob.getUserMetadata(), blob.getName(), modificationTime)); + blob -> ObjectStoreFilter.filterByModificationTime(blob.getUserMetadata(), blob.getName(), modificationTime)); } @Override public T processFileContent(String space, String id, FileContentProcessor fileContentProcessor) throws FileStorageException { - FileEntry fileEntry = ObjectStoreUtil.createFileEntry(space, id); + FileEntry fileEntry = ObjectStoreMapper.createFileEntry(space, id); try { Payload payload = getBlobPayload(fileEntry); return processContent(fileContentProcessor, payload); @@ -109,7 +110,7 @@ public T processFileContent(String space, String id, FileContentProcessor @Override public T processArchiveEntryContent(FileContentToProcess fileContentToProcess, FileContentProcessor fileContentProcessor) throws FileStorageException { - FileEntry fileEntry = ObjectStoreUtil.createFileEntry(fileContentToProcess.getSpaceGuid(), fileContentToProcess.getGuid()); + FileEntry fileEntry = ObjectStoreMapper.createFileEntry(fileContentToProcess.getSpaceGuid(), fileContentToProcess.getGuid()); try { Payload payload = getBlobPayloadWithOffset(fileEntry, fileContentToProcess.getStartOffset(), fileContentToProcess.getEndOffset()); @@ -155,7 +156,7 @@ private Payload getBlobPayload(FileEntry fileEntry) throws FileStorageException @Override public InputStream openInputStream(String space, String id) throws FileStorageException { - FileEntry fileEntry = ObjectStoreUtil.createFileEntry(space, id); + FileEntry fileEntry = ObjectStoreMapper.createFileEntry(space, id); Payload payload = getBlobPayload(fileEntry); return openPayloadInputStream(payload); } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreUtil.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreFilter.java similarity index 65% rename from multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreUtil.java rename to multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreFilter.java index ec41f7fbe0..22ee92687e 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreUtil.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreFilter.java @@ -4,21 +4,18 @@ import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; -import java.util.HashMap; import java.util.List; import java.util.Map; import org.cloudfoundry.multiapps.controller.persistence.Constants; import org.cloudfoundry.multiapps.controller.persistence.Messages; -import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; -import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.CollectionUtils; -public class ObjectStoreUtil { +public class ObjectStoreFilter { - private static final Logger LOGGER = LoggerFactory.getLogger(ObjectStoreUtil.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ObjectStoreFilter.class); public static boolean filterBySpaceIds(Map metadata, List spaceIds) { if (CollectionUtils.isEmpty(metadata)) { @@ -58,27 +55,4 @@ public static boolean filterByModificationTime(Map metadata, Str return true; } } - - public static Map createFileEntryMetadata(FileEntry fileEntry) { - Map metadata = new HashMap<>(); - metadata.put(Constants.FILE_ENTRY_SPACE.toLowerCase(), fileEntry.getSpace()); - metadata.put(Constants.FILE_ENTRY_MODIFIED.toLowerCase(), - Long.toString(fileEntry.getModified() - .atZone( - ZoneId.systemDefault()) - .toInstant() - .toEpochMilli())); - if (fileEntry.getNamespace() != null) { - metadata.put(Constants.FILE_ENTRY_NAMESPACE.toLowerCase(), fileEntry.getNamespace()); - } - return metadata; - } - - public static FileEntry createFileEntry(String space, String id) { - return ImmutableFileEntry.builder() - .space(space) - .id(id) - .build(); - } - } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapper.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapper.java new file mode 100644 index 0000000000..1875b88a50 --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapper.java @@ -0,0 +1,35 @@ +package org.cloudfoundry.multiapps.controller.persistence.util; + +import java.time.ZoneId; +import java.util.HashMap; +import java.util.Map; + +import org.cloudfoundry.multiapps.controller.persistence.Constants; +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; + +public class ObjectStoreMapper { + + public static Map createFileEntryMetadata(FileEntry fileEntry) { + Map metadata = new HashMap<>(); + metadata.put(Constants.FILE_ENTRY_SPACE.toLowerCase(), fileEntry.getSpace()); + metadata.put(Constants.FILE_ENTRY_MODIFIED.toLowerCase(), + Long.toString(fileEntry.getModified() + .atZone( + ZoneId.systemDefault()) + .toInstant() + .toEpochMilli())); + if (fileEntry.getNamespace() != null) { + metadata.put(Constants.FILE_ENTRY_NAMESPACE.toLowerCase(), fileEntry.getNamespace()); + } + return metadata; + } + + public static FileEntry createFileEntry(String space, String id) { + return ImmutableFileEntry.builder() + .space(space) + .id(id) + .build(); + } + +} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java index 0e7fa581c7..177103fff2 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java @@ -66,7 +66,7 @@ public void tearDown() { } @Override - protected void assertBlobInNull(String blobWithNoMetadataId) { + protected void assertBlobDoesNotExist(String blobWithNoMetadataId) { assertNull(storage.get(blobWithNoMetadataId)); } diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorageTest.java index 55147da04a..f9477f7601 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorageTest.java @@ -162,10 +162,10 @@ void deleteFilesModifiedBefore() throws Exception { assertFileExists(true, fileEntryToRemain2); assertFileExists(false, fileEntryToDelete1); assertFileExists(false, fileEntryToDelete2); - assertBlobInNull(blobWithNoMetadataId); + assertBlobDoesNotExist(blobWithNoMetadataId); } - protected void assertBlobInNull(String blobWithNoMetadataId) { + protected void assertBlobDoesNotExist(String blobWithNoMetadataId) { assertNull(blobStoreContext.getBlobStore() .getBlob(CONTAINER, blobWithNoMetadataId)); } @@ -179,7 +179,7 @@ void testConnection() { void testDeleteFilesByIds() throws Exception { FileEntry fileEntry = addFile(TEST_FILE_LOCATION); fileStorage.deleteFilesByIds(List.of(fileEntry.getId())); - assertBlobInNull(fileEntry.getId()); + assertBlobDoesNotExist(fileEntry.getId()); } protected String addBlobWithNoMetadata() throws Exception { @@ -313,12 +313,12 @@ private FileEntry enrichFileEntry(FileEntry fileEntry, Path path, LocalDateTime .build(); } - protected void assertFileExists(boolean exceptedFileExist, FileEntry actualFile) { + protected void assertFileExists(boolean expectedFileExist, FileEntry actualFile) { Blob blob = blobStoreContext.getBlobStore() .getBlob(CONTAINER, actualFile.getId()); boolean blobExists = blob != null; - assertEquals(exceptedFileExist, blobExists); + assertEquals(expectedFileExist, blobExists); } } diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreFilterTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreFilterTest.java new file mode 100644 index 0000000000..a0b60a096b --- /dev/null +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreFilterTest.java @@ -0,0 +1,79 @@ +package org.cloudfoundry.multiapps.controller.persistence.util; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; + +import org.cloudfoundry.multiapps.controller.persistence.Constants; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ObjectStoreFilterTest { + + private static final String SPACE_ID = "spaceId"; + private static final String BLOBNAME = "blobName"; + private static final String SPACE_ID_2 = "spaceId2"; + private static final String NAMESPACE = "namespace"; + private static final String NAMESPACE_2 = "namespace2"; + + @Test + void testFilterBySpaceIdsWhenMetadataIsEmpty() { + assertFalse(ObjectStoreFilter.filterBySpaceIds(Map.of(), List.of())); + } + + @Test + void testFilterBySpaceIdsWhenTheMetadataContainsTheSpaceId() { + assertTrue(ObjectStoreFilter.filterBySpaceIds(buildMetadata(), List.of(SPACE_ID))); + } + + @Test + void testFilterBySpaceIdsWhenTheMetadataDoesNotContainTheSpaceId() { + assertFalse(ObjectStoreFilter.filterBySpaceIds(buildMetadata(), List.of(SPACE_ID_2))); + } + + @Test + void testFilterBySpaceAndNamespaceWhenMetadataIsEmpty() { + assertFalse(ObjectStoreFilter.filterBySpaceAndNamespace(Map.of(), SPACE_ID, NAMESPACE)); + } + + @Test + void testFilterBySpaceAndNamespaceWhenTheSpacesAreDifferent() { + assertFalse(ObjectStoreFilter.filterBySpaceAndNamespace(buildMetadata(), SPACE_ID_2, NAMESPACE)); + } + + @Test + void testFilterBySpaceAndNamespaceWhenTheNamespacesAreDifferent() { + assertFalse(ObjectStoreFilter.filterBySpaceAndNamespace(buildMetadata(), SPACE_ID, NAMESPACE_2)); + } + + @Test + void testFilterBySpaceAndNamespaceWhenAllMatch() { + assertTrue(ObjectStoreFilter.filterBySpaceAndNamespace(buildMetadata(), SPACE_ID, NAMESPACE)); + } + + @Test + void testFilterByModificationTimeWhenMetadataIsEmpty() { + assertTrue(ObjectStoreFilter.filterByModificationTime(Map.of(), BLOBNAME, LocalDateTime.now())); + } + + @Test + void testFilterByModificationTimeWhenModificationTimeIsBeforeTheMetadataTime() { + assertFalse(ObjectStoreFilter.filterByModificationTime(buildMetadata(), BLOBNAME, LocalDateTime.now() + .minusMinutes(20))); + } + + @Test + void testFilterByModificationTimeWhenModificationTimeIsAfterTheMetadataTime() { + assertTrue(ObjectStoreFilter.filterByModificationTime(buildMetadata(), BLOBNAME, LocalDateTime.now() + .plusMinutes(20))); + } + + private Map buildMetadata() { + return Map.of(Constants.FILE_ENTRY_SPACE.toLowerCase(), SPACE_ID, Constants.FILE_ENTRY_NAMESPACE.toLowerCase(), NAMESPACE, + Constants.FILE_ENTRY_MODIFIED.toLowerCase(), String.valueOf(Instant.now() + .toEpochMilli())); + } +} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapperTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapperTest.java new file mode 100644 index 0000000000..d0cd9a23c3 --- /dev/null +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreMapperTest.java @@ -0,0 +1,52 @@ +package org.cloudfoundry.multiapps.controller.persistence.util; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Map; +import java.util.UUID; + +import org.cloudfoundry.multiapps.controller.persistence.Constants; +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class ObjectStoreMapperTest { + + private static final String SPACE_ID = "spaceId"; + private static final String NAMESPACE = "namespace"; + + @Test + void testCreateFileEntryMetadata() { + LocalDateTime modifiedTime = LocalDateTime.now(); + String modifiedTimeInstantString = Long.toString(modifiedTime + .atZone( + ZoneId.systemDefault()) + .toInstant() + .toEpochMilli()); + Map metadata = ObjectStoreMapper.createFileEntryMetadata(buildCreateFileEntry(modifiedTime)); + + assertEquals(SPACE_ID, metadata.get(Constants.FILE_ENTRY_SPACE.toLowerCase())); + assertEquals(modifiedTimeInstantString, metadata.get(Constants.FILE_ENTRY_MODIFIED.toLowerCase())); + assertEquals(NAMESPACE, metadata.get(Constants.FILE_ENTRY_NAMESPACE.toLowerCase())); + } + + @Test + void testCreateFileEntry() { + String id = UUID.randomUUID() + .toString(); + FileEntry fileEntry = ObjectStoreMapper.createFileEntry(SPACE_ID, id); + + assertEquals(SPACE_ID, fileEntry.getSpace()); + assertEquals(id, fileEntry.getId()); + } + + private FileEntry buildCreateFileEntry(LocalDateTime modifiedTime) { + return ImmutableFileEntry.builder() + .space(SPACE_ID) + .modified(modifiedTime) + .namespace(NAMESPACE) + .build(); + } +} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreUtilTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreUtilTest.java deleted file mode 100644 index b94ca600ca..0000000000 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreUtilTest.java +++ /dev/null @@ -1,117 +0,0 @@ -package org.cloudfoundry.multiapps.controller.persistence.util; - -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import org.cloudfoundry.multiapps.controller.persistence.Constants; -import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; -import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class ObjectStoreUtilTest { - - private static final String SPACE_ID = "spaceId"; - private static final String SPACE_ID_2 = "spaceId2"; - private static final String NAMESPACE = "namespace"; - private static final String BLOBNAME = "blobName"; - private static final String NAMESPACE_2 = "namespace2"; - - @Test - void testFilterBySpaceIdsWhenMetadataIsEmpty() { - assertFalse(ObjectStoreUtil.filterBySpaceIds(Map.of(), List.of())); - } - - @Test - void testFilterBySpaceIdsWhenTheMetadataContainsTheSpaceId() { - assertTrue(ObjectStoreUtil.filterBySpaceIds(buildMetadata(), List.of(SPACE_ID))); - } - - @Test - void testFilterBySpaceIdsWhenTheMetadataDoesNotContainTheSpaceId() { - assertFalse(ObjectStoreUtil.filterBySpaceIds(buildMetadata(), List.of(SPACE_ID_2))); - } - - @Test - void testFilterBySpaceAndNamespaceWhenMetadataIsEmpty() { - assertFalse(ObjectStoreUtil.filterBySpaceAndNamespace(Map.of(), SPACE_ID, NAMESPACE)); - } - - @Test - void testFilterBySpaceAndNamespaceWhenTheSpacesAreDifferent() { - assertFalse(ObjectStoreUtil.filterBySpaceAndNamespace(buildMetadata(), SPACE_ID_2, NAMESPACE)); - } - - @Test - void testFilterBySpaceAndNamespaceWhenTheNamespacesAreDifferent() { - assertFalse(ObjectStoreUtil.filterBySpaceAndNamespace(buildMetadata(), SPACE_ID, NAMESPACE_2)); - } - - @Test - void testFilterBySpaceAndNamespaceWhenAllMatch() { - assertTrue(ObjectStoreUtil.filterBySpaceAndNamespace(buildMetadata(), SPACE_ID, NAMESPACE)); - } - - @Test - void testFilterByModificationTimeWhenMetadataIsEmpty() { - assertTrue(ObjectStoreUtil.filterByModificationTime(Map.of(), BLOBNAME, LocalDateTime.now())); - } - - @Test - void testFilterByModificationTimeWhenModificationTimeIsBeforeTheMetadataTime() { - assertFalse(ObjectStoreUtil.filterByModificationTime(buildMetadata(), BLOBNAME, LocalDateTime.now() - .minusMinutes(20))); - } - - @Test - void testFilterByModificationTimeWhenModificationTimeIsAfterTheMetadataTime() { - assertTrue(ObjectStoreUtil.filterByModificationTime(buildMetadata(), BLOBNAME, LocalDateTime.now() - .plusMinutes(20))); - } - - @Test - void testCreateFileEntryMetadata() { - LocalDateTime modifiedTime = LocalDateTime.now(); - String modifiedTimeInstantString = Long.toString(modifiedTime - .atZone( - ZoneId.systemDefault()) - .toInstant() - .toEpochMilli()); - Map metadata = ObjectStoreUtil.createFileEntryMetadata(buildCreateFileEntry(modifiedTime)); - - assertEquals(SPACE_ID, metadata.get(Constants.FILE_ENTRY_SPACE.toLowerCase())); - assertEquals(modifiedTimeInstantString, metadata.get(Constants.FILE_ENTRY_MODIFIED.toLowerCase())); - assertEquals(NAMESPACE, metadata.get(Constants.FILE_ENTRY_NAMESPACE.toLowerCase())); - } - - @Test - void testCreateFileEntry() { - String id = UUID.randomUUID() - .toString(); - FileEntry fileEntry = ObjectStoreUtil.createFileEntry(SPACE_ID, id); - - assertEquals(SPACE_ID, fileEntry.getSpace()); - assertEquals(id, fileEntry.getId()); - } - - private FileEntry buildCreateFileEntry(LocalDateTime modifiedTime) { - return ImmutableFileEntry.builder() - .space(SPACE_ID) - .modified(modifiedTime) - .namespace(NAMESPACE) - .build(); - } - - private Map buildMetadata() { - return Map.of(Constants.FILE_ENTRY_SPACE.toLowerCase(), SPACE_ID, Constants.FILE_ENTRY_NAMESPACE.toLowerCase(), NAMESPACE, - Constants.FILE_ENTRY_MODIFIED.toLowerCase(), String.valueOf(Instant.now() - .toEpochMilli())); - } -} diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java index 45ca4b71f0..fadf4effb6 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java @@ -238,7 +238,7 @@ public FileStorage getObject() { @Override public Class getObjectType() { - return ObjectStoreFileStorage.class; + return FileStorage.class; } } \ No newline at end of file diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java index 5bd108715e..86c445e306 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java @@ -1,6 +1,5 @@ package org.cloudfoundry.multiapps.controller.web.configuration.service; -import com.google.cloud.storage.Storage; import com.google.common.base.Supplier; import org.cloudfoundry.multiapps.common.Nullable; import org.immutables.value.Value; @@ -31,8 +30,4 @@ public interface ObjectStoreServiceInfo { @Nullable String getHost(); - - @Nullable - Storage getGcpStorage(); - } diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java index c233caac67..6439250543 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java @@ -7,8 +7,6 @@ import java.util.Map; import java.util.stream.Stream; -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; import io.pivotal.cfenv.core.CfCredentials; import io.pivotal.cfenv.core.CfService; import org.cloudfoundry.multiapps.controller.web.Constants; @@ -32,8 +30,6 @@ class ObjectStoreServiceInfoCreatorTest { private static final String SAS_TOKEN_VALUE = "sas_token_value"; private static final String CONTAINER_NAME_VALUE = "container_name_value"; private static final String CONTAINER_URI_VALUE = "https://container.com:8080"; - private static final Storage STORAGE = LocalStorageHelper.getOptions() - .getService(); private ObjectStoreServiceInfoCreator objectStoreServiceInfoCreator; diff --git a/pom.xml b/pom.xml index 3d29899d6c..7445d807b4 100644 --- a/pom.xml +++ b/pom.xml @@ -57,7 +57,7 @@ 1.28.0 3.2.2 1.3.1 - 2.56.0 + 2.59.0 0.128.5 From 347917ef8ca693d6e6059fb606c8ac64d06f9b06 Mon Sep 17 00:00:00 2001 From: Yavor16 Date: Tue, 4 Nov 2025 11:21:27 +0200 Subject: [PATCH 08/12] Fix stef comments for renaming class --- .../ApplicationHealthCalculatorTest.java | 20 +++++++++---------- ...ava => JCloudsObjectStoreFileStorage.java} | 6 +++--- .../GcpObjectStoreFileStorageTest.java | 2 +- ...=> JCloudsObjectStoreFileStorageTest.java} | 4 ++-- .../ObjectStoreFileStorageFactoryBean.java | 6 +++--- ...ObjectStoreFileStorageFactoryBeanTest.java | 14 ++++++------- 6 files changed, 26 insertions(+), 26 deletions(-) rename multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/{ObjectStoreFileStorage.java => JCloudsObjectStoreFileStorage.java} (98%) rename multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/{ObjectStoreFileStorageTest.java => JCloudsObjectStoreFileStorageTest.java} (98%) diff --git a/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculatorTest.java b/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculatorTest.java index e067eb7e88..8a67a4bda3 100644 --- a/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculatorTest.java +++ b/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculatorTest.java @@ -1,9 +1,5 @@ package org.cloudfoundry.multiapps.controller.core.application.health; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - import org.cloudfoundry.multiapps.common.SLException; import org.cloudfoundry.multiapps.controller.client.util.ResilientOperationExecutor; import org.cloudfoundry.multiapps.controller.core.application.health.database.DatabaseWaitingLocksAnalyzer; @@ -11,7 +7,7 @@ import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseHealthService; import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseMonitoringService; -import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage; +import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; @@ -20,10 +16,14 @@ import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + class ApplicationHealthCalculatorTest { @Mock - private ObjectStoreFileStorage objectStoreFileStorage; + private JCloudsObjectStoreFileStorage jCloudsObjectStoreFileStorage; @Mock private ApplicationConfiguration applicationConfiguration; @Mock @@ -41,7 +41,7 @@ void setUp() throws Exception { .close(); Mockito.when(applicationConfiguration.isHealthCheckEnabled()) .thenReturn(true); - applicationHealthCalculator = new ApplicationHealthCalculatorMock(objectStoreFileStorage, + applicationHealthCalculator = new ApplicationHealthCalculatorMock(jCloudsObjectStoreFileStorage, applicationConfiguration, databaseHealthService, databaseMonitoringService, @@ -51,7 +51,7 @@ void setUp() throws Exception { @Test void testUpdateWithFailingObjectStore() { Mockito.doThrow(new SLException("Object store not working")) - .when(objectStoreFileStorage) + .when(jCloudsObjectStoreFileStorage) .testConnection(); applicationHealthCalculator.updateHealthStatus(); ResponseEntity applicationHealthResultResponseEntity = applicationHealthCalculator.calculateApplicationHealth(); @@ -117,12 +117,12 @@ void testSuccessfulUpdateWithMissingObjectStore() { } private static class ApplicationHealthCalculatorMock extends ApplicationHealthCalculator { - public ApplicationHealthCalculatorMock(ObjectStoreFileStorage objectStoreFileStorage, + public ApplicationHealthCalculatorMock(JCloudsObjectStoreFileStorage JCloudsObjectStoreFileStorage, ApplicationConfiguration applicationConfiguration, DatabaseHealthService databaseHealthService, DatabaseMonitoringService databaseMonitoringService, DatabaseWaitingLocksAnalyzer databaseWaitingLocksAnalyzer) { - super(objectStoreFileStorage, + super(JCloudsObjectStoreFileStorage, applicationConfiguration, databaseHealthService, databaseMonitoringService, diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java similarity index 98% rename from multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java rename to multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java index b402694ab0..ad1b6f81ec 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java @@ -30,16 +30,16 @@ import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; -public class ObjectStoreFileStorage implements FileStorage { +public class JCloudsObjectStoreFileStorage implements FileStorage { - private static final Logger LOGGER = LoggerFactory.getLogger(ObjectStoreFileStorage.class); + private static final Logger LOGGER = LoggerFactory.getLogger(JCloudsObjectStoreFileStorage.class); private static final int MAX_RETRIES_COUNT = 3; private static final long RETRY_BASE_WAIT_TIME_IN_MILLIS = 5000L; private final BlobStore blobStore; private final String container; - public ObjectStoreFileStorage(BlobStore blobStore, String container) { + public JCloudsObjectStoreFileStorage(BlobStore blobStore, String container) { this.blobStore = blobStore; this.container = container; } diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java index 177103fff2..cc4b0aa773 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java @@ -21,7 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; -class GcpObjectStoreFileStorageTest extends ObjectStoreFileStorageTest { +class GcpObjectStoreFileStorageTest extends JCloudsObjectStoreFileStorageTest { private Storage storage; diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java similarity index 98% rename from multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorageTest.java rename to multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java index f9477f7601..6356afd641 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java @@ -34,7 +34,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; -class ObjectStoreFileStorageTest { +class JCloudsObjectStoreFileStorageTest { protected static final String TEST_FILE_LOCATION = "src/test/resources/pexels-photo-401794.jpeg"; protected static final String SECOND_FILE_TEST_LOCATION = "src/test/resources/pexels-photo-463467.jpeg"; @@ -51,7 +51,7 @@ class ObjectStoreFileStorageTest { @BeforeEach protected void setUp() { createBlobStoreContext(); - fileStorage = new ObjectStoreFileStorage(blobStoreContext.getBlobStore(), CONTAINER) { + fileStorage = new JCloudsObjectStoreFileStorage(blobStoreContext.getBlobStore(), CONTAINER) { @Override protected long getRetryWaitTime() { return 1; diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java index fadf4effb6..0001e13be1 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java @@ -18,7 +18,7 @@ import org.cloudfoundry.multiapps.controller.core.util.UriUtil; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; -import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage; +import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder; import org.cloudfoundry.multiapps.controller.web.Constants; import org.cloudfoundry.multiapps.controller.web.Messages; @@ -222,8 +222,8 @@ private void resolveContextEndpoint(ObjectStoreServiceInfo serviceInfo, ContextB } } - protected ObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) { - return new ObjectStoreFileStorage(context.getBlobStore(), objectStoreServiceInfo.getContainer()); + protected JCloudsObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) { + return new JCloudsObjectStoreFileStorage(context.getBlobStore(), objectStoreServiceInfo.getContainer()); } protected GcpObjectStoreFileStorage createGcpFileStorage() { diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java index a1b8b4d809..5b5617b7a6 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java @@ -10,7 +10,7 @@ import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; -import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage; +import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder; import org.cloudfoundry.multiapps.controller.web.Constants; import org.cloudfoundry.multiapps.controller.web.Messages; @@ -49,7 +49,7 @@ class ObjectStoreFileStorageFactoryBeanTest { @Mock private ApplicationConfiguration applicationConfiguration; @Mock - private ObjectStoreFileStorage objectStoreFileStorage; + private JCloudsObjectStoreFileStorage jCloudsObjectStoreFileStorage; @Mock private GcpObjectStoreFileStorage gcpObjectStoreFileStorage; @@ -90,7 +90,7 @@ void testObjectStoreCreationWhenEnvIsValid() { assertNotNull(createdObjectStoreFileStorage); verify(spy, never()) .createObjectStoreFromFirstReachableProvider(anyMap(), anyList()); - verify(objectStoreFileStorage, times(1)) + verify(jCloudsObjectStoreFileStorage, times(1)) .testConnection(); } @@ -113,7 +113,7 @@ void testObjectStoreCreationWhenEnvIsInvalid() { void testObjectStoreCreationWhenEnvProviderFailsToConnect() { mockCfService(); when(applicationConfiguration.getObjectStoreClientType()).thenReturn(Constants.AWS); - doThrow(new IllegalStateException("Cannot create object store")).when(objectStoreFileStorage) + doThrow(new IllegalStateException("Cannot create object store")).when(jCloudsObjectStoreFileStorage) .testConnection(); Exception exception = assertThrows(IllegalStateException.class, () -> objectStoreFileStorageFactoryBean.afterPropertiesSet()); @@ -123,7 +123,7 @@ void testObjectStoreCreationWhenEnvProviderFailsToConnect() { @Test void testObjectStoreCreationWithoutValidServiceInstance() { mockCfService(); - doThrow(new IllegalStateException("Cannot create object store")).when(objectStoreFileStorage) + doThrow(new IllegalStateException("Cannot create object store")).when(jCloudsObjectStoreFileStorage) .testConnection(); doThrow(new IllegalStateException("Cannot create object store")).when(gcpObjectStoreFileStorage) .testConnection(); @@ -155,8 +155,8 @@ public ObjectStoreFileStorageFactoryBeanMock(String serviceName, EnvironmentServ } @Override - protected ObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) { - return ObjectStoreFileStorageFactoryBeanTest.this.objectStoreFileStorage; + protected JCloudsObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) { + return ObjectStoreFileStorageFactoryBeanTest.this.jCloudsObjectStoreFileStorage; } @Override From ce37f433f1d30976419027815108d53f90efd8d4 Mon Sep 17 00:00:00 2001 From: Yavor16 Date: Tue, 4 Nov 2025 12:29:50 +0200 Subject: [PATCH 09/12] Remove magic number --- .../persistence/services/JCloudsObjectStoreFileStorage.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java index ad1b6f81ec..2e5ec53acb 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java @@ -57,7 +57,7 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage .userMetadata(ObjectStoreMapper.createFileEntryMetadata(fileEntry)) .build(); try { - putBlobWithRetries(blob, 3); + putBlobWithRetries(blob, MAX_RETRIES_COUNT); LOGGER.debug(MessageFormat.format(Messages.STORED_FILE_0_WITH_SIZE_1, fileEntry.getId(), fileSize)); } catch (ContainerNotFoundException e) { throw new FileStorageException(MessageFormat.format(Messages.FILE_UPLOAD_FAILED, fileEntry.getName(), @@ -146,7 +146,7 @@ private Blob getBlobWithRetriesWithOffset(FileEntry fileEntry, int retries, long } private Payload getBlobPayload(FileEntry fileEntry) throws FileStorageException { - Blob blob = getBlobWithRetries(fileEntry, 3); + Blob blob = getBlobWithRetries(fileEntry, MAX_RETRIES_COUNT); if (blob == null) { throw new FileStorageException(MessageFormat.format(Messages.FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST, fileEntry.getId(), fileEntry.getSpace())); From f707eca512499143e790e3ec4637292b85ec2f91 Mon Sep 17 00:00:00 2001 From: Yavor16 Date: Tue, 4 Nov 2025 14:56:52 +0200 Subject: [PATCH 10/12] Fix commnets --- .../src/main/java/module-info.java | 6 +++--- .../services/GcpObjectStoreFileStorage.java | 12 ++++++------ pom.xml | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index 557d727bc3..1b10271ef5 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -30,6 +30,8 @@ requires aliyun.sdk.oss; requires com.fasterxml.jackson.annotation; requires com.fasterxml.jackson.databind; + requires com.google.auth; + requires com.google.auth.oauth2; requires com.google.auto.service; requires com.google.common; requires com.google.guice; @@ -50,6 +52,7 @@ requires org.cloudfoundry.multiapps.common; requires org.eclipse.persistence.core; requires org.slf4j; + requires org.threeten.bp; requires spring.context; requires spring.core; @@ -57,7 +60,4 @@ requires static org.immutables.value; requires jakarta.xml.bind; requires org.bouncycastle.fips.pkix; - requires com.google.auth.oauth2; - requires com.google.auth; - requires org.threeten.bp; } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java index 92006d87c2..e184d905fd 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java @@ -40,9 +40,9 @@ public class GcpObjectStoreFileStorage implements FileStorage { private static final String BUCKET = "bucket"; private static final int OBJECTSTORE_MAX_ATTEMPTS_CONFIG = 6; private static final double OBJECTSTORE_RETRY_DELAY_MULTIPLIER_CONFIG = 2.0; - private static final Duration OBJECTSTORE_TOTAL_TIMEOUT_CONFIG = Duration.ofMinutes(10); - private static final Duration OBJECTSTORE_MAX_RETRY_DELAY_CONFIG = Duration.ofSeconds(10); - private static final Duration OBJECTSTORE_INITIAL_RETRY_DELAY_CONFIG = Duration.ofMillis(250); + private static final Duration OBJECTSTORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES = Duration.ofMinutes(10); + private static final Duration OBJECTSTORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS = Duration.ofSeconds(10); + private static final Duration OBJECTSTORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS = Duration.ofMillis(250); private static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData"; public GcpObjectStoreFileStorage(Map credentials) { @@ -57,9 +57,9 @@ protected Storage createObjectStoreStorage(Map credentials) { .setRetrySettings( RetrySettings.newBuilder() .setMaxAttempts(OBJECTSTORE_MAX_ATTEMPTS_CONFIG) - .setTotalTimeout(OBJECTSTORE_TOTAL_TIMEOUT_CONFIG) - .setMaxRetryDelay(OBJECTSTORE_MAX_RETRY_DELAY_CONFIG) - .setInitialRetryDelay(OBJECTSTORE_INITIAL_RETRY_DELAY_CONFIG) + .setTotalTimeout(OBJECTSTORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES) + .setMaxRetryDelay(OBJECTSTORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS) + .setInitialRetryDelay(OBJECTSTORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS) .setRetryDelayMultiplier(OBJECTSTORE_RETRY_DELAY_MULTIPLIER_CONFIG) .build()) .build() diff --git a/pom.xml b/pom.xml index 7445d807b4..7ff0a2110a 100644 --- a/pom.xml +++ b/pom.xml @@ -58,7 +58,7 @@ 3.2.2 1.3.1 2.59.0 - 0.128.5 + 0.128.7 multiapps-controller-client From 1fa2483c64a7f6ea034392ed0060641827f14ffc Mon Sep 17 00:00:00 2001 From: Yavor16 Date: Tue, 4 Nov 2025 15:11:38 +0200 Subject: [PATCH 11/12] Fix commnets --- .../src/main/java/module-info.java | 1 - .../services/GcpObjectStoreFileStorage.java | 22 +++++++++---------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index 1b10271ef5..875fb82e2d 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -52,7 +52,6 @@ requires org.cloudfoundry.multiapps.common; requires org.eclipse.persistence.core; requires org.slf4j; - requires org.threeten.bp; requires spring.context; requires spring.core; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java index e184d905fd..ab81b1750f 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java @@ -5,6 +5,7 @@ import java.io.InputStream; import java.nio.channels.Channels; import java.text.MessageFormat; +import java.time.Duration; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Base64; @@ -31,18 +32,17 @@ import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; import org.springframework.http.MediaType; -import org.threeten.bp.Duration; public class GcpObjectStoreFileStorage implements FileStorage { private final String bucketName; private final Storage storage; private static final String BUCKET = "bucket"; - private static final int OBJECTSTORE_MAX_ATTEMPTS_CONFIG = 6; - private static final double OBJECTSTORE_RETRY_DELAY_MULTIPLIER_CONFIG = 2.0; - private static final Duration OBJECTSTORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES = Duration.ofMinutes(10); - private static final Duration OBJECTSTORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS = Duration.ofSeconds(10); - private static final Duration OBJECTSTORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS = Duration.ofMillis(250); + private static final int OBJECT_STORE_MAX_ATTEMPTS_CONFIG = 6; + private static final double OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG = 2.0; + private static final Duration OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES = Duration.ofMinutes(10); + private static final Duration OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS = Duration.ofSeconds(10); + private static final Duration OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS = Duration.ofMillis(250); private static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData"; public GcpObjectStoreFileStorage(Map credentials) { @@ -56,11 +56,11 @@ protected Storage createObjectStoreStorage(Map credentials) { .setStorageRetryStrategy(StorageRetryStrategy.getDefaultStorageRetryStrategy()) .setRetrySettings( RetrySettings.newBuilder() - .setMaxAttempts(OBJECTSTORE_MAX_ATTEMPTS_CONFIG) - .setTotalTimeout(OBJECTSTORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES) - .setMaxRetryDelay(OBJECTSTORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS) - .setInitialRetryDelay(OBJECTSTORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS) - .setRetryDelayMultiplier(OBJECTSTORE_RETRY_DELAY_MULTIPLIER_CONFIG) + .setMaxAttempts(OBJECT_STORE_MAX_ATTEMPTS_CONFIG) + .setTotalTimeoutDuration(OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES) + .setMaxRetryDelayDuration(OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS) + .setInitialRetryDelayDuration(OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS) + .setRetryDelayMultiplier(OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG) .build()) .build() .getService(); From edaf6b154211b667a9498bfea02244f07a725d16 Mon Sep 17 00:00:00 2001 From: Yavor16 Date: Mon, 10 Nov 2025 15:22:53 +0200 Subject: [PATCH 12/12] Attempts for fix --- .../multiapps/controller/process/Messages.java | 2 +- .../process/stream/LazyArchiveInputStream.java | 9 ++++----- .../stream/LazyArchiveInputStreamTest.java | 16 +++++++--------- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java index e2c264a8d0..de999943a7 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java @@ -327,7 +327,7 @@ public class Messages { public static final String DELETED_FILE_ENTRIES_0 = "Deleted file entries: {0}"; public static final String APPLICATION_WITH_NAME_0_SAVED_TO_1 = "Application with name \"{0}\" saved to \"{1}\""; public static final String CLOSING_STREAM_FOR_PART_0 = "Closing stream for part: {0}"; - public static final String CLOSING_STREAM_FOR_PART_STREAM_FINISHED_0 = "Closing stream for part, stream finished: {0}"; + public static final String REACHED_THE_END_OF_THE_INPUTSTREAM = "Reached the end of the InputStream"; public static final String CLOSING_LAST_STREAM_FOR_PART_0 = "Closing the last stream, part: {0}"; public static final String SCALING_UP_OLD_APPLICATION = "Scaling up old application: \"{0}\" to {1} instances"; public static final String FILE_WITH_ID_0_WAS_DELETED = "File with id \"{0}\" was deleted"; diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/stream/LazyArchiveInputStream.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/stream/LazyArchiveInputStream.java index 89daad8fc3..ffc2f886ed 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/stream/LazyArchiveInputStream.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/stream/LazyArchiveInputStream.java @@ -28,7 +28,6 @@ public class LazyArchiveInputStream extends InputStream { private final long archiveSize; private final AtomicInteger totalBytesRead; private final AtomicInteger partIndex; - private InputStream currentInputStream; public LazyArchiveInputStream(FileService fileService, List archiveFileEntries, StepLogger stepLogger, long archiveSize) { @@ -52,8 +51,7 @@ public synchronized int read() throws IOException { currentInputStream = openBufferedInputStream(archiveFileEntries.get(partIndex.incrementAndGet())); c = currentInputStream.read(); } else if (c == -1) { - LOGGER.info(MessageFormat.format(Messages.CLOSING_STREAM_FOR_PART_STREAM_FINISHED_0, partIndex)); - IOUtils.closeQuietly(currentInputStream, e -> LOGGER.warn(e.getMessage(), e)); + LOGGER.info(MessageFormat.format(Messages.REACHED_THE_END_OF_THE_INPUTSTREAM, partIndex)); } if (c >= 0) { totalBytesRead.incrementAndGet(); @@ -73,8 +71,7 @@ public synchronized int read(byte[] b, int off, int len) throws IOException { currentInputStream = openBufferedInputStream(archiveFileEntries.get(partIndex.incrementAndGet())); bytesRead = currentInputStream.read(b, off, len); } else if (bytesRead == -1) { - LOGGER.info(MessageFormat.format(Messages.CLOSING_STREAM_FOR_PART_STREAM_FINISHED_0, partIndex)); - IOUtils.closeQuietly(currentInputStream, e -> LOGGER.warn(e.getMessage(), e)); + LOGGER.info(MessageFormat.format(Messages.REACHED_THE_END_OF_THE_INPUTSTREAM, partIndex)); } if (bytesRead > 0) { totalBytesRead.addAndGet(bytesRead); @@ -107,6 +104,8 @@ private BufferedInputStream openBufferedInputStream(FileEntry archiveFileEntry) try { stepLogger.debug(Messages.OPENING_A_NEW_INPUT_STREAM_FOR_FILE_WITH_ID_0_AND_NAME_1, archiveFileEntry.getId(), archiveFileEntry.getName()); + LOGGER.info(MessageFormat.format(Messages.OPENING_A_NEW_INPUT_STREAM_FOR_FILE_WITH_ID_0_AND_NAME_1, archiveFileEntry.getId(), + archiveFileEntry.getName())); InputStream inputStream = fileService.openInputStream(archiveFileEntry.getSpace(), archiveFileEntry.getId()); return new BufferedInputStream(inputStream, BUFFERED_SIZE); } catch (FileStorageException e) { diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/stream/LazyArchiveInputStreamTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/stream/LazyArchiveInputStreamTest.java index 8216949aa5..b366865068 100644 --- a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/stream/LazyArchiveInputStreamTest.java +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/stream/LazyArchiveInputStreamTest.java @@ -1,12 +1,5 @@ package org.cloudfoundry.multiapps.controller.process.stream; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; @@ -22,6 +15,13 @@ import org.mockito.Mock; import org.mockito.MockitoAnnotations; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + class LazyArchiveInputStreamTest { private static final String FILE_ID_1 = "file-1"; @@ -71,7 +71,6 @@ void readWhenEOFReached() throws IOException, FileStorageException { assertEquals(-1, read); assertEquals(0, lazyArchiveInputStream.available()); verify(firstPartInputStream).close(); - verify(secondPartInputStream).close(); } @Test @@ -104,7 +103,6 @@ void readBufferedWhenEOFReached() throws IOException, FileStorageException { assertEquals(-1, read); assertEquals(0, lazyArchiveInputStream.available()); verify(firstPartInputStream).close(); - verify(secondPartInputStream).close(); } private void prepareStream(int archiveSize) {