diff --git a/multiapps-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/v1/FilesApi.java b/multiapps-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/v1/FilesApi.java index beb6dc1a84..d6b813166c 100644 --- a/multiapps-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/v1/FilesApi.java +++ b/multiapps-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/v1/FilesApi.java @@ -1,9 +1,13 @@ package org.cloudfoundry.multiapps.controller.api.v1; import java.util.List; - +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import io.swagger.annotations.Authorization; import jakarta.inject.Inject; - import org.cloudfoundry.multiapps.controller.api.Constants.Endpoints; import org.cloudfoundry.multiapps.controller.api.Constants.PathVariables; import org.cloudfoundry.multiapps.controller.api.Constants.RequestVariables; @@ -23,13 +27,6 @@ import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartHttpServletRequest; -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import io.swagger.annotations.ApiParam; -import io.swagger.annotations.ApiResponse; -import io.swagger.annotations.ApiResponses; -import io.swagger.annotations.Authorization; - @Api @RestController @RequestMapping(Resources.FILES) @@ -45,8 +42,8 @@ public class FilesApi { }) }, tags = {}) @ApiResponses(value = { @ApiResponse(code = 200, message = "OK", response = FileMetadata.class, responseContainer = "List") }) public ResponseEntity> - getFiles(@ApiParam(value = "GUID of space with mtas") @PathVariable(PathVariables.SPACE_GUID) String spaceGuid, - @ApiParam(value = "Filter mtas by namespace") @RequestParam(name = RequestVariables.NAMESPACE, required = false) String namespace) { + getFiles(@ApiParam(value = "GUID of space with mtas") @PathVariable(PathVariables.SPACE_GUID) String spaceGuid, + @ApiParam(value = "Filter mtas by namespace") @RequestParam(name = RequestVariables.NAMESPACE, required = false) String namespace) { return delegate.getFiles(spaceGuid, namespace); } @@ -57,9 +54,9 @@ public class FilesApi { }) }, tags = {}) @ApiResponses(value = { @ApiResponse(code = 201, message = "Created", response = FileMetadata.class) }) public ResponseEntity - uploadFile(MultipartHttpServletRequest request, - @ApiParam(value = "GUID of space you wish to deploy in") @PathVariable(PathVariables.SPACE_GUID) String spaceGuid, - @ApiParam(value = "file namespace") @RequestParam(name = RequestVariables.NAMESPACE, required = false) String namespace) { + uploadFile(MultipartHttpServletRequest request, + @ApiParam(value = "GUID of space you wish to deploy in") @PathVariable(PathVariables.SPACE_GUID) String spaceGuid, + @ApiParam(value = "file namespace") @RequestParam(name = RequestVariables.NAMESPACE, required = false) String namespace) { return delegate.uploadFile(request, spaceGuid, namespace); } @@ -70,9 +67,9 @@ public class FilesApi { }) }, tags = {}) @ApiResponses(value = { @ApiResponse(code = 202, message = "Accepted") }) public ResponseEntity - startUploadFromUrl(@ApiParam(value = "GUID of space you wish to deploy in") @PathVariable(PathVariables.SPACE_GUID) String spaceGuid, - @ApiParam(value = "file namespace") @RequestParam(name = RequestVariables.NAMESPACE, required = false) String namespace, - @ApiParam(value = "URL reference to a remote file") @RequestBody FileUrl fileUrl) { + startUploadFromUrl(@ApiParam(value = "GUID of space you wish to deploy in") @PathVariable(PathVariables.SPACE_GUID) String spaceGuid, + @ApiParam(value = "file namespace") @RequestParam(name = RequestVariables.NAMESPACE, required = false) String namespace, + @ApiParam(value = "URL reference to a remote file") @RequestBody FileUrl fileUrl) { return delegate.startUploadFromUrl(spaceGuid, namespace, fileUrl); } @@ -84,9 +81,9 @@ public class FilesApi { @ApiResponses(value = { @ApiResponse(code = 200, message = "OK"), @ApiResponse(code = 201, message = "Created", response = AsyncUploadResult.class) }) public ResponseEntity - getUploadFromUrlJob(@ApiParam(value = "GUID of space you wish to deploy in") @PathVariable(PathVariables.SPACE_GUID) String spaceGuid, - @ApiParam(value = "file namespace") @RequestParam(name = RequestVariables.NAMESPACE, required = false) String namespace, - @ApiParam(value = "ID of the upload job") @PathVariable(PathVariables.JOB_ID) String jobId) { + getUploadFromUrlJob(@ApiParam(value = "GUID of space you wish to deploy in") @PathVariable(PathVariables.SPACE_GUID) String spaceGuid, + @ApiParam(value = "file namespace") @RequestParam(name = RequestVariables.NAMESPACE, required = false) String namespace, + @ApiParam(value = "ID of the upload job") @PathVariable(PathVariables.JOB_ID) String jobId) { return delegate.getUploadFromUrlJob(spaceGuid, namespace, jobId); } diff --git a/multiapps-controller-persistence/pom.xml b/multiapps-controller-persistence/pom.xml index eade772907..ba4f2b1bd4 100644 --- a/multiapps-controller-persistence/pom.xml +++ b/multiapps-controller-persistence/pom.xml @@ -100,12 +100,12 @@ test - org.apache.jclouds.provider - aws-s3 + software.amazon.awssdk + s3 - org.apache.jclouds.provider - azureblob + software.amazon.awssdk + url-connection-client org.apache.jclouds diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index 8c3c8226e0..a86e1a244d 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -60,7 +60,16 @@ requires spring.beans; requires spring.context; requires spring.core; - + requires software.amazon.awssdk.services.s3; + requires software.amazon.awssdk.core; + requires software.amazon.awssdk.awscore; + requires software.amazon.awssdk.regions; + requires software.amazon.awssdk.utils; + requires software.amazon.awssdk.auth; + requires software.amazon.awssdk.http; + requires software.amazon.awssdk.http.urlconnection; + requires software.amazon.awssdk.retries; + requires software.amazon.awssdk.retries.api; requires static java.compiler; requires static org.immutables.value; } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java index 4a2daa7c71..986e97632e 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java @@ -48,6 +48,8 @@ public final class Messages { public static final String APPLICATION_SHUTDOWN_WITH_APPLICATION_INSTANCE_ID_ALREADY_EXIST = "Application shutdown application instance ID \"{0}\" already exist"; public static final String SECRET_TOKEN_WITH_ID_NOT_EXIST = "Secret token with ID \"{0}\" does not exist"; public static final String DATABASE_HEALTH_CHECK_FAILED = "Database health check failed"; + public static final String OBJECT_STORE_BUCKET_NOT_FOUND = "Object store bucket \"{0}\" not found"; + public static final String UPLOAD_OF_FILE_WITH_NAMESPACE_FAILED = "Upload of file: \"{0}\" with namespace: \"{1}\" failed"; // ERROR log messages: public static final String UPLOAD_STREAM_FAILED_TO_CLOSE = "Cannot close file upload stream"; @@ -67,6 +69,7 @@ public final class Messages { // DEBUG log messages: public static final String STORED_FILE_0 = "Stored file: \"{0}\""; public static final String STORED_FILE_0_WITH_SIZE_1 = "Stored file \"{0}\" with size {1}"; + public static final String STORED_FILE_0_WITH_SIZE_1_LOG = "Stored file \"{}\" with size {}"; public static final String DELETED_0_FILES_WITH_SPACEIDS_1 = "Deleted {0} files with space ids \"{1}\"."; public static final String DELETED_0_FILES_WITH_SPACE_1_AND_NAMESPACE_2 = "Deleted {0} files with space \"{1}\" and namespace \"{2}\"."; public static final String DELETED_0_FILES_MODIFIED_BEFORE_1 = "Deleted {0} files modified before \"{1}\"."; @@ -77,6 +80,8 @@ public final class Messages { public static final String RETRIEVED_SECRET_TOKEN_WITH_ID_0_FOR_PROCESS_WITH_ID_1 = "Retrieved secret token with id \"{0}\" for process with id \"{1}\""; public static final String DELETED_0_SECRET_TOKENS_FOR_PROCESS_WITH_ID_1 = "Deleted \"{0}\" secret tokens for process with id \"{1}\""; public static final String DELETED_0_SECRET_TOKENS_WITH_EXPIRATION_DATE_1 = "Deleted secret tokens \"{0}\" with an expiration date \"{1}\""; + public static final String FAILED_TO_DELETE_FILE_0_IN_OBJECT_STORE_REASON_1 = "Failed to delete file \"{}\" in object store. Reason: {}"; + public static final String S3_UPLOAD_FAILED_FILE_0_SIZE_1 = "S3 upload failed for file \"{0}\" (size={1}). Root cause chain: {2}"; protected Messages() { } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java new file mode 100644 index 0000000000..9a19456d5a --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java @@ -0,0 +1,316 @@ +package org.cloudfoundry.multiapps.controller.persistence.services; + +import org.cloudfoundry.multiapps.controller.persistence.Messages; +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.MediaType; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.retries.StandardRetryStrategy; +import software.amazon.awssdk.retries.api.BackoffStrategy; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Error; +import software.amazon.awssdk.services.s3.model.S3Object; + +import java.io.BufferedInputStream; +import java.io.InputStream; +import java.net.URI; +import java.text.MessageFormat; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.BiPredicate; +import java.util.function.Consumer; + +public class AwsS3ObjectStoreFileStorage extends ObjectStoreFileStorage { + + private static final Logger LOGGER = LoggerFactory.getLogger(AwsS3ObjectStoreFileStorage.class); + private static final String HTTPS_SCHEME = "https://"; + private static final int DELETE_BATCH_SIZE = 1000; + + private final S3Client s3Client; + private final String bucketName; + + public AwsS3ObjectStoreFileStorage(Map credentials) { + this.bucketName = (String) credentials.get(CredentialKeys.BUCKET); + this.s3Client = createS3Client(credentials); + } + + protected S3Client createS3Client(Map credentials) { + AwsBasicCredentials awsCredentials = AwsBasicCredentials.create((String) credentials.get(CredentialKeys.ACCESS_KEY_ID), + (String) credentials.get(CredentialKeys.SECRET_ACCESS_KEY)); + S3ClientBuilder builder = S3Client.builder() + .credentialsProvider(StaticCredentialsProvider.create(awsCredentials)) + .overrideConfiguration(buildClientOverrideConfig()) + .httpClientBuilder(UrlConnectionHttpClient.builder() + .socketTimeout( + ObjectStoreConstants.AWS_OBJECT_STORE_SOCKET_TIMEOUT_CONFIG_IN_MINUTES) + .connectionTimeout( + ObjectStoreConstants.AWS_OBJECT_STORE_CONNECTION_TIMEOUT_CONFIG_IN_SECONDS)); + builder.endpointOverride(URI.create(HTTPS_SCHEME + credentials.get(CredentialKeys.HOST))); + builder.region(software.amazon.awssdk.regions.Region.of((String) credentials.get(CredentialKeys.REGION))); + return builder.build(); + } + + protected ClientOverrideConfiguration buildClientOverrideConfig() { + StandardRetryStrategy retryStrategy = StandardRetryStrategy.builder() + .maxAttempts(ObjectStoreConstants.OBJECT_STORE_MAX_ATTEMPTS_CONFIG) + .backoffStrategy(BackoffStrategy.exponentialDelayHalfJitter( + ObjectStoreConstants.OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS, + ObjectStoreConstants.OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS)) + .build(); + return ClientOverrideConfiguration.builder() + .retryStrategy(retryStrategy) + .apiCallTimeout(ObjectStoreConstants.AWS_OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES) + .apiCallAttemptTimeout(ObjectStoreConstants.OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES) + .build(); + } + + @Override + public void addFile(FileEntry fileEntry, InputStream content) throws FileStorageException { + long fileSize = fileEntry.getSize() + .longValue(); + PutObjectRequest request = PutObjectRequest.builder() + .bucket(bucketName) + .key(fileEntry.getId()) + .contentType(MediaType.APPLICATION_OCTET_STREAM_VALUE) + .contentDisposition(fileEntry.getName()) + .metadata(ObjectStoreMapper.createFileEntryMetadata(fileEntry)) + .build(); + try { + InputStream markableContent = content.markSupported() ? content : new BufferedInputStream(content); + s3Client.putObject(request, RequestBody.fromInputStream(markableContent, fileSize)); + LOGGER.debug(Messages.STORED_FILE_0_WITH_SIZE_1_LOG, fileEntry.getId(), fileSize); + } catch (Exception e) { + LOGGER.error(MessageFormat.format(Messages.S3_UPLOAD_FAILED_FILE_0_SIZE_1, fileEntry.getName(), fileSize, e)); + throw new FileStorageException(MessageFormat.format(Messages.UPLOAD_OF_FILE_WITH_NAMESPACE_FAILED, fileEntry.getName(), + fileEntry.getNamespace()), e); + } + } + + @Override + public List getFileEntriesWithoutContent(List fileEntries) { + Set existingKeys = new HashSet<>(listAllObjectKeys()); + return fileEntries.stream() + .filter(fileEntry -> !existingKeys.contains(fileEntry.getId())) + .toList(); + } + + private List listAllObjectKeys() { + List keys = new ArrayList<>(); + forEachPage(page -> page.contents() + .stream() + .map(S3Object::key) + .forEach(keys::add)); + return keys; + } + + private void forEachPage(Consumer pageConsumer) { + ListObjectsV2Request request = ListObjectsV2Request.builder() + .bucket(bucketName) + .build(); + ListObjectsV2Response response; + do { + response = s3Client.listObjectsV2(request); + pageConsumer.accept(response); + request = request.toBuilder() + .continuationToken(response.nextContinuationToken()) + .build(); + } while (Boolean.TRUE.equals(response.isTruncated())); + } + + @Override + protected boolean existsInObjectStore(FileEntry fileEntry) { + try { + s3Client.headObject(HeadObjectRequest.builder() + .bucket(bucketName) + .key(fileEntry.getId()) + .build()); + return true; + } catch (NoSuchKeyException _) { + return false; + } + } + + @Override + public void deleteFile(String id, String space) { + s3Client.deleteObject(DeleteObjectRequest.builder() + .bucket(bucketName) + .key(id) + .build()); + } + + @Override + public void deleteFilesBySpaceIds(List spaceIds) { + deleteByFilterAndCount((key, metadata) -> ObjectStoreFilter.filterBySpaceIds(metadata, spaceIds)); + } + + @Override + public void deleteFilesBySpaceAndNamespace(String space, String namespace) { + deleteByFilterAndCount((key, metadata) -> ObjectStoreFilter.filterBySpaceAndNamespace(metadata, space, namespace)); + } + + @Override + public int deleteFilesModifiedBefore(LocalDateTime modificationTime) { + return deleteByFilterAndCount((key, metadata) -> ObjectStoreFilter.filterByModificationTime(metadata, key, modificationTime)); + } + + private int deleteByFilterAndCount(BiPredicate> predicate) { + List toDelete = new ArrayList<>(); + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + forEachPage(page -> toDelete.addAll(collectMatchingKeys(page, predicate, executor))); + } + batchDelete(toDelete); + return toDelete.size(); + } + + private List collectMatchingKeys(ListObjectsV2Response page, BiPredicate> predicate, + ExecutorService executor) { + return page.contents() + .stream() + .map(obj -> CompletableFuture.supplyAsync(() -> fetchMetadataAndFilter(obj.key(), predicate), executor)) + .toList() + .stream() + .map(CompletableFuture::join) + .filter(Objects::nonNull) + .toList(); + } + + private String fetchMetadataAndFilter(String key, BiPredicate> predicate) { + try { + HeadObjectResponse response = s3Client.headObject(HeadObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build()); + Map metadata = response.metadata() != null ? response.metadata() : Map.of(); + return predicate.test(key, metadata) ? key : null; + } catch (NoSuchKeyException _) { + return null; + } + } + + private void batchDelete(List keys) { + for (int i = 0; i < keys.size(); i += DELETE_BATCH_SIZE) { + List identifiers = keys.subList(i, Math.min(i + DELETE_BATCH_SIZE, keys.size())) + .stream() + .map(k -> ObjectIdentifier.builder() + .key(k) + .build()) + .toList(); + DeleteObjectsResponse response = s3Client.deleteObjects(DeleteObjectsRequest.builder() + .bucket(bucketName) + .delete(d -> d.objects(identifiers)) + .build()); + for (S3Error error : response.errors()) { + LOGGER.warn(Messages.FAILED_TO_DELETE_FILE_0_IN_OBJECT_STORE_REASON_1, error.key(), error.message()); + } + } + } + + @Override + public T processFileContent(String space, String id, FileContentProcessor fileContentProcessor) throws FileStorageException { + GetObjectRequest request = GetObjectRequest.builder() + .bucket(bucketName) + .key(id) + .build(); + try (ResponseInputStream stream = getObjectStream(request, id, space)) { + return fileContentProcessor.process(stream); + } catch (Exception e) { + throw new FileStorageException(e); + } + } + + @Override + public InputStream openInputStream(String space, String id) throws FileStorageException { + GetObjectRequest request = GetObjectRequest.builder() + .bucket(bucketName) + .key(id) + .build(); + return getObjectStream(request, id, space); + } + + private ResponseInputStream getObjectStream(GetObjectRequest request, String id, + String space) throws FileStorageException { + try { + return s3Client.getObject(request); + } catch (NoSuchKeyException _) { + throw new FileStorageException(MessageFormat.format(Messages.FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST, id, space)); + } + } + + @Override + public void testConnection() { + s3Client.listObjectsV2(ListObjectsV2Request.builder() + .bucket(bucketName) + .maxKeys(1) + .build()); + } + + @Override + public void deleteFilesByIds(List fileIds) { + if (fileIds.isEmpty()) { + return; + } + batchDelete(fileIds); + } + + @Override + public T processArchiveEntryContent(FileContentToProcess fileContentToProcess, FileContentProcessor fileContentProcessor) + throws FileStorageException { + String id = fileContentToProcess.getGuid(); + String space = fileContentToProcess.getSpaceGuid(); + GetObjectRequest request = GetObjectRequest.builder() + .bucket(bucketName) + .key(id) + .range("bytes=" + fileContentToProcess.getStartOffset() + "-" + + fileContentToProcess.getEndOffset()) + .build(); + try (ResponseInputStream stream = getObjectStream(request, id, space)) { + return fileContentProcessor.process(stream); + } catch (Exception e) { + throw new FileStorageException(e); + } + } + + @Override + public void destroy() { + super.destroy(); + s3Client.close(); + } + + private static final class CredentialKeys { + static final String ACCESS_KEY_ID = "access_key_id"; + static final String SECRET_ACCESS_KEY = "secret_access_key"; + static final String BUCKET = "bucket"; + static final String HOST = "host"; + static final String REGION = "region"; + } +} diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java index 0cdca9cf4f..c84abf119f 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java @@ -10,7 +10,7 @@ public interface FileStorage { void addFile(FileEntry fileEntry, InputStream content) throws FileStorageException; - @Deprecated // This method is not reliable for aws as BlobStore::list might not return a complete list + @Deprecated List getFileEntriesWithoutContent(List fileEntries) throws FileStorageException; List getExistingFileEntries(List fileEntries) throws FileStorageException; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java index 2fe30e8162..511ad1fbfc 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java @@ -1,20 +1,5 @@ package org.cloudfoundry.multiapps.controller.persistence.services; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.channels.Channels; -import java.text.MessageFormat; -import java.time.Duration; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Base64; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; - import com.google.api.gax.retrying.RetrySettings; import com.google.auth.Credentials; import com.google.auth.oauth2.GoogleCredentials; @@ -28,6 +13,7 @@ import com.google.cloud.storage.StorageRetryStrategy; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; import org.springframework.http.MediaType; @@ -51,16 +37,9 @@ public class GcpObjectStoreFileStorage implements FileStorage { private final String bucketName; private final Storage storage; - private static final String BUCKET = "bucket"; - private static final int OBJECT_STORE_MAX_ATTEMPTS_CONFIG = 6; - private static final double OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG = 2.0; - private static final Duration OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES = Duration.ofMinutes(10); - private static final Duration OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS = Duration.ofSeconds(10); - private static final Duration OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS = Duration.ofMillis(250); - private static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData"; public GcpObjectStoreFileStorage(Map credentials) { - this.bucketName = (String) credentials.get(BUCKET); + this.bucketName = (String) credentials.get(CredentialKeys.BUCKET); this.storage = createObjectStoreStorage(credentials); } @@ -70,22 +49,23 @@ protected Storage createObjectStoreStorage(Map credentials) { .setStorageRetryStrategy(StorageRetryStrategy.getUniformStorageRetryStrategy()) .setRetrySettings( RetrySettings.newBuilder() - .setMaxAttempts(OBJECT_STORE_MAX_ATTEMPTS_CONFIG) - .setTotalTimeoutDuration(OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES) - .setMaxRetryDelayDuration(OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS) - .setInitialRetryDelayDuration(OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS) - .setRetryDelayMultiplier(OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG) + .setMaxAttempts(ObjectStoreConstants.OBJECT_STORE_MAX_ATTEMPTS_CONFIG) + .setTotalTimeoutDuration(ObjectStoreConstants.OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES) + .setMaxRetryDelayDuration(ObjectStoreConstants.OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS) + .setInitialRetryDelayDuration( + ObjectStoreConstants.OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS) + .setRetryDelayMultiplier(ObjectStoreConstants.OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG) .build()) .build() .getService(); } private Credentials getGcpCredentialsSupplier(Map credentials) { - if (!credentials.containsKey(BASE_64_ENCODED_PRIVATE_KEY_DATA)) { + if (!credentials.containsKey(CredentialKeys.BASE_64_ENCODED_PRIVATE_KEY_DATA)) { return null; } byte[] decodedKey = Base64.getDecoder() - .decode((String) credentials.get(BASE_64_ENCODED_PRIVATE_KEY_DATA)); + .decode((String) credentials.get(CredentialKeys.BASE_64_ENCODED_PRIVATE_KEY_DATA)); try { return GoogleCredentials.fromStream(new ByteArrayInputStream(decodedKey)); } catch (IOException e) { @@ -212,7 +192,10 @@ public InputStream openInputStream(String space, String id) throws FileStorageEx @Override public void testConnection() { - storage.get(bucketName, "test"); + var bucket = storage.get(bucketName); + if (bucket == null) { + throw new IllegalStateException(MessageFormat.format(Messages.OBJECT_STORE_BUCKET_NOT_FOUND, bucketName)); + } } @Override @@ -284,4 +267,9 @@ private InputStream getBlobPayloadWithOffset(FileEntry fileEntry, long startOffs throw new FileStorageException(e); } } + + private static final class CredentialKeys { + static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData"; + static final String BUCKET = "bucket"; + } } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java index 2ba0ff70a7..21f748e49e 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java @@ -65,6 +65,7 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage } } + // BlobStore::list may not return a complete list, making this method unreliable @Override public List getFileEntriesWithoutContent(List fileEntries) { Set existingFiles = getAllEntries(new ListContainerOptions()).stream() @@ -176,7 +177,9 @@ private InputStream openPayloadInputStream(Payload payload) throws FileStorageEx @Override public void testConnection() { - blobStore.blobExists(container, "test"); + if (!blobStore.containerExists(container)) { + throw new IllegalStateException(MessageFormat.format(Messages.OBJECT_STORE_BUCKET_NOT_FOUND, container)); + } } @Override diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreConstants.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreConstants.java index e69de29bb2..74a8d50c71 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreConstants.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/ObjectStoreConstants.java @@ -0,0 +1,21 @@ +package org.cloudfoundry.multiapps.controller.persistence.util; + +import java.time.Duration; + +public class ObjectStoreConstants { + + private ObjectStoreConstants() { + } + + // Shared / default + public static final int OBJECT_STORE_MAX_ATTEMPTS_CONFIG = 6; + public static final double OBJECT_STORE_RETRY_DELAY_MULTIPLIER_CONFIG = 2.0; + public static final Duration OBJECT_STORE_INITIAL_RETRY_DELAY_CONFIG_IN_MILLIS = Duration.ofMillis(250); + public static final Duration OBJECT_STORE_MAX_RETRY_DELAY_CONFIG_IN_SECONDS = Duration.ofSeconds(10); + public static final Duration OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES = Duration.ofMinutes(10); + + // AWS S3 + public static final Duration AWS_OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES = Duration.ofMinutes(30); + public static final Duration AWS_OBJECT_STORE_SOCKET_TIMEOUT_CONFIG_IN_MINUTES = Duration.ofMinutes(10); + public static final Duration AWS_OBJECT_STORE_CONNECTION_TIMEOUT_CONFIG_IN_SECONDS = Duration.ofSeconds(10); +} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java new file mode 100644 index 0000000000..02bae97ce4 --- /dev/null +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java @@ -0,0 +1,606 @@ +package org.cloudfoundry.multiapps.controller.persistence.services; + +import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.retries.StandardRetryStrategy; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.S3Object; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class AwsS3ObjectStoreFileStorageTest { + + private static final String BUCKET_NAME = "test-bucket"; + + @Mock + private S3Client s3Client; + + @Mock + private FileContentProcessor fileContentProcessor; + + private AwsS3ObjectStoreFileStorage fileStorage; + private final InputStream inputStream = new ByteArrayInputStream(new byte[] {}); + private static final String TEST_SPACE_ID = UUID.randomUUID() + .toString(); + private static final String TEST_SPACE_ID_2 = UUID.randomUUID() + .toString(); + private static final String TEST_ID = UUID.randomUUID() + .toString(); + private static final String TEST_ID_2 = UUID.randomUUID() + .toString(); + private static final String NAMESPACE = "namespace"; + private static final String NAMESPACE_2 = "namespace_2"; + + @BeforeEach + void setUp() throws Exception { + MockitoAnnotations.openMocks(this) + .close(); + + fileStorage = new AwsS3ObjectStoreFileStorage(Map.of("bucket", BUCKET_NAME)) { + + @Override + protected S3Client createS3Client(Map credentials) { + return s3Client; + } + }; + } + + @Test + void testAddFileWithSuccessfulUpload() throws FileStorageException { + FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + + fileStorage.addFile(fileEntry, inputStream); + + ArgumentCaptor captor = ArgumentCaptor.forClass(PutObjectRequest.class); + verify(s3Client).putObject(captor.capture(), any(software.amazon.awssdk.core.sync.RequestBody.class)); + PutObjectRequest request = captor.getValue(); + assertEquals(BUCKET_NAME, request.bucket()); + assertEquals(TEST_ID, request.key()); + } + + @Test + void testAddFileWithFailedUpload() { + when(s3Client.putObject(any(PutObjectRequest.class), any(software.amazon.awssdk.core.sync.RequestBody.class))) + .thenThrow(S3Exception.builder() + .message("upload failed") + .build()); + FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + + assertThrows(FileStorageException.class, () -> fileStorage.addFile(fileEntry, inputStream)); + } + + @Test + void testGetFileEntriesWithoutContent() { + setupListObjects(TEST_ID_2); + FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + + List result = fileStorage.getFileEntriesWithoutContent(List.of(fileEntry)); + + assertEquals(1, result.size()); + assertEquals(TEST_ID, result.getFirst() + .getId()); + } + + @Test + void testGetFileEntriesWithoutContentWithoutMatches() { + setupListObjects(TEST_ID, TEST_ID_2); + FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + + List result = fileStorage.getFileEntriesWithoutContent(List.of(fileEntry)); + + assertEquals(0, result.size()); + } + + @Test + void testExistsInObjectStoreWhenFileExists() { + when(s3Client.headObject(any(HeadObjectRequest.class))).thenReturn(HeadObjectResponse.builder() + .build()); + FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + + List result = fileStorage.getExistingFileEntries(List.of(fileEntry)); + + assertEquals(1, result.size()); + assertEquals(TEST_ID, result.getFirst() + .getId()); + } + + @Test + void testExistsInObjectStoreWhenFileDoesNotExist() { + when(s3Client.headObject(any(HeadObjectRequest.class))).thenThrow(NoSuchKeyException.builder() + .build()); + FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + + List result = fileStorage.getExistingFileEntries(List.of(fileEntry)); + + assertTrue(result.isEmpty()); + } + + @Test + void testDeleteFile() { + fileStorage.deleteFile(TEST_ID, TEST_SPACE_ID); + + ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteObjectRequest.class); + verify(s3Client).deleteObject(captor.capture()); + assertEquals(BUCKET_NAME, captor.getValue() + .bucket()); + assertEquals(TEST_ID, captor.getValue() + .key()); + } + + @Test + void testDeleteFilesBySpaceIdsWithAllMatchingItems() { + setupListObjectsWithKeys(TEST_ID, TEST_ID_2); + setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, LocalDateTime.now()); + setupHeadObjectWithMetadata(TEST_ID_2, TEST_SPACE_ID_2, NAMESPACE_2, LocalDateTime.now()); + + fileStorage.deleteFilesBySpaceIds(List.of(TEST_SPACE_ID, TEST_SPACE_ID_2)); + + verify(s3Client).deleteObjects(any(DeleteObjectsRequest.class)); + } + + @Test + void testDeleteFilesBySpaceIdsWithOneMatchingItem() { + setupListObjectsWithKeys(TEST_ID, TEST_ID_2); + setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, LocalDateTime.now()); + setupHeadObjectWithMetadata(TEST_ID_2, TEST_SPACE_ID_2, NAMESPACE_2, LocalDateTime.now()); + + fileStorage.deleteFilesBySpaceIds(List.of(TEST_SPACE_ID)); + + ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3Client).deleteObjects(captor.capture()); + assertEquals(1, captor.getValue() + .delete() + .objects() + .size()); + } + + @Test + void testDeleteFilesBySpaceIdsWithoutMatchingItem() { + setupListObjectsWithKeys(TEST_ID); + setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, LocalDateTime.now()); + + fileStorage.deleteFilesBySpaceIds(List.of("non-existing-space")); + + verify(s3Client, never()).deleteObjects(any(DeleteObjectsRequest.class)); + } + + @Test + void testDeleteFilesBySpaceAndNamespaceWithOneMatch() { + setupListObjectsWithKeys(TEST_ID, TEST_ID_2); + setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, LocalDateTime.now()); + setupHeadObjectWithMetadata(TEST_ID_2, TEST_SPACE_ID_2, NAMESPACE_2, LocalDateTime.now()); + + fileStorage.deleteFilesBySpaceAndNamespace(TEST_SPACE_ID, NAMESPACE); + + ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3Client).deleteObjects(captor.capture()); + assertEquals(1, captor.getValue() + .delete() + .objects() + .size()); + assertEquals(TEST_ID, captor.getValue() + .delete() + .objects() + .getFirst() + .key()); + } + + @Test + void testDeleteFilesModifiedBefore() { + LocalDateTime oldModified = LocalDateTime.now() + .minusMinutes(15); + + setupListObjectsWithKeys(TEST_ID, TEST_ID_2); + setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, oldModified); + setupHeadObjectWithMetadata(TEST_ID_2, TEST_SPACE_ID_2, NAMESPACE_2, oldModified); + + int deletedCount = fileStorage.deleteFilesModifiedBefore(LocalDateTime.now()); + + assertEquals(2, deletedCount); + verify(s3Client).deleteObjects(any(DeleteObjectsRequest.class)); + } + + @Test + void testDeleteFilesModifiedBeforeWithNoOldFiles() { + setupListObjectsWithKeys(TEST_ID); + setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, LocalDateTime.now()); + + LocalDateTime cutoff = LocalDateTime.now() + .minusDays(1); + int deletedCount = fileStorage.deleteFilesModifiedBefore(cutoff); + + assertEquals(0, deletedCount); + verify(s3Client, never()).deleteObjects(any(DeleteObjectsRequest.class)); + } + + @SuppressWarnings("unchecked") + @Test + void testProcessFileContent() throws FileStorageException, IOException { + ResponseInputStream mockStream = mock(ResponseInputStream.class); + when(s3Client.getObject(any(GetObjectRequest.class))).thenReturn(mockStream); + when(fileContentProcessor.process(any(InputStream.class))).thenReturn("result"); + + String result = fileStorage.processFileContent(TEST_SPACE_ID, TEST_ID, fileContentProcessor); + + assertEquals("result", result); + verify(fileContentProcessor).process(mockStream); + } + + @Test + void testProcessFileContentWithNoSuchKeyException() { + when(s3Client.getObject(any(GetObjectRequest.class))).thenThrow(NoSuchKeyException.builder() + .build()); + + assertThrows(FileStorageException.class, () -> fileStorage.processFileContent(TEST_SPACE_ID, TEST_ID, fileContentProcessor)); + } + + @SuppressWarnings("unchecked") + @Test + void testProcessFileContentWithProcessorException() throws IOException { + ResponseInputStream mockStream = mock(ResponseInputStream.class); + when(s3Client.getObject(any(GetObjectRequest.class))).thenReturn(mockStream); + when(fileContentProcessor.process(any(InputStream.class))).thenThrow(new IOException("processing failed")); + + assertThrows(FileStorageException.class, () -> fileStorage.processFileContent(TEST_SPACE_ID, TEST_ID, fileContentProcessor)); + } + + @SuppressWarnings("unchecked") + @Test + void testOpenInputStream() throws FileStorageException { + ResponseInputStream mockStream = mock(ResponseInputStream.class); + when(s3Client.getObject(any(GetObjectRequest.class))).thenReturn(mockStream); + + InputStream result = fileStorage.openInputStream(TEST_SPACE_ID, TEST_ID); + + assertNotNull(result); + ArgumentCaptor captor = ArgumentCaptor.forClass(GetObjectRequest.class); + verify(s3Client).getObject(captor.capture()); + assertEquals(BUCKET_NAME, captor.getValue() + .bucket()); + assertEquals(TEST_ID, captor.getValue() + .key()); + } + + @Test + void testOpenInputStreamWithNoSuchKeyException() { + when(s3Client.getObject(any(GetObjectRequest.class))).thenThrow(NoSuchKeyException.builder() + .build()); + + assertThrows(FileStorageException.class, () -> fileStorage.openInputStream(TEST_SPACE_ID, TEST_ID)); + } + + @Test + void testTestConnection() { + when(s3Client.listObjectsV2(any(ListObjectsV2Request.class))).thenReturn(ListObjectsV2Response.builder() + .isTruncated(false) + .build()); + + fileStorage.testConnection(); + + ArgumentCaptor captor = ArgumentCaptor.forClass(ListObjectsV2Request.class); + verify(s3Client).listObjectsV2(captor.capture()); + assertEquals(BUCKET_NAME, captor.getValue() + .bucket()); + assertEquals(1, captor.getValue() + .maxKeys()); + } + + @Test + void testDeleteFilesByIds() { + when(s3Client.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(DeleteObjectsResponse.builder() + .build()); + + fileStorage.deleteFilesByIds(List.of(TEST_ID)); + + ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3Client).deleteObjects(captor.capture()); + assertEquals(1, captor.getValue() + .delete() + .objects() + .size()); + assertEquals(TEST_ID, captor.getValue() + .delete() + .objects() + .getFirst() + .key()); + } + + @Test + void testDeleteFilesByIdsWithEmptyList() { + fileStorage.deleteFilesByIds(List.of()); + + verify(s3Client, never()).listObjectsV2(any(ListObjectsV2Request.class)); + verify(s3Client, never()).deleteObjects(any(DeleteObjectsRequest.class)); + } + + @Test + void testDeleteFilesByIdsWithAllMatching() { + when(s3Client.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(DeleteObjectsResponse.builder() + .build()); + + fileStorage.deleteFilesByIds(List.of(TEST_ID, TEST_ID_2)); + + ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3Client).deleteObjects(captor.capture()); + assertEquals(2, captor.getValue() + .delete() + .objects() + .size()); + } + + @SuppressWarnings("unchecked") + @Test + void testProcessArchiveEntryContent() throws FileStorageException, IOException { + ResponseInputStream mockStream = mock(ResponseInputStream.class); + when(s3Client.getObject(any(GetObjectRequest.class))).thenReturn(mockStream); + when(fileContentProcessor.process(any(InputStream.class))).thenReturn("archive-result"); + + FileContentToProcess fileContentToProcess = ImmutableFileContentToProcess.builder() + .guid(TEST_ID) + .spaceGuid(TEST_SPACE_ID) + .startOffset(100L) + .endOffset(500L) + .build(); + + String result = fileStorage.processArchiveEntryContent(fileContentToProcess, fileContentProcessor); + + assertEquals("archive-result", result); + ArgumentCaptor captor = ArgumentCaptor.forClass(GetObjectRequest.class); + verify(s3Client).getObject(captor.capture()); + assertEquals(BUCKET_NAME, captor.getValue() + .bucket()); + assertEquals(TEST_ID, captor.getValue() + .key()); + assertEquals("bytes=100-500", captor.getValue() + .range()); + } + + @Test + void testProcessArchiveEntryContentWithNoSuchKeyException() { + when(s3Client.getObject(any(GetObjectRequest.class))).thenThrow(NoSuchKeyException.builder() + .build()); + FileContentToProcess fileContentToProcess = ImmutableFileContentToProcess.builder() + .guid(TEST_ID) + .spaceGuid(TEST_SPACE_ID) + .startOffset(0L) + .endOffset(100L) + .build(); + + assertThrows(FileStorageException.class, () -> fileStorage.processArchiveEntryContent(fileContentToProcess, fileContentProcessor)); + } + + @Test + void getExistingFileEntriesWhenAllEntriesExist() { + when(s3Client.headObject(any(HeadObjectRequest.class))).thenReturn(HeadObjectResponse.builder() + .build()); + FileEntry firstEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + FileEntry secondEntry = createFileEntry(TEST_SPACE_ID_2, TEST_ID_2); + + List result = fileStorage.getExistingFileEntries(List.of(firstEntry, secondEntry)); + + assertEquals(2, result.size()); + List returnedIds = result.stream() + .map(FileEntry::getId) + .toList(); + assertTrue(returnedIds.contains(TEST_ID)); + assertTrue(returnedIds.contains(TEST_ID_2)); + } + + @Test + void getExistingFileEntriesWhenNoEntriesExist() { + when(s3Client.headObject(any(HeadObjectRequest.class))).thenThrow(NoSuchKeyException.builder() + .build()); + FileEntry firstEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + FileEntry secondEntry = createFileEntry(TEST_SPACE_ID_2, TEST_ID_2); + + List result = fileStorage.getExistingFileEntries(List.of(firstEntry, secondEntry)); + + assertTrue(result.isEmpty()); + } + + @Test + void getExistingFileEntriesWhenSomeEntriesExist() { + when(s3Client.headObject(headObjectRequestForKey(TEST_ID))).thenReturn(HeadObjectResponse.builder() + .build()); + when(s3Client.headObject(headObjectRequestForKey(TEST_ID_2))).thenThrow(NoSuchKeyException.builder() + .build()); + FileEntry existingEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); + FileEntry nonExistingEntry = createFileEntry(TEST_SPACE_ID_2, TEST_ID_2); + + List result = fileStorage.getExistingFileEntries(List.of(existingEntry, nonExistingEntry)); + + assertEquals(1, result.size()); + assertEquals(TEST_ID, result.getFirst() + .getId()); + } + + @Test + void testDeleteFilesBySpaceIdsWithPagination() { + String thirdId = UUID.randomUUID() + .toString(); + ListObjectsV2Response firstPage = ListObjectsV2Response.builder() + .contents(S3Object.builder() + .key(TEST_ID) + .build()) + .isTruncated(true) + .nextContinuationToken("token-1") + .build(); + ListObjectsV2Response secondPage = ListObjectsV2Response.builder() + .contents(S3Object.builder() + .key(TEST_ID_2) + .build(), + S3Object.builder() + .key(thirdId) + .build()) + .isTruncated(false) + .build(); + when(s3Client.listObjectsV2(any(ListObjectsV2Request.class))).thenReturn(firstPage, secondPage); + when(s3Client.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(DeleteObjectsResponse.builder() + .build()); + setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, LocalDateTime.now()); + setupHeadObjectWithMetadata(TEST_ID_2, TEST_SPACE_ID, NAMESPACE, LocalDateTime.now()); + setupHeadObjectWithMetadata(thirdId, TEST_SPACE_ID_2, NAMESPACE_2, LocalDateTime.now()); + + fileStorage.deleteFilesBySpaceIds(List.of(TEST_SPACE_ID)); + + ArgumentCaptor captor = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3Client).deleteObjects(captor.capture()); + assertEquals(2, captor.getValue() + .delete() + .objects() + .size()); + verify(s3Client, times(2)).listObjectsV2(any(ListObjectsV2Request.class)); + } + + @Test + void testDeleteFilesBySpaceIdsWhenHeadObjectReturnsNoSuchKey() { + setupListObjectsWithKeys(TEST_ID); + when(s3Client.headObject(any(HeadObjectRequest.class))).thenThrow(NoSuchKeyException.builder() + .build()); + + fileStorage.deleteFilesBySpaceIds(List.of(TEST_SPACE_ID)); + + verify(s3Client, never()).deleteObjects(any(DeleteObjectsRequest.class)); + } + + @Test + void getExistingFileEntriesWithEmptyList() { + List result = fileStorage.getExistingFileEntries(List.of()); + + assertTrue(result.isEmpty()); + verify(s3Client, never()).headObject(any(HeadObjectRequest.class)); + } + + @Test + void testBuildClientOverrideConfigSetsApiCallTimeout() { + ClientOverrideConfiguration config = fileStorage.buildClientOverrideConfig(); + + assertTrue(config.apiCallTimeout() + .isPresent()); + assertEquals(ObjectStoreConstants.AWS_OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES, config.apiCallTimeout() + .get()); + } + + @Test + void testBuildClientOverrideConfigSetsApiCallAttemptTimeout() { + ClientOverrideConfiguration config = fileStorage.buildClientOverrideConfig(); + + assertTrue(config.apiCallAttemptTimeout() + .isPresent()); + assertEquals(ObjectStoreConstants.OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES, config.apiCallAttemptTimeout() + .get()); + } + + @Test + void testBuildClientOverrideConfigSetsRetryStrategy() { + ClientOverrideConfiguration config = fileStorage.buildClientOverrideConfig(); + + assertTrue(config.retryStrategy() + .isPresent()); + assertInstanceOf(StandardRetryStrategy.class, config.retryStrategy() + .get()); + assertEquals(ObjectStoreConstants.OBJECT_STORE_MAX_ATTEMPTS_CONFIG, config.retryStrategy() + .get() + .maxAttempts()); + } + + @Test + void testCreateS3ClientWithValidCredentials() { + Map credentials = Map.of("access_key_id", "test-key", + "secret_access_key", "test-secret", + "bucket", "test-bucket", + "host", "s3.amazonaws.com", + "region", "eu-central-1"); + AwsS3ObjectStoreFileStorage storage = new AwsS3ObjectStoreFileStorage(credentials); + + assertNotNull(storage); + storage.destroy(); + } + + private HeadObjectRequest headObjectRequestForKey(String key) { + return HeadObjectRequest.builder() + .bucket(BUCKET_NAME) + .key(key) + .build(); + } + + private void setupListObjects(String... keys) { + ListObjectsV2Response.Builder responseBuilder = ListObjectsV2Response.builder() + .isTruncated(false); + List s3Objects = Arrays.stream(keys) + .map(key -> S3Object.builder() + .key(key) + .build()) + .toList(); + responseBuilder.contents(s3Objects); + when(s3Client.listObjectsV2(any(ListObjectsV2Request.class))).thenReturn(responseBuilder.build()); + } + + private void setupListObjectsWithKeys(String... keys) { + setupListObjects(keys); + when(s3Client.deleteObjects(any(DeleteObjectsRequest.class))).thenReturn(DeleteObjectsResponse.builder() + .build()); + } + + private void setupHeadObjectWithMetadata(String key, String space, String namespace, LocalDateTime modified) { + long modifiedMillis = modified.atZone(ZoneId.systemDefault()) + .toInstant() + .toEpochMilli(); + HeadObjectResponse response = HeadObjectResponse.builder() + .metadata(Map.of("space", space, + "namespace", namespace, + "modified", Long.toString(modifiedMillis))) + .build(); + when(s3Client.headObject(headObjectRequestForKey(key))).thenReturn(response); + } + + private static FileEntry createFileEntry(String space, String id) { + return ImmutableFileEntry.builder() + .space(space) + .size(BigInteger.TEN) + .modified(LocalDateTime.now()) + .id(id) + .build(); + } +} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java index 3d2cfeeb19..14264ffa72 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java @@ -3,6 +3,7 @@ import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobId; import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Bucket; import com.google.cloud.storage.Storage; import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; @@ -20,8 +21,10 @@ import java.util.Map; import java.util.UUID; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.mock; @@ -151,6 +154,33 @@ void getExistingFileEntriesPassesCorrectBlobIdsToStorage() { verify(mockedStorage).get(List.of(BlobId.of(CONTAINER, entry.getId()))); } + @Override + @Test + void testConnection() { + Bucket mockBucket = mock(Bucket.class); + when(mockedStorage.get(CONTAINER)).thenReturn(mockBucket); + assertDoesNotThrow(mockedGcpFileStorage::testConnection); + } + + @Test + void testTestConnectionWhenBucketExists() { + Bucket mockBucket = mock(Bucket.class); + when(mockedStorage.get(CONTAINER)).thenReturn(mockBucket); + + assertDoesNotThrow(mockedGcpFileStorage::testConnection); + verify(mockedStorage).get(CONTAINER); + } + + @Test + void testTestConnectionWhenBucketDoesNotExist() { + when(mockedStorage.get(CONTAINER)).thenReturn(null); + + var exception = assertThrows(IllegalStateException.class, mockedGcpFileStorage::testConnection); + assertTrue(exception.getMessage() + .contains(CONTAINER)); + verify(mockedStorage).get(CONTAINER); + } + private void mockStorageGetToReturn(List blobs) { when(mockedStorage.get(anyList())).thenReturn(blobs); } diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java index 0addf3485f..11bcddac57 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java @@ -230,6 +230,30 @@ void testConnection() { assertDoesNotThrow(() -> fileStorage.testConnection()); } + @Test + void testTestConnectionWhenContainerExists() { + BlobStore mockBlobStore = mock(BlobStore.class); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER); + + when(mockBlobStore.containerExists(CONTAINER)).thenReturn(true); + + assertDoesNotThrow(testFileStorage::testConnection); + verify(mockBlobStore).containerExists(CONTAINER); + } + + @Test + void testTestConnectionWhenContainerDoesNotExist() { + BlobStore mockBlobStore = mock(BlobStore.class); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER); + + when(mockBlobStore.containerExists(CONTAINER)).thenReturn(false); + + var exception = assertThrows(IllegalStateException.class, testFileStorage::testConnection); + assertTrue(exception.getMessage() + .contains(CONTAINER)); + verify(mockBlobStore).containerExists(CONTAINER); + } + @Test void testDeleteFilesByIds() throws Exception { FileEntry fileEntry = addFile(TEST_FILE_LOCATION); diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/OperationInErrorStateHandlerTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/OperationInErrorStateHandlerTest.java index 862eb246eb..987b28d5f0 100644 --- a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/OperationInErrorStateHandlerTest.java +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/OperationInErrorStateHandlerTest.java @@ -201,11 +201,11 @@ private void prepareOperationService() { } private void assertErrorStateSet() { - Operation updatedOperation = ImmutableOperation.builder() - .state(Operation.State.ERROR) - .build(); + Operation errorOperation = ImmutableOperation.builder() + .state(Operation.State.ERROR) + .build(); Mockito.verify(operationService) - .update(updatedOperation, updatedOperation); + .update(errorOperation, errorOperation); } private OperationInErrorStateHandlerMock mockHandler() { diff --git a/multiapps-controller-web/pom.xml b/multiapps-controller-web/pom.xml index 969683e148..eae9d89841 100644 --- a/multiapps-controller-web/pom.xml +++ b/multiapps-controller-web/pom.xml @@ -207,5 +207,13 @@ com.google.cloud google-cloud-nio + + software.amazon.awssdk + s3 + + + software.amazon.awssdk + url-connection-client + diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Constants.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Constants.java index eea2d64634..f7fecd8d0c 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Constants.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Constants.java @@ -32,13 +32,12 @@ private Constants() { public static final String REGION = "region"; public static final String ENDPOINT = "endpoint"; public static final String ENDPOINT_URL = "endpoint-url"; - public static final String ACCOUNT_NAME = "account_name"; public static final String SAS_TOKEN = "sas_token"; - public static final String CONTAINER_NAME = "container_name"; public static final String CONTAINER_NAME_WITH_DASH = "container-name"; - public static final String CONTAINER_URI = "container_uri"; public static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData"; public static final String HOST = "host"; + public static final Set OBJECT_STORE_CUSTOM_REGIONS = Set.of("eu-south-1"); + public static final String OBJECT_STORE_JCLOUDS_REGIONS = "jclouds.regions"; public static final String AWS_S_3 = "aws-s3"; public static final String AZUREBLOB = "azureblob"; diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Messages.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Messages.java index 24d2c046d5..fbb162bf9a 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Messages.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Messages.java @@ -19,7 +19,7 @@ public final class Messages { public static final String MTAR_ENDPOINT_NOT_SECURE_FOR_JOB_WITH_ID = "Remote MTAR endpoint is not a secure connection. HTTPS required. Job id: {0}"; public static final String CANNOT_PARSE_CONTAINER_URI_OF_OBJECT_STORE = "Cannot parse container_uri of object store"; public static final String REQUEST_0_1_FAILED_WITH_2 = "Request \"{0} {1}\" failed with \"{2}\""; - public static final String ERROR_OCCURRED_WHILE_DELETING_JOB_ENTRY = "Error occurred while deleting job entry"; + public static final String ERROR_OCCURRED_WHILE_DELETING_JOB_ENTRY = "Error occurred while deleting job entry with id: {}"; public static final String CANNOT_CREATE_OBJECT_STORE_CLIENT_WITH_PROVIDER_0 = "Cannot create Object Store client with provider: {0}"; public static final String NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND = "No valid Object Store configuration found!"; public static final String MISSING_PROPERTIES_FOR_CREATING_THE_SPECIFIC_PROVIDER = "Missing properties for creating the specific provider!"; @@ -54,10 +54,17 @@ public final class Messages { public static final String UNSUPPORTED_TOKEN_TYPE = "Unsupported token type: \"{0}\"."; public static final String CLEARING_FLOWABLE_LOCK_OWNER_THREW_AN_EXCEPTION_0 = "Clearing Flowable lock owner on JVM shutdown threw an exception: {0}"; public static final String FETCHING_FILE_FAILED = "Fetching file {0} in space {1} failed with: {2}"; - public static final String ASYNC_UPLOAD_JOB_FAILED = "Async upload job {0} failed with: {1}"; + public static final String ASYNC_UPLOAD_JOB_FAILED = "Async upload job {0} for file \"{1}\" failed with: {2}"; + public static final String ASYNC_UPLOAD_JOB_ERROR = "{0} Async upload job id: {1}"; // WARN log messages + public static final String FILE_UPLOAD_ATTEMPT_FAILED = "Upload attempt {0}/{1} failed. Retrying in {2} ms. Cause: {3}"; + public static final String FILE_UPLOAD_ALL_ATTEMPTS_EXHAUSTED = "All {0} upload attempts exhausted. Last error: {1}"; + + public static final String NO_OBJECTSTORE_PROVIDER_FOUND_FOR_0 = "No ObjectStore provider found for {}!"; + public static final String NO_OBJECT_STORE_PROVIDERS_DETECTED = "No object store providers detected from credentials. Service name: {}"; + // INFO log messages public static final String ALM_SERVICE_ENV_INITIALIZED = "Deploy service environment initialized"; public static final String STORING_TOKEN_FOR_USER_WITH_GUID_0_WHICH_EXPIRES_AT_1 = "Storing token for user with GUID \"{0}\" which expires at: {1}"; @@ -67,8 +74,12 @@ public final class Messages { public static final String OBJECTSTORE_FOR_BINARIES_STORAGE = "Objectstore will be used for binaries storage"; public static final String CLEARING_LOCK_OWNER = "Clearing lock owner {0}..."; public static final String CLEARED_LOCK_OWNER = "Cleared lock owner {0}"; + public static final String OBJECT_STORE_PROVIDERS_DETECTED = "Object store providers detected: {}"; + public static final String ATTEMPTING_TO_CREATE_OBJECT_STORE_CLIENT = "Attempting to create object store client with provider: {}, container: {}, region: {}, host: {}, endpoint: {}"; public static final String OBJECT_STORE_WITH_PROVIDER_0_CREATED = "Object store with provider: {0} created"; - public static final String JOB_WITH_ID_WAS_NOT_UPDATED_WITHIN_SECONDS = "Job with ID: {} was not updated within: {} seconds"; + public static final String JOB_WITH_ID_WAS_NOT_UPDATED_WITHIN_SECONDS_ON_START = "startUploadFromUrl: Job with ID: {} was not updated within: {} seconds, clearing and re-triggering"; + public static final String JOB_WITH_ID_WAS_NOT_UPDATED_WITHIN_SECONDS_ON_GET = "getUploadFromUrlJob: Job with ID: {} was not updated within: {} seconds, returning error"; + public static final String STALE_JOB_DETAILS = "Stale job details - id: {}, state: {}, updatedAt: {}, addedAt: {}, startedAt: {}, bytesRead: {}, url: {}, space: {}, namespace: {}, user: {}, instance: {}"; public static final String CLEARING_OLD_ENTRY = "Clearing old entry with id: {0}"; // DEBUG log messages @@ -83,6 +94,7 @@ public final class Messages { public static final String ASYNC_UPLOAD_JOB_FINISHED = "Async upload job {} finished"; public static final String UPLOADING_MTAR_STREAM_FROM_REMOTE_ENDPOINT_WITH_JOB_ID = "Uploading MTAR stream from remote endpoint: {}. Job id: {}"; public static final String CALLING_REMOTE_MTAR_ENDPOINT_FOR_JOB_WITH_ID = "Calling remote MTAR endpoint {}. Job id: {}"; + public static final String ASYNC_UPLOAD_JOB_MONITOR_UPDATE = "Job {} monitor update - state: {}, bytesRead: {}, updatedAt: {}"; private Messages() { } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/FilesApiServiceImpl.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/FilesApiServiceImpl.java index 9dbcb55d33..dacfeede56 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/FilesApiServiceImpl.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/FilesApiServiceImpl.java @@ -133,6 +133,11 @@ public ResponseEntity startUploadFromUrl(String spaceGuid, String namespac return triggerUploadFromUrl(spaceGuid, namespace, urlWithoutUserInfo, decodedUrl, fileUrl.getUserCredentials()); } if (hasJobStuck(existingJob)) { + LOGGER.warn(Messages.JOB_WITH_ID_WAS_NOT_UPDATED_WITHIN_SECONDS_ON_START, existingJob.getId(), UPDATE_JOB_TIMEOUT); + LOGGER.warn(Messages.STALE_JOB_DETAILS, existingJob.getId(), existingJob.getState(), existingJob.getUpdatedAt(), + existingJob.getAddedAt(), existingJob.getStartedAt(), existingJob.getBytesRead(), existingJob.getUrl(), + existingJob.getSpaceGuid(), existingJob.getNamespace(), existingJob.getUser(), + existingJob.getInstanceIndex()); deleteAsyncJobEntry(existingJob); return triggerUploadFromUrl(spaceGuid, namespace, urlWithoutUserInfo, decodedUrl, fileUrl.getUserCredentials()); } @@ -166,7 +171,10 @@ public ResponseEntity getUploadFromUrlJob(String spaceGuid, S private ResponseEntity getAsyncUploadResult(AsyncUploadJobEntry job) { if (job.getState() == State.RUNNING || job.getState() == State.INITIAL) { if (hasJobStuck(job)) { - LOGGER.info(Messages.JOB_WITH_ID_WAS_NOT_UPDATED_WITHIN_SECONDS, job.getId(), UPDATE_JOB_TIMEOUT); + LOGGER.warn(Messages.JOB_WITH_ID_WAS_NOT_UPDATED_WITHIN_SECONDS_ON_GET, job.getId(), UPDATE_JOB_TIMEOUT); + LOGGER.warn(Messages.STALE_JOB_DETAILS, job.getId(), job.getState(), job.getUpdatedAt(), job.getAddedAt(), + job.getStartedAt(), job.getBytesRead(), job.getUrl(), job.getSpaceGuid(), job.getNamespace(), + job.getUser(), job.getInstanceIndex()); return ResponseEntity.ok( createErrorResult(MessageFormat.format(Messages.JOB_NOT_UPDATED_FOR_0_SECONDS, UPDATE_JOB_TIMEOUT), AsyncUploadResult.ClientAction.RETRY_UPLOAD)); @@ -256,7 +264,7 @@ private void deleteAsyncJobEntry(AsyncUploadJobEntry entry) { .id(entry.getId()) .delete(); } catch (Exception e) { - LOGGER.error(Messages.ERROR_OCCURRED_WHILE_DELETING_JOB_ENTRY, e); + LOGGER.error(Messages.ERROR_OCCURRED_WHILE_DELETING_JOB_ENTRY, entry.getId(), e); } } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java index 0001e13be1..8b70d64b28 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java @@ -1,43 +1,40 @@ package org.cloudfoundry.multiapps.controller.web.configuration.bean.factory; -import java.text.MessageFormat; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - import com.google.common.base.Joiner; import io.pivotal.cfenv.core.CfService; import org.apache.commons.lang3.StringUtils; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.util.UriUtil; +import org.cloudfoundry.multiapps.controller.persistence.services.AwsS3ObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder; import org.cloudfoundry.multiapps.controller.web.Constants; import org.cloudfoundry.multiapps.controller.web.Messages; -import org.cloudfoundry.multiapps.controller.web.configuration.service.ImmutableObjectStoreServiceInfo; import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfo; import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfoCreator; import org.jclouds.ContextBuilder; -import org.jclouds.aws.domain.Region; import org.jclouds.blobstore.BlobStoreContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.FactoryBean; import org.springframework.beans.factory.InitializingBean; +import java.text.MessageFormat; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + public class ObjectStoreFileStorageFactoryBean implements FactoryBean, InitializingBean { private static final Logger LOGGER = LoggerFactory.getLogger(ObjectStoreFileStorageFactoryBean.class); - private static final Set CUSTOM_REGIONS = Set.of("eu-south-1"); - private static final String JCLOUDS_REGIONS = "jclouds.regions"; private final String serviceName; private final EnvironmentServicesFinder environmentServicesFinder; @@ -58,7 +55,13 @@ public void afterPropertiesSet() { private FileStorage createObjectStoreFileStorage() { List providersServiceInfo = getProvidersServiceInfo(); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(Messages.OBJECT_STORE_PROVIDERS_DETECTED, providersServiceInfo.stream() + .map(ObjectStoreServiceInfo::getProvider) + .collect(Collectors.joining(", "))); + } if (providersServiceInfo.isEmpty()) { + LOGGER.warn(Messages.NO_OBJECT_STORE_PROVIDERS_DETECTED, serviceName); return null; } Map exceptions = new HashMap<>(); @@ -66,17 +69,36 @@ private FileStorage createObjectStoreFileStorage() { if (!isObjectStoreEnvValid(objectStoreProviderName)) { return createObjectStoreFromFirstReachableProvider(exceptions, providersServiceInfo); } - Optional optionalFileStorage = createObjectStoreBasedOnProvider(objectStoreProviderName, providersServiceInfo, exceptions); - if (optionalFileStorage.isPresent()) { return optionalFileStorage.get(); } - throw buildNoValidObjectStoreException(exceptions); } + public List getProvidersServiceInfo() { + Map credentials = getServiceCredentials(); + if (credentials.isEmpty()) { + return Collections.emptyList(); + } + return new ObjectStoreServiceInfoCreator().getAllProvidersServiceInfo(credentials); + } + + private Map getServiceCredentials() { + CfService service = environmentServicesFinder.findService(serviceName); + if (service == null) { + return Map.of(); + } + return service.getCredentials() + .getMap(); + } + + private boolean isObjectStoreEnvValid(String objectStoreProviderName) { + return objectStoreProviderName != null && !objectStoreProviderName.isEmpty() && Constants.ENV_TO_OS_PROVIDER.containsKey( + objectStoreProviderName); + } + public FileStorage createObjectStoreFromFirstReachableProvider(Map exceptions, List providersServiceInfo) { for (ObjectStoreServiceInfo objectStoreServiceInfo : providersServiceInfo) { @@ -85,45 +107,22 @@ public FileStorage createObjectStoreFromFirstReachableProvider(Map gcpObjectStoreOpt = tryToCreateGcpObjectStore(exceptions); - if (gcpObjectStoreOpt.isPresent()) { - return gcpObjectStoreOpt.get(); - } throw buildNoValidObjectStoreException(exceptions); } - private Optional createObjectStoreBasedOnProvider(String objectStoreProviderName, - List providersServiceInfo, - Map exceptions) { - Optional objectStoreServiceInfoOptional = getAppropriateProvider(objectStoreProviderName, - providersServiceInfo); - Optional createdObjectStore; - if (objectStoreServiceInfoOptional.isPresent()) { - ObjectStoreServiceInfo objectStoreServiceInfo = objectStoreServiceInfoOptional.get(); - createdObjectStore = tryToCreateObjectStore(objectStoreServiceInfo, exceptions); - } else { - createdObjectStore = tryToCreateGcpObjectStore(exceptions); - } - return createdObjectStore; - } - - private Optional getAppropriateProvider(String objectStoreProviderName, - List providersServiceInfo) { - String appropriateProvider = Constants.ENV_TO_OS_PROVIDER.get(objectStoreProviderName); - return providersServiceInfo.stream() - .filter(provider -> appropriateProvider.equals(provider.getProvider())) - .findFirst(); - } - - private Optional tryToCreateGcpObjectStore(Map exceptions) { - return tryToCreateObjectStore(ImmutableObjectStoreServiceInfo.builder() - .provider(Constants.GOOGLE_CLOUD_STORAGE) - .build(), exceptions); - } - private Optional tryToCreateObjectStore(ObjectStoreServiceInfo objectStoreServiceInfo, Map exceptions) { try { + LOGGER.info(Messages.ATTEMPTING_TO_CREATE_OBJECT_STORE_CLIENT, + objectStoreServiceInfo.getProvider(), + objectStoreServiceInfo.getCredentials() + .get(Constants.BUCKET), + objectStoreServiceInfo.getCredentials() + .get(Constants.REGION), + objectStoreServiceInfo.getCredentials() + .get(Constants.HOST), + objectStoreServiceInfo.getCredentials() + .get(Constants.ENDPOINT)); FileStorage fileStorage = getFileStorageBasedOnProvider(objectStoreServiceInfo); fileStorage.testConnection(); LOGGER.info(MessageFormat.format(Messages.OBJECT_STORE_WITH_PROVIDER_0_CREATED, objectStoreServiceInfo.getProvider())); @@ -135,41 +134,22 @@ private Optional tryToCreateObjectStore(ObjectStoreServiceInfo obje } private FileStorage getFileStorageBasedOnProvider(ObjectStoreServiceInfo objectStoreServiceInfo) { - if (Constants.GOOGLE_CLOUD_STORAGE.equals(objectStoreServiceInfo.getProvider())) { - return createGcpFileStorage(); - } else { - BlobStoreContext context = getBlobStoreContext(objectStoreServiceInfo); - return createFileStorage(objectStoreServiceInfo, context); - } + return switch (objectStoreServiceInfo.getProvider()) { + case Constants.GOOGLE_CLOUD_STORAGE -> createGcpFileStorage(objectStoreServiceInfo); + case Constants.AWS_S_3 -> createAwsS3FileStorage(objectStoreServiceInfo); + default -> { + BlobStoreContext context = getBlobStoreContext(objectStoreServiceInfo); + yield createFileStorage(objectStoreServiceInfo, context); + } + }; } - private boolean isObjectStoreEnvValid(String objectStoreProviderName) { - return objectStoreProviderName != null && !objectStoreProviderName.isEmpty() && Constants.ENV_TO_OS_PROVIDER.containsKey( - objectStoreProviderName); + protected GcpObjectStoreFileStorage createGcpFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { + return new GcpObjectStoreFileStorage(objectStoreServiceInfo.getCredentials()); } - private IllegalStateException buildNoValidObjectStoreException(Map exceptions) { - exceptions.forEach((provider, exception) -> LOGGER.error( - MessageFormat.format(Messages.CANNOT_CREATE_OBJECT_STORE_CLIENT_WITH_PROVIDER_0, provider), - exception)); - return new IllegalStateException(Messages.NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND); - } - - public List getProvidersServiceInfo() { - Map credentials = getServiceCredentials(); - if (credentials.isEmpty()) { - return Collections.emptyList(); - } - return new ObjectStoreServiceInfoCreator().getAllProvidersServiceInfo(credentials); - } - - private Map getServiceCredentials() { - CfService service = environmentServicesFinder.findService(serviceName); - if (service == null) { - return Map.of(); - } - return service.getCredentials() - .getMap(); + protected AwsS3ObjectStoreFileStorage createAwsS3FileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { + return new AwsS3ObjectStoreFileStorage(objectStoreServiceInfo.getCredentials()); } private BlobStoreContext getBlobStoreContext(ObjectStoreServiceInfo serviceInfo) { @@ -177,58 +157,77 @@ private BlobStoreContext getBlobStoreContext(ObjectStoreServiceInfo serviceInfo) applyCredentials(serviceInfo, contextBuilder); addCustomRegions(contextBuilder); resolveContextEndpoint(serviceInfo, contextBuilder); - BlobStoreContext context = contextBuilder.buildView(BlobStoreContext.class); if (context == null) { throw new IllegalStateException(Messages.FAILED_TO_CREATE_BLOB_STORE_CONTEXT); } - return context; } + private void applyCredentials(ObjectStoreServiceInfo serviceInfo, ContextBuilder contextBuilder) { + Map credentials = serviceInfo.getCredentials(); + String identity = (String) credentials.get(Constants.ACCESS_KEY_ID); + String credential = (String) credentials.get(Constants.SECRET_ACCESS_KEY); + if (StringUtils.isBlank(identity) || StringUtils.isBlank(credential)) { + throw new IllegalArgumentException(Messages.MISSING_PROPERTIES_FOR_CREATING_THE_SPECIFIC_PROVIDER); + } + contextBuilder.credentials(identity, credential); + } + private void addCustomRegions(ContextBuilder contextBuilder) { Properties properties = new Properties(); - Set mergedRegions = Stream.of(CUSTOM_REGIONS, Region.DEFAULT_REGIONS, applicationConfiguration.getObjectStoreRegions()) + Set mergedRegions = Stream.of(Constants.OBJECT_STORE_CUSTOM_REGIONS, applicationConfiguration.getObjectStoreRegions()) .flatMap(Set::stream) .collect(Collectors.toSet()); - properties.setProperty(JCLOUDS_REGIONS, Joiner.on(',') - .join(mergedRegions)); - + properties.setProperty(Constants.OBJECT_STORE_JCLOUDS_REGIONS, Joiner.on(',') + .join(mergedRegions)); contextBuilder.overrides(properties); } - private void applyCredentials(ObjectStoreServiceInfo serviceInfo, ContextBuilder contextBuilder) { - if (serviceInfo.getCredentialsSupplier() != null) { - contextBuilder.credentialsSupplier(serviceInfo.getCredentialsSupplier()); - } else { - String identity = serviceInfo.getIdentity(); - String credential = serviceInfo.getCredential(); - - if (StringUtils.isBlank(identity) || StringUtils.isBlank(credential)) { - throw new IllegalArgumentException(Messages.MISSING_PROPERTIES_FOR_CREATING_THE_SPECIFIC_PROVIDER); - } - - contextBuilder.credentials(identity, credential); - } - } - private void resolveContextEndpoint(ObjectStoreServiceInfo serviceInfo, ContextBuilder contextBuilder) { - if (StringUtils.isNotEmpty(serviceInfo.getEndpoint())) { - contextBuilder.endpoint(serviceInfo.getEndpoint()); + Map credentials = serviceInfo.getCredentials(); + String endpoint = (String) credentials.get(Constants.ENDPOINT); + String host = (String) credentials.get(Constants.HOST); + if (StringUtils.isNotEmpty(endpoint)) { + contextBuilder.endpoint(endpoint); return; } - if (StringUtils.isNotEmpty(serviceInfo.getHost())) { - contextBuilder.endpoint(UriUtil.HTTPS_PROTOCOL + UriUtil.DEFAULT_SCHEME_SEPARATOR + serviceInfo.getHost()); + if (StringUtils.isNotEmpty(host)) { + contextBuilder.endpoint(UriUtil.HTTPS_PROTOCOL + UriUtil.DEFAULT_SCHEME_SEPARATOR + host); } } protected JCloudsObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) { - return new JCloudsObjectStoreFileStorage(context.getBlobStore(), objectStoreServiceInfo.getContainer()); + return new JCloudsObjectStoreFileStorage(context.getBlobStore(), + (String) objectStoreServiceInfo.getCredentials() + .get(Constants.BUCKET)); } - protected GcpObjectStoreFileStorage createGcpFileStorage() { - Map credentials = getServiceCredentials(); - return new GcpObjectStoreFileStorage(credentials); + private Optional createObjectStoreBasedOnProvider(String objectStoreProviderName, + List providersServiceInfo, + Map exceptions) { + Optional objectStoreServiceInfoOptional = getAppropriateProvider(objectStoreProviderName, + providersServiceInfo); + if (objectStoreServiceInfoOptional.isEmpty()) { + LOGGER.warn(Messages.NO_OBJECTSTORE_PROVIDER_FOUND_FOR_0, objectStoreProviderName); + return Optional.empty(); + } + return tryToCreateObjectStore(objectStoreServiceInfoOptional.get(), exceptions); + } + + private Optional getAppropriateProvider(String objectStoreProviderName, + List providersServiceInfo) { + String appropriateProvider = Constants.ENV_TO_OS_PROVIDER.get(objectStoreProviderName); + return providersServiceInfo.stream() + .filter(provider -> appropriateProvider.equals(provider.getProvider())) + .findFirst(); + } + + private IllegalStateException buildNoValidObjectStoreException(Map exceptions) { + exceptions.forEach((provider, exception) -> LOGGER.error( + MessageFormat.format(Messages.CANNOT_CREATE_OBJECT_STORE_CLIENT_WITH_PROVIDER_0, provider), + exception)); + return new IllegalStateException(Messages.NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND); } @Override @@ -241,4 +240,4 @@ public Class getObjectType() { return FileStorage.class; } -} \ No newline at end of file +} diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java index 86c445e306..1d093527ad 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfo.java @@ -1,9 +1,9 @@ package org.cloudfoundry.multiapps.controller.web.configuration.service; -import com.google.common.base.Supplier; +import java.util.Map; + import org.cloudfoundry.multiapps.common.Nullable; import org.immutables.value.Value; -import org.jclouds.domain.Credentials; @Value.Immutable public interface ObjectStoreServiceInfo { @@ -11,23 +11,5 @@ public interface ObjectStoreServiceInfo { String getProvider(); @Nullable - String getIdentity(); - - @Nullable - Supplier getCredentialsSupplier(); - - @Nullable - String getCredential(); - - @Nullable - String getContainer(); - - @Nullable - String getEndpoint(); - - @Nullable - String getRegion(); - - @Nullable - String getHost(); + Map getCredentials(); } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java index 57fd802b88..0f0914b1b6 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java @@ -1,89 +1,56 @@ package org.cloudfoundry.multiapps.controller.web.configuration.service; -import java.net.MalformedURLException; -import java.net.URL; +import org.cloudfoundry.multiapps.controller.web.Constants; + +import java.util.HashMap; import java.util.List; import java.util.Map; -import org.cloudfoundry.multiapps.controller.web.Constants; -import org.cloudfoundry.multiapps.controller.web.Messages; - public class ObjectStoreServiceInfoCreator { public List getAllProvidersServiceInfo(Map credentials) { - return List.of(createServiceInfoForAws(credentials), createServiceInfoForAliCloud(credentials), - createServiceInfoForAzure(credentials), createServiceInfoForCcee(credentials)); + return List.of( + createServiceInfoForAws(credentials), + createServiceInfoForCcee(credentials), + createServiceInfoForAliCloud(credentials), + createServiceInfoForAzure(credentials), + createServiceInfoForGcp(credentials) + ); } private ObjectStoreServiceInfo createServiceInfoForAws(Map credentials) { - String accessKeyId = (String) credentials.get(Constants.ACCESS_KEY_ID); - String secretAccessKey = (String) credentials.get(Constants.SECRET_ACCESS_KEY); - String bucket = (String) credentials.get(Constants.BUCKET); - String host = (String) credentials.get(Constants.HOST); - return ImmutableObjectStoreServiceInfo.builder() - .provider(Constants.AWS_S_3) - .identity(accessKeyId) - .credential(secretAccessKey) - .container(bucket) - .host(host) - .build(); + return createObjectStoreServiceInfo(Constants.AWS_S_3, credentials); + } + + private ObjectStoreServiceInfo createServiceInfoForCcee(Map credentials) { + Map translated = new HashMap<>(credentials); + Object containerName = credentials.get(Constants.CONTAINER_NAME_WITH_DASH); + if (containerName != null) { + translated.put(Constants.BUCKET, containerName); + } + Object endpointUrl = credentials.get(Constants.ENDPOINT_URL); + if (endpointUrl != null) { + translated.put(Constants.ENDPOINT, endpointUrl); + } + return createObjectStoreServiceInfo(Constants.AWS_S_3, translated); } private ObjectStoreServiceInfo createServiceInfoForAliCloud(Map credentials) { - String accessKeyId = (String) credentials.get(Constants.ACCESS_KEY_ID); - String secretAccessKey = (String) credentials.get(Constants.SECRET_ACCESS_KEY); - String bucket = (String) credentials.get(Constants.BUCKET); - String region = (String) credentials.get(Constants.REGION); - String endpoint = (String) credentials.get(Constants.ENDPOINT); - return ImmutableObjectStoreServiceInfo.builder() - .provider(Constants.ALIYUN_OSS) - .identity(accessKeyId) - .credential(secretAccessKey) - .container(bucket) - .endpoint(endpoint) - .region(region) - .build(); + return createObjectStoreServiceInfo(Constants.ALIYUN_OSS, credentials); } private ObjectStoreServiceInfo createServiceInfoForAzure(Map credentials) { - String accountName = (String) credentials.get(Constants.ACCOUNT_NAME); - String sasToken = (String) credentials.get(Constants.SAS_TOKEN); - String containerName = (String) credentials.get(Constants.CONTAINER_NAME); - URL containerUrl = getContainerUriEndpoint(credentials); - return ImmutableObjectStoreServiceInfo.builder() - .provider(Constants.AZUREBLOB) - .identity(accountName) - .credential(sasToken) - .endpoint(containerUrl == null ? null : containerUrl.toString()) - .container(containerName) - .build(); + return createObjectStoreServiceInfo(Constants.AZUREBLOB, credentials); } - private ObjectStoreServiceInfo createServiceInfoForCcee(Map credentials) { - String accessKeyId = (String) credentials.get(Constants.ACCESS_KEY_ID); - String containerName = (String) credentials.get(Constants.CONTAINER_NAME_WITH_DASH); - String endpointUrl = (String) credentials.get(Constants.ENDPOINT_URL); - String region = (String) credentials.get(Constants.REGION); - String secretAccessKey = (String) credentials.get(Constants.SECRET_ACCESS_KEY); - return ImmutableObjectStoreServiceInfo.builder() - .provider(Constants.AWS_S_3) - .identity(accessKeyId) - .container(containerName) - .endpoint(endpointUrl) - .region(region) - .credential(secretAccessKey) - .build(); + private ObjectStoreServiceInfo createServiceInfoForGcp(Map credentials) { + return createObjectStoreServiceInfo(Constants.GOOGLE_CLOUD_STORAGE, credentials); } - private URL getContainerUriEndpoint(Map credentials) { - if (!credentials.containsKey(Constants.CONTAINER_URI)) { - return null; - } - try { - URL containerUri = new URL((String) credentials.get(Constants.CONTAINER_URI)); - return new URL(containerUri.getProtocol(), containerUri.getHost(), containerUri.getPort(), ""); - } catch (MalformedURLException e) { - throw new IllegalStateException(Messages.CANNOT_PARSE_CONTAINER_URI_OF_OBJECT_STORE, e); - } + private ObjectStoreServiceInfo createObjectStoreServiceInfo(String provider, Map credentials) { + return ImmutableObjectStoreServiceInfo.builder() + .provider(provider) + .credentials(credentials) + .build(); } } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/upload/AsyncUploadJobOrchestrator.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/upload/AsyncUploadJobOrchestrator.java index d41dd1c727..638328a686 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/upload/AsyncUploadJobOrchestrator.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/upload/AsyncUploadJobOrchestrator.java @@ -1,27 +1,10 @@ package org.cloudfoundry.multiapps.controller.web.upload; -import java.io.BufferedInputStream; -import java.io.InputStream; -import java.math.BigInteger; -import java.net.URI; -import java.text.MessageFormat; -import java.time.Duration; -import java.time.LocalDateTime; -import java.time.temporal.ChronoUnit; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - import jakarta.inject.Inject; import jakarta.inject.Named; import org.cloudfoundry.multiapps.common.SLException; import org.cloudfoundry.multiapps.common.util.MiscUtil; import org.cloudfoundry.multiapps.controller.api.model.UserCredentials; -import org.cloudfoundry.multiapps.controller.client.util.CheckedSupplier; -import org.cloudfoundry.multiapps.controller.client.util.ResilientOperationExecutor; import org.cloudfoundry.multiapps.controller.core.helpers.DescriptorParserFacadeFactory; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.util.FileUtils; @@ -36,6 +19,7 @@ import org.cloudfoundry.multiapps.controller.web.upload.client.DeployFromUrlRemoteClient; import org.cloudfoundry.multiapps.controller.web.upload.client.FileFromUrlData; import org.cloudfoundry.multiapps.controller.web.upload.exception.RejectedAsyncUploadJobException; +import org.cloudfoundry.multiapps.controller.web.upload.resilience.FileUploadResilientOperationExecutor; import org.cloudfoundry.multiapps.controller.web.util.SecurityContextUtil; import org.cloudfoundry.multiapps.mta.handlers.ArchiveHandler; import org.cloudfoundry.multiapps.mta.handlers.DescriptorParserFacade; @@ -43,6 +27,21 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedInputStream; +import java.io.InputStream; +import java.math.BigInteger; +import java.net.URI; +import java.text.MessageFormat; +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + @Named public class AsyncUploadJobOrchestrator { @@ -53,7 +52,7 @@ public class AsyncUploadJobOrchestrator { private static final Logger LOGGER = LoggerFactory.getLogger(AsyncUploadJobOrchestrator.class); - private final ResilientOperationExecutor resilientOperationExecutor = getResilientOperationExecutor(); + private final FileUploadResilientOperationExecutor fileUploadResilientOperationExecutor = getFileUploadResilientOperationExecutor(); private final ExecutorService asyncFileUploadExecutor; private final ExecutorService deployFromUrlExecutor; @@ -129,7 +128,8 @@ private void deployFromUrl(AsyncUploadJobEntry jobEntry, String fileUrl, UserCre .singleResult(); monitorAsyncUploadJob(updatedJobEntry, lock, counterRef); } catch (Exception e) { - LOGGER.error(MessageFormat.format(Messages.ASYNC_UPLOAD_JOB_FAILED, jobEntry.getId(), e.getMessage()), e); + LOGGER.error(MessageFormat.format(Messages.ASYNC_UPLOAD_JOB_FAILED, jobEntry.getId(), + extractFileName(jobEntry.getUrl()), e.getMessage()), e); updateFailedAsyncUploadJob(jobEntry, e, lock); } } @@ -139,7 +139,11 @@ private void startAsyncUploadFromUrlUpload(UploadFromUrlContext uploadFromUrlCon try { startSyncUploadFromUrlUpload(uploadFromUrlContext, lock); } catch (Exception e) { - LOGGER.error(e.getMessage(), e); + LOGGER.error(MessageFormat.format(Messages.ASYNC_UPLOAD_JOB_FAILED, uploadFromUrlContext.getJobEntry() + .getId(), + extractFileName(uploadFromUrlContext.getJobEntry() + .getUrl()), + e.getMessage()), e); updateFailedAsyncUploadJob(uploadFromUrlContext.getJobEntry(), e, lock); throw new SLException(e, e.getMessage()); } @@ -147,8 +151,7 @@ private void startAsyncUploadFromUrlUpload(UploadFromUrlContext uploadFromUrlCon } private void startSyncUploadFromUrlUpload(UploadFromUrlContext uploadFromUrlContext, Lock lock) throws Exception { - FileEntry fileEntry = resilientOperationExecutor.execute( - (CheckedSupplier) () -> doUploadMtarFromUrl(uploadFromUrlContext, lock)); + FileEntry fileEntry = fileUploadResilientOperationExecutor.execute(() -> doUploadMtarFromUrl(uploadFromUrlContext, lock)); LOGGER.trace(Messages.UPLOADED_MTAR_FROM_REMOTE_ENDPOINT_AND_JOB_ID, uploadFromUrlContext.getJobEntry() .getUrl(), uploadFromUrlContext.getJobEntry() @@ -248,6 +251,8 @@ private void monitorAsyncUploadJob(AsyncUploadJobEntry updatedJobEntry, Lock loc } finally { lock.unlock(); } + LOGGER.info(Messages.ASYNC_UPLOAD_JOB_MONITOR_UPDATE, updatedJobEntry.getId(), updatedJobEntry.getState(), + updatedJobEntry.getBytesRead(), updatedJobEntry.getUpdatedAt()); waitBetweenUpdates(); } } @@ -265,15 +270,18 @@ private void updateFailedAsyncUploadJob(AsyncUploadJobEntry jobEntry, Exception asyncUploadJobService.update(failedEntry, ImmutableAsyncUploadJobEntry.copyOf(failedEntry) .withUpdatedAt(LocalDateTime.now()) .withFinishedAt(LocalDateTime.now()) - .withError(e.getMessage()) + .withError( + MessageFormat.format(Messages.ASYNC_UPLOAD_JOB_ERROR, + e.getMessage(), + jobEntry.getId())) .withState(AsyncUploadJobEntry.State.ERROR)); } finally { lock.unlock(); } } - protected ResilientOperationExecutor getResilientOperationExecutor() { - return new ResilientOperationExecutor(); + protected FileUploadResilientOperationExecutor getFileUploadResilientOperationExecutor() { + return new FileUploadResilientOperationExecutor(); } } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/upload/resilience/FileUploadResilientOperationExecutor.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/upload/resilience/FileUploadResilientOperationExecutor.java new file mode 100644 index 0000000000..96626294e3 --- /dev/null +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/upload/resilience/FileUploadResilientOperationExecutor.java @@ -0,0 +1,68 @@ +package org.cloudfoundry.multiapps.controller.web.upload.resilience; + +import org.cloudfoundry.multiapps.common.util.MiscUtil; +import org.cloudfoundry.multiapps.controller.client.util.CheckedSupplier; +import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; +import org.cloudfoundry.multiapps.controller.web.Messages; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.text.MessageFormat; +import java.time.Duration; + +public class FileUploadResilientOperationExecutor { + + private static final Logger LOGGER = LoggerFactory.getLogger(FileUploadResilientOperationExecutor.class); + + private static final int MAX_ATTEMPTS = 7; + private static final int BACKOFF_FACTOR = 2; + private static final long INITIAL_WAIT_TIME_IN_MILLIS = Duration.ofSeconds(10) + .toMillis(); + private static final long MAX_WAIT_TIME_IN_MILLIS = Duration.ofSeconds(300) + .toMillis(); + + public T execute(CheckedSupplier operation) throws Exception { + Exception lastException = null; + for (int attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + try { + return operation.get(); + } catch (Exception e) { + if (!isTransientError(e)) { + throw e; + } + lastException = e; + if (attempt == MAX_ATTEMPTS) { + LOGGER.error(MessageFormat.format(Messages.FILE_UPLOAD_ALL_ATTEMPTS_EXHAUSTED, MAX_ATTEMPTS, e.getMessage()), e); + throw e; + } + long waitTime = calculateWaitTime(attempt); + LOGGER.warn(MessageFormat.format(Messages.FILE_UPLOAD_ATTEMPT_FAILED, attempt, MAX_ATTEMPTS, waitTime, e.getMessage()), e); + sleep(waitTime); + } + } + throw lastException; + } + + protected void sleep(long millis) { + MiscUtil.sleep(millis); + } + + private boolean isTransientError(Exception e) { + Throwable cause = e instanceof FileStorageException ? e.getCause() : e; + while (cause != null) { + if (cause instanceof IOException || cause instanceof UncheckedIOException) { + return true; + } + cause = cause.getCause(); + } + return false; + } + + private long calculateWaitTime(int attempt) { + long waitTime = (long) (INITIAL_WAIT_TIME_IN_MILLIS * Math.pow(BACKOFF_FACTOR, (double) attempt - 1)); + return Math.min(waitTime, MAX_WAIT_TIME_IN_MILLIS); + } + +} diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java index 5b5617b7a6..ac923d858e 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java @@ -1,43 +1,51 @@ package org.cloudfoundry.multiapps.controller.web.configuration.bean.factory; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - import io.pivotal.cfenv.core.CfCredentials; import io.pivotal.cfenv.core.CfService; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; +import org.cloudfoundry.multiapps.controller.persistence.services.AwsS3ObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder; import org.cloudfoundry.multiapps.controller.web.Constants; import org.cloudfoundry.multiapps.controller.web.Messages; +import org.cloudfoundry.multiapps.controller.web.configuration.service.ImmutableObjectStoreServiceInfo; import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfo; import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfoCreator; import org.jclouds.blobstore.BlobStoreContext; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; class ObjectStoreFileStorageFactoryBeanTest { + private static final String SERVICE_NAME = "deploy-service-os"; private static final String ACCESS_KEY_ID_VALUE = "access_key_id_value"; private static final String SECRET_ACCESS_KEY_VALUE = "secret_access_key_value"; private static final String BUCKET_VALUE = "bucket_value"; @@ -50,93 +58,163 @@ class ObjectStoreFileStorageFactoryBeanTest { private ApplicationConfiguration applicationConfiguration; @Mock private JCloudsObjectStoreFileStorage jCloudsObjectStoreFileStorage; - @Mock private GcpObjectStoreFileStorage gcpObjectStoreFileStorage; + @Mock + private AwsS3ObjectStoreFileStorage awsS3ObjectStoreFileStorage; @BeforeEach void setUp() throws Exception { MockitoAnnotations.openMocks(this) .close(); when(applicationConfiguration.getObjectStoreRegions()).thenReturn(Set.of()); - objectStoreFileStorageFactoryBean = new ObjectStoreFileStorageFactoryBeanMock("deploy-service-os", environmentServicesFinder, + objectStoreFileStorageFactoryBean = new ObjectStoreFileStorageFactoryBeanMock(SERVICE_NAME, environmentServicesFinder, applicationConfiguration); } + @Test + void testGetObjectType() { + assertEquals(FileStorage.class, objectStoreFileStorageFactoryBean.getObjectType()); + } + + @Test + void testGetObjectBeforeInitialization() { + assertNull(objectStoreFileStorageFactoryBean.getObject()); + } + @Test void testObjectStoreCreationWithoutServiceInstance() { objectStoreFileStorageFactoryBean.afterPropertiesSet(); - FileStorage objectStoreFileStorage = objectStoreFileStorageFactoryBean.getObject(); - assertNull(objectStoreFileStorage); + assertNull(objectStoreFileStorageFactoryBean.getObject()); } @Test void testObjectStoreCreationWithValidServiceInstance() { mockCfService(); objectStoreFileStorageFactoryBean.afterPropertiesSet(); - FileStorage objectStoreFileStorage = objectStoreFileStorageFactoryBean.getObject(); - assertNotNull(objectStoreFileStorage); + assertNotNull(objectStoreFileStorageFactoryBean.getObject()); } @Test - void testObjectStoreCreationWhenEnvIsValid() { + void testObjectStoreCreationWhenEnvIsValidForAws() { mockCfService(); when(applicationConfiguration.getObjectStoreClientType()).thenReturn(Constants.AWS); - ObjectStoreFileStorageFactoryBean spy = spy(objectStoreFileStorageFactoryBean); + var factoryBeanSpy = spy(objectStoreFileStorageFactoryBean); - spy.afterPropertiesSet(); - FileStorage createdObjectStoreFileStorage = spy.getObject(); + factoryBeanSpy.afterPropertiesSet(); + var result = factoryBeanSpy.getObject(); - assertNotNull(createdObjectStoreFileStorage); - verify(spy, never()) - .createObjectStoreFromFirstReachableProvider(anyMap(), anyList()); - verify(jCloudsObjectStoreFileStorage, times(1)) - .testConnection(); + assertInstanceOf(AwsS3ObjectStoreFileStorage.class, result); + verify(factoryBeanSpy, never()).createObjectStoreFromFirstReachableProvider(anyMap(), anyList()); + verify(awsS3ObjectStoreFileStorage).testConnection(); } @Test - void testObjectStoreCreationWhenEnvIsInvalid() { + void testObjectStoreCreationWhenEnvIsValidForGcp() { + mockCfService(); + when(applicationConfiguration.getObjectStoreClientType()).thenReturn(Constants.GCP); + var factoryBeanSpy = spy(objectStoreFileStorageFactoryBean); + + factoryBeanSpy.afterPropertiesSet(); + var result = factoryBeanSpy.getObject(); + + assertInstanceOf(GcpObjectStoreFileStorage.class, result); + verify(factoryBeanSpy, never()).createObjectStoreFromFirstReachableProvider(anyMap(), anyList()); + verify(gcpObjectStoreFileStorage).testConnection(); + } + + static Stream testObjectStoreCreationFallsBackToFirstReachableProviderWhenEnvIsInvalid() { + return Stream.of( + // @formatter:off + // (0) Unknown provider name: + Arguments.of("WRONG_PROVIDER"), + // (1) Null env value: + Arguments.of((String) null), + // (2) Empty env value: + Arguments.of("") + // @formatter:on + ); + } + + @ParameterizedTest + @MethodSource + void testObjectStoreCreationFallsBackToFirstReachableProviderWhenEnvIsInvalid(String envValue) { mockCfService(); - when(applicationConfiguration.getObjectStoreClientType()).thenReturn("WRONG_PROVIDER"); + when(applicationConfiguration.getObjectStoreClientType()).thenReturn(envValue); + var factoryBeanSpy = spy(objectStoreFileStorageFactoryBean); + + factoryBeanSpy.afterPropertiesSet(); + var result = factoryBeanSpy.getObject(); - ObjectStoreFileStorageFactoryBean spy = spy(objectStoreFileStorageFactoryBean); + assertInstanceOf(AwsS3ObjectStoreFileStorage.class, result); + verify(factoryBeanSpy).createObjectStoreFromFirstReachableProvider(anyMap(), anyList()); + } + + @Test + void testObjectStoreCreationFallsBackToNextProviderWhenFirstFails() { + mockCfService(); + when(applicationConfiguration.getObjectStoreClientType()).thenReturn(null); + doThrow(new IllegalStateException("AWS connection failed")).when(awsS3ObjectStoreFileStorage) + .testConnection(); - spy.afterPropertiesSet(); - FileStorage createdObjectStoreFileStorage = spy.getObject(); + objectStoreFileStorageFactoryBean.afterPropertiesSet(); + var result = objectStoreFileStorageFactoryBean.getObject(); - assertNotNull(createdObjectStoreFileStorage); - verify(spy, times(1)) - .createObjectStoreFromFirstReachableProvider(anyMap(), anyList()); + // AWS (two entries) fails, then Aliyun (JClouds default path) succeeds + assertInstanceOf(JCloudsObjectStoreFileStorage.class, result); + verify(jCloudsObjectStoreFileStorage).testConnection(); } @Test void testObjectStoreCreationWhenEnvProviderFailsToConnect() { mockCfService(); when(applicationConfiguration.getObjectStoreClientType()).thenReturn(Constants.AWS); - doThrow(new IllegalStateException("Cannot create object store")).when(jCloudsObjectStoreFileStorage) + doThrow(new IllegalStateException("Cannot create object store")).when(awsS3ObjectStoreFileStorage) .testConnection(); - Exception exception = assertThrows(IllegalStateException.class, () -> objectStoreFileStorageFactoryBean.afterPropertiesSet()); + var exception = assertThrows(IllegalStateException.class, () -> objectStoreFileStorageFactoryBean.afterPropertiesSet()); assertEquals(Messages.NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND, exception.getMessage()); } @Test - void testObjectStoreCreationWithoutValidServiceInstance() { + void testObjectStoreCreationWhenEnvProviderNotFoundInAvailableProviders() { mockCfService(); + when(applicationConfiguration.getObjectStoreClientType()).thenReturn(Constants.GCP); + var factoryBeanSpy = spy(objectStoreFileStorageFactoryBean); + doReturn(List.of(buildServiceInfo(Constants.AWS_S_3))).when(factoryBeanSpy) + .getProvidersServiceInfo(); + + var exception = assertThrows(IllegalStateException.class, factoryBeanSpy::afterPropertiesSet); + assertEquals(Messages.NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND, exception.getMessage()); + } + + @Test + void testObjectStoreCreationWhenAllProvidersFail() { + mockCfService(); + doThrow(new IllegalStateException("Cannot create object store")).when(awsS3ObjectStoreFileStorage) + .testConnection(); doThrow(new IllegalStateException("Cannot create object store")).when(jCloudsObjectStoreFileStorage) .testConnection(); doThrow(new IllegalStateException("Cannot create object store")).when(gcpObjectStoreFileStorage) .testConnection(); - Exception exception = assertThrows(IllegalStateException.class, () -> objectStoreFileStorageFactoryBean.afterPropertiesSet()); + + var exception = assertThrows(IllegalStateException.class, () -> objectStoreFileStorageFactoryBean.afterPropertiesSet()); assertEquals(Messages.NO_VALID_OBJECT_STORE_CONFIGURATION_FOUND, exception.getMessage()); } + private ObjectStoreServiceInfo buildServiceInfo(String provider) { + return ImmutableObjectStoreServiceInfo.builder() + .provider(provider) + .credentials(buildCredentials()) + .build(); + } + private void mockCfService() { CfService cfService = Mockito.mock(CfService.class); CfCredentials cfCredentials = Mockito.mock(CfCredentials.class); when(cfCredentials.getMap()).thenReturn(buildCredentials()); when(cfService.getCredentials()).thenReturn(cfCredentials); - when(environmentServicesFinder.findService("deploy-service-os")).thenReturn(cfService); + when(environmentServicesFinder.findService(SERVICE_NAME)).thenReturn(cfService); } private static Map buildCredentials() { @@ -160,22 +238,24 @@ protected JCloudsObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo } @Override - protected GcpObjectStoreFileStorage createGcpFileStorage() { + protected GcpObjectStoreFileStorage createGcpFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { return ObjectStoreFileStorageFactoryBeanTest.this.gcpObjectStoreFileStorage; } + @Override + protected AwsS3ObjectStoreFileStorage createAwsS3FileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { + return ObjectStoreFileStorageFactoryBeanTest.this.awsS3ObjectStoreFileStorage; + } + @Override public List getProvidersServiceInfo() { - CfService service = environmentServicesFinder.findService("deploy-service-os"); + CfService service = environmentServicesFinder.findService(SERVICE_NAME); if (service != null) { - return new ObjectStoreServiceInfoCreatorMock().getAllProvidersServiceInfo(service.getCredentials() - .getMap()); - } else { - return List.of(); + return new ObjectStoreServiceInfoCreator().getAllProvidersServiceInfo(service.getCredentials() + .getMap()); } + return List.of(); } } - private class ObjectStoreServiceInfoCreatorMock extends ObjectStoreServiceInfoCreator { - } } diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java index 6439250543..fc2b5533cf 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreatorTest.java @@ -1,7 +1,5 @@ package org.cloudfoundry.multiapps.controller.web.configuration.service; -import java.net.MalformedURLException; -import java.net.URL; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -26,10 +24,10 @@ class ObjectStoreServiceInfoCreatorTest { private static final String BUCKET_VALUE = "bucket_value"; private static final String REGION_VALUE = "region_value"; private static final String ENDPOINT_VALUE = "endpoint_value"; - private static final String ACCOUNT_NAME_VALUE = "account_name_value"; private static final String SAS_TOKEN_VALUE = "sas_token_value"; - private static final String CONTAINER_NAME_VALUE = "container_name_value"; - private static final String CONTAINER_URI_VALUE = "https://container.com:8080"; + private static final String BASE64_PRIVATE_KEY_VALUE = "base64EncodedPrivateKeyDataValue"; + private static final Map AZURE_CREDENTIALS = Map.of(Constants.SAS_TOKEN, SAS_TOKEN_VALUE); + private static final Map GCP_CREDENTIALS = Map.of(Constants.BASE_64_ENCODED_PRIVATE_KEY_DATA, BASE64_PRIVATE_KEY_VALUE); private ObjectStoreServiceInfoCreator objectStoreServiceInfoCreator; @@ -38,10 +36,12 @@ void setUp() { objectStoreServiceInfoCreator = new ObjectStoreServiceInfoCreatorMock(); } - static Stream testDifferentProviders() throws MalformedURLException { + static Stream testDifferentProviders() { return Stream.of(Arguments.of(buildCfService(buildAliCloudCredentials()), buildAliCloudObjectStoreServiceInfo()), Arguments.of(buildCfService(buildAwsCredentials()), buildAwsObjectStoreServiceInfo()), - Arguments.of(buildCfService(buildAzureCredentials()), buildAzureObjectStoreServiceInfo())); + Arguments.of(buildCfService(buildCceeCredentials()), buildCceeObjectStoreServiceInfo()), + Arguments.of(buildCfService(AZURE_CREDENTIALS), buildAzureObjectStoreServiceInfo()), + Arguments.of(buildCfService(GCP_CREDENTIALS), buildGcpObjectStoreServiceInfo())); } @ParameterizedTest @@ -74,11 +74,7 @@ private static Map buildAliCloudCredentials() { private static ObjectStoreServiceInfo buildAliCloudObjectStoreServiceInfo() { return ImmutableObjectStoreServiceInfo.builder() .provider(Constants.ALIYUN_OSS) - .identity(ACCESS_KEY_ID_VALUE) - .credential(SECRET_ACCESS_KEY_VALUE) - .container(BUCKET_VALUE) - .endpoint(ENDPOINT_VALUE) - .region(REGION_VALUE) + .credentials(buildAliCloudCredentials()) .build(); } @@ -93,28 +89,41 @@ private static Map buildAwsCredentials() { private static ObjectStoreServiceInfo buildAwsObjectStoreServiceInfo() { return ImmutableObjectStoreServiceInfo.builder() .provider(Constants.AWS_S_3) - .identity(ACCESS_KEY_ID_VALUE) - .credential(SECRET_ACCESS_KEY_VALUE) - .container(BUCKET_VALUE) + .credentials(buildAwsCredentials()) .build(); } - private static Map buildAzureCredentials() { + private static Map buildCceeCredentials() { Map credentials = new HashMap<>(); - credentials.put(Constants.ACCOUNT_NAME, ACCOUNT_NAME_VALUE); - credentials.put(Constants.SAS_TOKEN, SAS_TOKEN_VALUE); - credentials.put(Constants.CONTAINER_NAME, CONTAINER_NAME_VALUE); - credentials.put(Constants.CONTAINER_URI, CONTAINER_URI_VALUE); + credentials.put(Constants.ACCESS_KEY_ID, ACCESS_KEY_ID_VALUE); + credentials.put(Constants.SECRET_ACCESS_KEY, SECRET_ACCESS_KEY_VALUE); + credentials.put(Constants.CONTAINER_NAME_WITH_DASH, BUCKET_VALUE); + credentials.put(Constants.ENDPOINT_URL, ENDPOINT_VALUE); + credentials.put(Constants.REGION, REGION_VALUE); return credentials; } - private static ObjectStoreServiceInfo buildAzureObjectStoreServiceInfo() throws MalformedURLException { + private static ObjectStoreServiceInfo buildCceeObjectStoreServiceInfo() { + Map expectedCredentials = new HashMap<>(buildCceeCredentials()); + expectedCredentials.put(Constants.BUCKET, BUCKET_VALUE); + expectedCredentials.put(Constants.ENDPOINT, ENDPOINT_VALUE); + return ImmutableObjectStoreServiceInfo.builder() + .provider(Constants.AWS_S_3) + .credentials(expectedCredentials) + .build(); + } + + private static ObjectStoreServiceInfo buildAzureObjectStoreServiceInfo() { return ImmutableObjectStoreServiceInfo.builder() .provider(Constants.AZUREBLOB) - .identity(ACCOUNT_NAME_VALUE) - .credential(SAS_TOKEN_VALUE) - .endpoint(new URL("https", "container.com", 8080, "").toString()) - .container(CONTAINER_NAME_VALUE) + .credentials(AZURE_CREDENTIALS) + .build(); + } + + private static ObjectStoreServiceInfo buildGcpObjectStoreServiceInfo() { + return ImmutableObjectStoreServiceInfo.builder() + .provider(Constants.GOOGLE_CLOUD_STORAGE) + .credentials(GCP_CREDENTIALS) .build(); } diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/upload/AsyncUploadJobOrchestratorTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/upload/AsyncUploadJobOrchestratorTest.java index c0533e5ac2..edd8576e9a 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/upload/AsyncUploadJobOrchestratorTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/upload/AsyncUploadJobOrchestratorTest.java @@ -10,7 +10,6 @@ import org.cloudfoundry.multiapps.common.SLException; import org.cloudfoundry.multiapps.controller.api.model.UserCredentials; import org.cloudfoundry.multiapps.controller.client.util.CheckedSupplier; -import org.cloudfoundry.multiapps.controller.client.util.ResilientOperationExecutor; import org.cloudfoundry.multiapps.controller.core.helpers.DescriptorParserFacadeFactory; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.persistence.model.AsyncUploadJobEntry; @@ -22,6 +21,7 @@ import org.cloudfoundry.multiapps.controller.persistence.services.FileService; import org.cloudfoundry.multiapps.controller.web.upload.client.DeployFromUrlRemoteClient; import org.cloudfoundry.multiapps.controller.web.upload.client.FileFromUrlData; +import org.cloudfoundry.multiapps.controller.web.upload.resilience.FileUploadResilientOperationExecutor; import org.cloudfoundry.multiapps.controller.web.util.SecurityContextUtil; import org.cloudfoundry.multiapps.mta.handlers.DescriptorParserFacade; import org.cloudfoundry.multiapps.mta.model.DeploymentDescriptor; @@ -445,8 +445,8 @@ protected void waitBetweenUpdates() { } @Override - protected ResilientOperationExecutor getResilientOperationExecutor() { - return new ResilientOperationExecutor() { + protected FileUploadResilientOperationExecutor getFileUploadResilientOperationExecutor() { + return new FileUploadResilientOperationExecutor() { @Override public T execute(CheckedSupplier operation) throws Exception { return operation.get(); diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/upload/FileUploadResilientOperationExecutorTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/upload/FileUploadResilientOperationExecutorTest.java new file mode 100644 index 0000000000..b00f9e038c --- /dev/null +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/upload/FileUploadResilientOperationExecutorTest.java @@ -0,0 +1,207 @@ +package org.cloudfoundry.multiapps.controller.web.upload; + +import org.cloudfoundry.multiapps.common.SLException; +import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; +import org.cloudfoundry.multiapps.controller.web.upload.resilience.FileUploadResilientOperationExecutor; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.exception.SdkException; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class FileUploadResilientOperationExecutorTest { + + private List recordedSleepDurations; + private FileUploadResilientOperationExecutor executor; + + @BeforeEach + void setUp() { + recordedSleepDurations = new ArrayList<>(); + executor = new FileUploadResilientOperationExecutor() { + @Override + protected void sleep(long millis) { + recordedSleepDurations.add(millis); + } + }; + } + + @Test + void testSuccessfulFirstAttempt() throws Exception { + var result = executor.execute(() -> "success"); + + assertEquals("success", result); + assertTrue(recordedSleepDurations.isEmpty()); + } + + @Test + void testRetryOnFileStorageExceptionWithIOExceptionCause() throws Exception { + var attempts = new AtomicInteger(0); + + var result = executor.execute(() -> { + if (attempts.incrementAndGet() < 3) { + throw new FileStorageException("Storage failed", new IOException("Connection reset")); + } + return "recovered"; + }); + + assertEquals("recovered", result); + assertEquals(2, recordedSleepDurations.size()); + assertEquals(10000L, recordedSleepDurations.get(0)); + assertEquals(20000L, recordedSleepDurations.get(1)); + } + + @Test + void testNoRetryOnFileStorageExceptionWithSdkExceptionCause() { + var attempts = new AtomicInteger(0); + + assertThrows(FileStorageException.class, () -> executor.execute(() -> { + attempts.incrementAndGet(); + throw new FileStorageException("S3 error", SdkException.builder() + .message("Service unavailable") + .build()); + })); + + assertEquals(1, attempts.get()); + assertTrue(recordedSleepDurations.isEmpty()); + } + + @Test + void testRetryOnFileStorageExceptionWithSdkExceptionWrappingIOException() throws Exception { + var attempts = new AtomicInteger(0); + + var result = executor.execute(() -> { + if (attempts.incrementAndGet() < 2) { + // Mirrors what AWS SDK v2 does: SdkClientException wrapping a network IOException + var networkError = new IOException("Connection reset by peer"); + var sdkException = SdkException.builder() + .message("network error") + .cause(networkError) + .build(); + throw new FileStorageException("S3 upload failed", sdkException); + } + return "recovered"; + }); + + assertEquals("recovered", result); + assertEquals(1, recordedSleepDurations.size()); + assertEquals(10000L, recordedSleepDurations.get(0)); + } + + @Test + void testRetryOnFileStorageExceptionWithUncheckedIOExceptionCause() throws Exception { + var attempts = new AtomicInteger(0); + + var result = executor.execute(() -> { + if (attempts.incrementAndGet() < 2) { + throw new FileStorageException("Wrapped IO", new UncheckedIOException(new IOException("broken pipe"))); + } + return "recovered"; + }); + + assertEquals("recovered", result); + assertEquals(1, recordedSleepDurations.size()); + assertEquals(10000L, recordedSleepDurations.get(0)); + } + + @Test + void testRetryOnDirectIOException() throws Exception { + var attempts = new AtomicInteger(0); + + var result = executor.execute(() -> { + if (attempts.incrementAndGet() < 2) { + throw new IOException("Network failure"); + } + return "recovered"; + }); + + assertEquals("recovered", result); + assertEquals(1, recordedSleepDurations.size()); + assertEquals(10000L, recordedSleepDurations.get(0)); + } + + @Test + void testRetryOnDirectUncheckedIOException() throws Exception { + var attempts = new AtomicInteger(0); + + var result = executor.execute(() -> { + if (attempts.incrementAndGet() < 2) { + throw new UncheckedIOException(new IOException("timeout")); + } + return "recovered"; + }); + + assertEquals("recovered", result); + assertEquals(1, recordedSleepDurations.size()); + assertEquals(10000L, recordedSleepDurations.get(0)); + } + + @Test + void testNoRetryOnSLException() { + var attempts = new AtomicInteger(0); + + var exception = assertThrows(SLException.class, () -> executor.execute(() -> { + attempts.incrementAndGet(); + throw new SLException("Validation error"); + })); + + assertEquals(1, attempts.get()); + assertEquals("Validation error", exception.getMessage()); + assertTrue(recordedSleepDurations.isEmpty()); + } + + @Test + void testNoRetryOnFileStorageExceptionWithNonTransientCause() { + var attempts = new AtomicInteger(0); + + assertThrows(FileStorageException.class, () -> executor.execute(() -> { + attempts.incrementAndGet(); + throw new FileStorageException("Bad request", new IllegalArgumentException("invalid key")); + })); + + assertEquals(1, attempts.get()); + assertTrue(recordedSleepDurations.isEmpty()); + } + + @Test + void testExponentialBackoffTiming() throws Exception { + var attempts = new AtomicInteger(0); + + var result = executor.execute(() -> { + if (attempts.incrementAndGet() < 6) { + throw new IOException("transient"); + } + return "finally"; + }); + + assertEquals("finally", result); + assertEquals(5, recordedSleepDurations.size()); + assertEquals(10000L, recordedSleepDurations.get(0)); + assertEquals(20000L, recordedSleepDurations.get(1)); + assertEquals(40000L, recordedSleepDurations.get(2)); + assertEquals(80000L, recordedSleepDurations.get(3)); + assertEquals(160000L, recordedSleepDurations.get(4)); + } + + @Test + void testAllAttemptsExhausted() { + var attempts = new AtomicInteger(0); + + var exception = assertThrows(IOException.class, () -> executor.execute(() -> { + attempts.incrementAndGet(); + throw new IOException("persistent failure"); + })); + + assertEquals(7, attempts.get()); + assertEquals("persistent failure", exception.getMessage()); + assertEquals(6, recordedSleepDurations.size()); + } + +} diff --git a/pom.xml b/pom.xml index 7376507892..1f39e3f4f6 100644 --- a/pom.xml +++ b/pom.xml @@ -48,6 +48,7 @@ 2.12.1 1.16.4 3.18.5 + 2.44.12 1.1.1 1.21.0 @@ -63,8 +64,6 @@ 1.3.6 2.64.1 0.128.14 - 12.33.3 - 1.13.3 4.0.1 6.3.0 @@ -317,16 +316,6 @@ ${google-cloud-nio.version} test - - com.azure - azure-storage-blob - ${azure-storage-blob.version} - - - com.azure - azure-core-http-okhttp - ${azure-core-http-okhttp.version} - org.immutables @@ -750,12 +739,6 @@ jclouds-blobstore ${jclouds.version} - - - org.apache.jclouds.provider - aws-s3 - ${jclouds.version} - @@ -811,6 +794,18 @@ auto-service-annotations ${auto-service.version} + + + software.amazon.awssdk + s3 + ${aws.sdk.version} + + + + software.amazon.awssdk + url-connection-client + ${aws.sdk.version} +