diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index 4cd474e861..3411f802eb 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -16,6 +16,7 @@ exports org.cloudfoundry.multiapps.controller.persistence.services; exports org.cloudfoundry.multiapps.controller.persistence.util; exports org.cloudfoundry.multiapps.controller.persistence.stream; + exports org.cloudfoundry.multiapps.controller.persistence.query.options; requires transitive io.pivotal.cfenv.core; requires transitive io.pivotal.cfenv.jdbc; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/dialects/DataSourceDialect.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/dialects/DataSourceDialect.java index 8f2fbb98d2..c296e71f41 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/dialects/DataSourceDialect.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/dialects/DataSourceDialect.java @@ -6,10 +6,14 @@ import java.sql.ResultSet; import java.sql.SQLException; +import org.cloudfoundry.multiapps.controller.persistence.query.options.StreamFetchingOptions; + public interface DataSourceDialect { String getSequenceNextValueSyntax(String sequenceName); + InputStream getBinaryStreamFromBlob(ResultSet rs, String columnName, StreamFetchingOptions streamFetchingOptions) throws SQLException; + InputStream getBinaryStreamFromBlob(ResultSet rs, String columnName) throws SQLException; void setBlobAsBinaryStream(PreparedStatement ps, int index, InputStream is) throws SQLException; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/dialects/DefaultDataSourceDialect.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/dialects/DefaultDataSourceDialect.java index 6ab30c5fe1..47d9cd137b 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/dialects/DefaultDataSourceDialect.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/dialects/DefaultDataSourceDialect.java @@ -6,6 +6,8 @@ import java.sql.ResultSet; import java.sql.SQLException; +import org.cloudfoundry.multiapps.controller.persistence.query.options.StreamFetchingOptions; + public class DefaultDataSourceDialect implements DataSourceDialect { @Override @@ -19,6 +21,17 @@ public InputStream getBinaryStreamFromBlob(ResultSet rs, String columnName) thro .getBinaryStream(); } + @Override + public InputStream getBinaryStreamFromBlob(ResultSet rs, String columnName, StreamFetchingOptions streamFetchingOptions) + throws SQLException { + return rs.getBlob(columnName) + // + 1 is required as the first position is 1 instead of 0 + // pos – the offset to the first byte of the partial value to be retrieved. The first byte in the Blob is at + // position 1. + .getBinaryStream(streamFetchingOptions.startOffset() + 1, + streamFetchingOptions.endOffset() - streamFetchingOptions.startOffset() + 1); + } + @Override public void setBlobAsBinaryStream(PreparedStatement ps, int index, InputStream is) throws SQLException { ps.setBlob(index, is); diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/query/options/StreamFetchingOptions.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/query/options/StreamFetchingOptions.java new file mode 100644 index 0000000000..c4277351b4 --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/query/options/StreamFetchingOptions.java @@ -0,0 +1,4 @@ +package org.cloudfoundry.multiapps.controller.persistence.query.options; + +public record StreamFetchingOptions(long startOffset, long endOffset) { +} diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/query/providers/BlobSqlFileQueryProvider.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/query/providers/BlobSqlFileQueryProvider.java index 6190d4d735..cd1049d319 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/query/providers/BlobSqlFileQueryProvider.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/query/providers/BlobSqlFileQueryProvider.java @@ -6,6 +6,7 @@ import java.sql.SQLException; import org.cloudfoundry.multiapps.controller.persistence.dialects.DataSourceDialect; +import org.cloudfoundry.multiapps.controller.persistence.query.options.StreamFetchingOptions; public class BlobSqlFileQueryProvider extends SqlFileQueryProvider { @@ -23,4 +24,10 @@ protected InputStream getContentBinaryStream(ResultSet resultSet, String columnN return getDataSourceDialect().getBinaryStreamFromBlob(resultSet, columnName); } + @Override + protected InputStream getContentBinaryStreamWithOffset(ResultSet resultSet, String columnName, + StreamFetchingOptions streamFetchingOptions) + throws SQLException { + return getDataSourceDialect().getBinaryStreamFromBlob(resultSet, columnName, streamFetchingOptions); + } } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/query/providers/ByteArraySqlFileQueryProvider.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/query/providers/ByteArraySqlFileQueryProvider.java index 995ce2f057..aae6c777d9 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/query/providers/ByteArraySqlFileQueryProvider.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/query/providers/ByteArraySqlFileQueryProvider.java @@ -6,6 +6,7 @@ import java.sql.SQLException; import org.cloudfoundry.multiapps.controller.persistence.dialects.DataSourceDialect; +import org.cloudfoundry.multiapps.controller.persistence.query.options.StreamFetchingOptions; public class ByteArraySqlFileQueryProvider extends SqlFileQueryProvider { @@ -30,4 +31,9 @@ protected InputStream getContentBinaryStream(ResultSet resultSet, String columnN return getDataSourceDialect().getBinaryStreamFromByteArray(resultSet, columnName); } + @Override + protected InputStream getContentBinaryStreamWithOffset(ResultSet resultSet, String columnName, + StreamFetchingOptions streamFetchingOptions) { + throw new UnsupportedOperationException(); + } } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/query/providers/ExternalSqlFileQueryProvider.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/query/providers/ExternalSqlFileQueryProvider.java index 300493012c..8ce5f0d1cd 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/query/providers/ExternalSqlFileQueryProvider.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/query/providers/ExternalSqlFileQueryProvider.java @@ -5,6 +5,7 @@ import java.sql.ResultSet; import org.cloudfoundry.multiapps.controller.persistence.dialects.DataSourceDialect; +import org.cloudfoundry.multiapps.controller.persistence.query.options.StreamFetchingOptions; public class ExternalSqlFileQueryProvider extends SqlFileQueryProvider { @@ -22,4 +23,9 @@ protected InputStream getContentBinaryStream(ResultSet resultSet, String columnN throw new UnsupportedOperationException(); } + @Override + protected InputStream getContentBinaryStreamWithOffset(ResultSet resultSet, String columnName, + StreamFetchingOptions streamFetchingOptions) { + throw new UnsupportedOperationException(); + } } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/query/providers/SqlFileQueryProvider.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/query/providers/SqlFileQueryProvider.java index 714021b0d7..021d972429 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/query/providers/SqlFileQueryProvider.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/query/providers/SqlFileQueryProvider.java @@ -18,19 +18,20 @@ import java.util.List; import java.util.stream.IntStream; -import jakarta.xml.bind.DatatypeConverter; - import org.cloudfoundry.multiapps.controller.persistence.Constants; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.dialects.DataSourceDialect; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; import org.cloudfoundry.multiapps.controller.persistence.query.SqlQuery; +import org.cloudfoundry.multiapps.controller.persistence.query.options.StreamFetchingOptions; import org.cloudfoundry.multiapps.controller.persistence.services.FileContentProcessor; import org.cloudfoundry.multiapps.controller.persistence.stream.DBInputStream; import org.cloudfoundry.multiapps.controller.persistence.util.JdbcUtil; import org.slf4j.Logger; +import jakarta.xml.bind.DatatypeConverter; + public abstract class SqlFileQueryProvider { private static final String INSERT_FILE_ATTRIBUTES_AND_CONTENT_WITHOUT_DIGEST = "INSERT INTO %s (FILE_ID, SPACE, FILE_NAME, NAMESPACE, FILE_SIZE, DIGEST_ALGORITHM, MODIFIED, OPERATION_ID, %s) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"; @@ -244,10 +245,7 @@ public SqlQuery openFileWithContentQuery(String space, String id) statement.setString(2, space); resultSet = statement.executeQuery(); if (resultSet.next()) { - return new DBInputStream(getContentBinaryStream(resultSet, getContentColumnName()), - statement, - resultSet, - connection); + return new DBInputStream(getContentBinaryStream(resultSet, getContentColumnName()), statement, resultSet, connection); } else { throw new SQLException(MessageFormat.format(Messages.FILE_NOT_FOUND, id)); } @@ -281,6 +279,29 @@ public SqlQuery getProcessFileWithContentQuery(String space, String id, F }; } + public SqlQuery getProcessFileWithContentQueryWithOffsetQuery(String space, String id, + StreamFetchingOptions streamFetchingOptions, + FileContentProcessor fileContentProcessor) { + return (Connection connection) -> { + PreparedStatement statement = null; + ResultSet resultSet = null; + try { + statement = connection.prepareStatement(getSelectWithContentQuery()); + statement.setString(1, id); + statement.setString(2, space); + resultSet = statement.executeQuery(); + if (resultSet.next()) { + return processFileContentWithOffset(resultSet, streamFetchingOptions, fileContentProcessor); + } else { + throw new SQLException(MessageFormat.format(Messages.FILE_NOT_FOUND, id)); + } + } finally { + JdbcUtil.closeQuietly(resultSet); + JdbcUtil.closeQuietly(statement); + } + }; + } + public SqlQuery getDeleteBySpaceAndNamespaceQuery(String space, String namespace) { return (Connection connection) -> { PreparedStatement statement = null; @@ -423,8 +444,31 @@ private T processFileContent(ResultSet resultSet, FileContentProcessor fi } } + private T processFileContentWithOffset(ResultSet resultSet, StreamFetchingOptions streamFetchingOptions, + FileContentProcessor fileContentProcessor) + throws SQLException { + InputStream fileStream = getContentBinaryStreamWithOffset(resultSet, getContentColumnName(), streamFetchingOptions); + try { + return fileContentProcessor.process(fileStream); + } catch (Exception e) { + throw new SQLException(e.getMessage(), e); + } finally { + if (fileStream != null) { + try { + fileStream.close(); + } catch (IOException e) { + logger.error(Messages.UPLOAD_STREAM_FAILED_TO_CLOSE, e); + } + } + } + } + protected abstract InputStream getContentBinaryStream(ResultSet resultSet, String columnName) throws SQLException; + protected abstract InputStream getContentBinaryStreamWithOffset(ResultSet resultSet, String columnName, + StreamFetchingOptions streamFetchingOptions) + throws SQLException; + private PreparedStatement getFilesStatementBasedOnNamespace(Connection connection, String space, String namespace) throws SQLException { PreparedStatement statement; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseFileService.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseFileService.java index bd79c54809..7a6586710a 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseFileService.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/DatabaseFileService.java @@ -9,6 +9,7 @@ import org.cloudfoundry.multiapps.controller.persistence.DataSourceWithDialect; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.query.options.StreamFetchingOptions; import org.cloudfoundry.multiapps.controller.persistence.query.providers.BlobSqlFileQueryProvider; import org.cloudfoundry.multiapps.controller.persistence.query.providers.SqlFileQueryProvider; @@ -26,6 +27,20 @@ protected DatabaseFileService(DataSourceWithDialect dataSourceWithDialect, SqlFi super(dataSourceWithDialect, sqlFileQueryProvider, null); } + @Override + public T processFileContentWithOffset(FileContentToProcess fileContentToProcess, FileContentProcessor fileContentProcessor) + throws FileStorageException { + try { + return getSqlQueryExecutor().execute(getSqlFileQueryProvider().getProcessFileWithContentQueryWithOffsetQuery(fileContentToProcess.getSpaceGuid(), + fileContentToProcess.getGuid(), + new StreamFetchingOptions(fileContentToProcess.getStartOffset(), + fileContentToProcess.getEndOffset()), + fileContentProcessor)); + } catch (SQLException e) { + throw new FileStorageException(e.getMessage(), e); + } + } + @Override public T processFileContent(String space, String id, FileContentProcessor fileContentProcessor) throws FileStorageException { try { diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileContentToProcess.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileContentToProcess.java new file mode 100644 index 0000000000..49767022e0 --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileContentToProcess.java @@ -0,0 +1,16 @@ +package org.cloudfoundry.multiapps.controller.persistence.services; + +import org.immutables.value.Value; + +@Value.Immutable +public interface FileContentToProcess { + + String getGuid(); + + String getSpaceGuid(); + + long getStartOffset(); + + long getEndOffset(); + +} diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileService.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileService.java index 789268d19d..b2d749dc6c 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileService.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileService.java @@ -16,8 +16,6 @@ import java.util.List; import java.util.UUID; -import jakarta.xml.bind.DatatypeConverter; - import org.cloudfoundry.multiapps.controller.persistence.Constants; import org.cloudfoundry.multiapps.controller.persistence.DataSourceWithDialect; import org.cloudfoundry.multiapps.controller.persistence.Messages; @@ -29,6 +27,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import jakarta.xml.bind.DatatypeConverter; + public class FileService { protected static final String DEFAULT_TABLE_NAME = "LM_SL_PERSISTENCE_FILE"; @@ -128,6 +128,19 @@ public void consumeFileContent(String space, String id, FileContentConsumer file }); } + public void consumeFileContentWithOffset(FileContentToProcess fileContentToProcess, FileContentConsumer fileContentConsumer) + throws FileStorageException { + processFileContentWithOffset(fileContentToProcess, inputStream -> { + fileContentConsumer.consume(inputStream); + return null; + }); + } + + public T processFileContentWithOffset(FileContentToProcess fileContentToProcess, FileContentProcessor fileContentProcessor) + throws FileStorageException { + return fileStorage.processArchiveEntryContent(fileContentToProcess, fileContentProcessor); + } + public T processFileContent(String space, String id, FileContentProcessor fileContentProcessor) throws FileStorageException { return fileStorage.processFileContent(space, id, fileContentProcessor); } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java index d10ec0017f..086e713708 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/FileStorage.java @@ -32,4 +32,6 @@ public interface FileStorage { void deleteFilesByIds(List fileIds) throws FileStorageException; + T processArchiveEntryContent(FileContentToProcess fileContentToProcess, FileContentProcessor fileContentProcessor) + throws FileStorageException; } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java index 6675593945..977b5968df 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/ObjectStoreFileStorage.java @@ -25,6 +25,7 @@ import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.PageSet; import org.jclouds.blobstore.domain.StorageMetadata; +import org.jclouds.blobstore.options.GetOptions; import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.blobstore.options.PutOptions; import org.jclouds.http.HttpResponseException; @@ -37,7 +38,7 @@ public class ObjectStoreFileStorage implements FileStorage { private static final Logger LOGGER = LoggerFactory.getLogger(ObjectStoreFileStorage.class); - + private static final int MAX_RETRIES_COUNT = 3; private static final long RETRY_BASE_WAIT_TIME_IN_MILLIS = 5000L; private final BlobStore blobStore; @@ -110,6 +111,44 @@ public T processFileContent(String space, String id, FileContentProcessor } } + @Override + public T processArchiveEntryContent(FileContentToProcess fileContentToProcess, FileContentProcessor fileContentProcessor) + throws FileStorageException { + FileEntry fileEntry = createFileEntry(fileContentToProcess.getSpaceGuid(), fileContentToProcess.getGuid()); + try { + Payload payload = getBlobPayloadWithOffset(fileEntry, fileContentToProcess.getStartOffset(), + fileContentToProcess.getEndOffset()); + return processContent(fileContentProcessor, payload); + } catch (Exception e) { + throw new FileStorageException(e); + } + } + + private Payload getBlobPayloadWithOffset(FileEntry fileEntry, long startOffset, long endOffset) throws FileStorageException { + Blob blob = getBlobWithRetriesWithOffset(fileEntry, MAX_RETRIES_COUNT, startOffset, endOffset); + if (blob == null) { + throw new FileStorageException(MessageFormat.format(Messages.FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST, fileEntry.getId(), + fileEntry.getSpace())); + } + return blob.getPayload(); + } + + private Blob getBlobWithRetriesWithOffset(FileEntry fileEntry, int retries, long startOffset, long endOffset) { + GetOptions getOptions = new GetOptions().range(startOffset, endOffset); + for (int i = 1; i <= retries; i++) { + Blob blob = blobStore.getBlob(container, fileEntry.getId(), getOptions); + if (blob != null) { + return blob; + } + LOGGER.warn(MessageFormat.format(Messages.ATTEMPT_TO_DOWNLOAD_MISSING_BLOB, i, retries, fileEntry.getId())); + if (i == retries) { + break; + } + MiscUtil.sleep(i * getRetryWaitTime()); + } + return null; + } + private Payload getBlobPayload(FileEntry fileEntry) throws FileStorageException { Blob blob = getBlobWithRetries(fileEntry, 3); if (blob == null) { diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/stream/DBInputStream.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/stream/DBInputStream.java index 1dcbed21aa..24c7449dc8 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/stream/DBInputStream.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/stream/DBInputStream.java @@ -37,7 +37,7 @@ public void close() throws IOException { try { JdbcUtil.commit(connection); } catch (SQLException e) { - throw new SLException(e.getMessage(), e); + throw new SLException(e, e.getMessage()); } finally { setAutoCommit(); JdbcUtil.closeQuietly(connection); @@ -48,7 +48,7 @@ private void setAutoCommit() { try { JdbcUtil.setAutoCommitSafely(connection); } catch (SQLException e) { - throw new SLException(e.getMessage(), e); + throw new SLException(e, e.getMessage()); } } } diff --git a/multiapps-controller-process/pom.xml b/multiapps-controller-process/pom.xml index b17978153a..58521ae7b9 100644 --- a/multiapps-controller-process/pom.xml +++ b/multiapps-controller-process/pom.xml @@ -29,6 +29,10 @@ + + org.apache.commons + commons-compress + jakarta.xml.bind jakarta.xml.bind-api diff --git a/multiapps-controller-process/src/main/java/module-info.java b/multiapps-controller-process/src/main/java/module-info.java index 6c698cfa72..cea4682659 100644 --- a/multiapps-controller-process/src/main/java/module-info.java +++ b/multiapps-controller-process/src/main/java/module-info.java @@ -34,6 +34,7 @@ requires java.sql; requires jakarta.xml.bind; requires jakarta.inject; + requires org.apache.commons.compress; requires org.apache.logging.log4j.core; requires org.apache.logging.log4j; requires org.apache.commons.collections4; diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java index eae6b94765..5fcdcf0f9e 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java @@ -13,7 +13,6 @@ public class Messages { public static final String ERROR_RETRIEVING_MTA_REQUIRED_DEPENDENCY_CONTENT = "Error retrieving content of MTA required dependency \"{0}\""; public static final String ERROR_RETRIEVING_MTA_RESOURCE_CONTENT = "Error retrieving content of MTA resource \"{0}\""; public static final String ERROR_SERVICE_NEEDS_TO_BE_RECREATED_BUT_FLAG_NOT_SET = "Service described by MTA resource \"{0}\" of type [{1}] does not match already existing service \"{2}\" of type [{3}] and needs to be recreated. Use command line option \"--delete-services\" to enable the deletion of the existing one."; - public static final String SIZE_OF_APP_EXCEEDS_MAX_SIZE_LIMIT = "The size of the application exceeds max size limit \"{0}\""; public static final String ERROR_RETRIEVING_REQUIRED_SERVICE_KEY_ELEMENT = "Unable to retrieve required service key element \"{0}\" for service \"{1}\""; public static final String ERROR_PARAMETER_1_MUST_NOT_BE_NEGATIVE = "Value \"{0}\" of parameter \"{1}\" must not be negative"; public static final String ERROR_PARAMETER_1_MUST_BE_LESS_THAN_2 = "Value \"{0}\" of parameter \"{1}\" must be less than {2}"; @@ -73,6 +72,7 @@ public class Messages { public static final String SIZE_OF_ALL_OPERATIONS_FILES_0_EXCEEDS_MAX_UPLOAD_SIZE_1 = "Size of all operation files \"{0}\" exceeds max upload size limit \"{1}\""; public static final String NOT_BOOLEAN_PARAMETER_VALUE = "Value \"{0}\" of parameter \"{1}\" is not boolean"; public static final String ERROR_OCCURRED_DURING_APPLICATION_UPLOAD_0 = "Error occurred during application upload: {0}"; + public static final String COMPRESSION_METHOD_WITH_VALUE_0_NOT_FOUND = "Compression method with value: {0} not found"; // Audit log messages @@ -741,6 +741,12 @@ public class Messages { public static final String PARAMETER_0_MUST_BE_POSITIVE_WITH_MAX_VALUE_1 = "Parameter \"{0}\" must be positive integer value up to {1}!"; public static final String SKIPPING_UPLOAD_OF_APPLICATION_0 = "Skipping upload of application: {0}"; public static final String UPLOAD_OF_APPLICATION_0_STARTED_ON_INSTANCE_1 = "Upload of application: {0} started on instance: {1}"; + public static final String MTA_ARCHIVE_ID_0_MESSAGE = "MTA Archive ID: {0}"; + public static final String MTA_DESCRIPTOR_LENGTH_0_MESSAGE = "MTA Descriptor length: {0}"; + public static final String MTA_ARCHIVE_MODULES_0_MESSAGE = "MTA Archive Modules: {0}"; + public static final String MTA_ARCHIVE_REQUIRES_0_MESSAGE = "MTA Archive Requires: {0}"; + public static final String MTA_ARCHIVE_RESOURCES_0_MESSAGE = "MTA Archive Resources: {0}"; + public static final String MODULE_0_CONTENT_IS_A_DIRECTORY = "Module: \"{0}\" content is a directory"; // Not log messages public static final String SERVICE_TYPE = "{0}/{1}"; diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/context/ApplicationToUploadContext.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/context/ApplicationToUploadContext.java index 35fc04b95d..a3054fcf68 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/context/ApplicationToUploadContext.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/context/ApplicationToUploadContext.java @@ -1,6 +1,10 @@ package org.cloudfoundry.multiapps.controller.process.context; +import java.util.List; + +import org.cloudfoundry.Nullable; import org.cloudfoundry.multiapps.controller.client.lib.domain.CloudApplicationExtended; +import org.cloudfoundry.multiapps.controller.process.util.ArchiveEntryWithStreamPositions; import org.cloudfoundry.multiapps.controller.process.util.StepLogger; import org.immutables.value.Value; @@ -25,4 +29,7 @@ public interface ApplicationToUploadContext { String getTaskId(); String getAppArchiveId(); + + @Nullable // TODO: backwards compatibility for one tact + List getArchiveEntries(); } diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/DetermineApplicationServiceBindingActionsStep.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/DetermineApplicationServiceBindingActionsStep.java index 10846a2fcf..2bf7f1f9ff 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/DetermineApplicationServiceBindingActionsStep.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/DetermineApplicationServiceBindingActionsStep.java @@ -6,9 +6,6 @@ import java.util.Objects; import java.util.UUID; -import jakarta.inject.Inject; -import jakarta.inject.Named; - import org.cloudfoundry.multiapps.controller.client.lib.domain.CloudApplicationExtended; import org.cloudfoundry.multiapps.controller.client.lib.domain.CloudApplicationExtended.AttributeUpdateStrategy; import org.cloudfoundry.multiapps.controller.core.cf.clients.AppBoundServiceInstanceNamesGetter; @@ -16,6 +13,7 @@ import org.cloudfoundry.multiapps.controller.core.security.token.TokenService; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; import org.cloudfoundry.multiapps.controller.process.Messages; +import org.cloudfoundry.multiapps.controller.process.util.ArchiveEntryExtractor; import org.cloudfoundry.multiapps.controller.process.util.ServiceBindingParametersGetter; import org.cloudfoundry.multiapps.controller.process.variables.Variables; import org.springframework.beans.factory.config.BeanDefinition; @@ -26,6 +24,9 @@ import com.sap.cloudfoundry.client.facade.domain.CloudApplication; import com.sap.cloudfoundry.client.facade.domain.CloudServiceBinding; +import jakarta.inject.Inject; +import jakarta.inject.Named; + @Named("determineApplicationServiceBindingActionsStep") @Scope(BeanDefinition.SCOPE_PROTOTYPE) public class DetermineApplicationServiceBindingActionsStep extends SyncFlowableStep { @@ -34,6 +35,8 @@ public class DetermineApplicationServiceBindingActionsStep extends SyncFlowableS private TokenService tokenService; @Inject private WebClientFactory webClientFactory; + @Inject + private ArchiveEntryExtractor archiveEntryExtractor; @Override protected StepPhase executeStep(ProcessContext context) throws FileStorageException { @@ -130,7 +133,7 @@ private boolean shouldKeepExistingServiceBindings(CloudApplicationExtended app) } protected ServiceBindingParametersGetter getServiceBindingParametersGetter(ProcessContext context) { - return new ServiceBindingParametersGetter(context, fileService, configuration.getMaxManifestSize()); + return new ServiceBindingParametersGetter(context, archiveEntryExtractor, configuration.getMaxManifestSize(), fileService); } protected AppBoundServiceInstanceNamesGetter getAppServicesGetter(CloudCredentials credentials, String correlationId) { diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/DetermineServiceCreateUpdateServiceActionsStep.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/DetermineServiceCreateUpdateServiceActionsStep.java index 7bfa757c67..bb8bfccf0d 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/DetermineServiceCreateUpdateServiceActionsStep.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/DetermineServiceCreateUpdateServiceActionsStep.java @@ -1,5 +1,6 @@ package org.cloudfoundry.multiapps.controller.process.steps; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.text.MessageFormat; @@ -10,8 +11,6 @@ import java.util.Map; import java.util.Objects; -import jakarta.inject.Named; - import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.ObjectUtils; @@ -26,7 +25,11 @@ import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; import org.cloudfoundry.multiapps.controller.process.Constants; import org.cloudfoundry.multiapps.controller.process.Messages; +import org.cloudfoundry.multiapps.controller.process.util.ArchiveEntryExtractor; +import org.cloudfoundry.multiapps.controller.process.util.ArchiveEntryExtractorUtil; +import org.cloudfoundry.multiapps.controller.process.util.ArchiveEntryWithStreamPositions; import org.cloudfoundry.multiapps.controller.process.util.DynamicResolvableParametersContextUpdater; +import org.cloudfoundry.multiapps.controller.process.util.ImmutableFileEntryProperties; import org.cloudfoundry.multiapps.controller.process.util.ServiceAction; import org.cloudfoundry.multiapps.controller.process.variables.Variables; import org.cloudfoundry.multiapps.mta.handlers.ArchiveHandler; @@ -40,10 +43,20 @@ import com.sap.cloudfoundry.client.facade.domain.CloudServiceKey; import com.sap.cloudfoundry.client.facade.domain.ServiceOperation; +import jakarta.inject.Inject; +import jakarta.inject.Named; + @Named("determineServiceCreateUpdateActionsStep") @Scope(BeanDefinition.SCOPE_PROTOTYPE) public class DetermineServiceCreateUpdateServiceActionsStep extends SyncFlowableStep { + private final ArchiveEntryExtractor archiveEntryExtractor; + + @Inject + public DetermineServiceCreateUpdateServiceActionsStep(ArchiveEntryExtractor archiveEntryExtractor) { + this.archiveEntryExtractor = archiveEntryExtractor; + } + @Override protected StepPhase executeStep(ProcessContext context) throws Exception { CloudControllerClient client = context.getControllerClient(); @@ -80,7 +93,7 @@ private List determineActionsAndHandleExceptions(ProcessContext c } } - private void setServiceParameters(ProcessContext context, CloudServiceInstanceExtended service) throws FileStorageException { + private void setServiceParameters(ProcessContext context, CloudServiceInstanceExtended service) { service = prepareServiceParameters(context, service); context.setVariable(Variables.SERVICE_TO_PROCESS, service); } @@ -134,9 +147,11 @@ private List determineActions(ProcessContext context, CloudServic } if (service.shouldSkipParametersUpdate()) { - getStepLogger().warnWithoutProgressMessage(Messages.WILL_NOT_UPDATE_SERVICE_PARAMS_BECAUSE_PARAMETER_SKIP_SERVICE_UPDATES, service.getName()); + getStepLogger().warnWithoutProgressMessage(Messages.WILL_NOT_UPDATE_SERVICE_PARAMS_BECAUSE_PARAMETER_SKIP_SERVICE_UPDATES, + service.getName()); } else if (MapUtils.isEmpty(service.getCredentials())) { - getStepLogger().warnWithoutProgressMessage(Messages.WILL_NOT_UPDATE_SERVICE_PARAMS_BECAUSE_UNDEFINED_OR_EMPTY, service.getName()); + getStepLogger().warnWithoutProgressMessage(Messages.WILL_NOT_UPDATE_SERVICE_PARAMS_BECAUSE_UNDEFINED_OR_EMPTY, + service.getName()); } else { getStepLogger().debug(Messages.WILL_UPDATE_SERVICE_PARAMETERS); getStepLogger().debug(Messages.NEW_SERVICE_PARAMETERS, SecureSerialization.toJson(service.getCredentials())); @@ -195,8 +210,7 @@ private boolean shouldUpdateMetadata(CloudServiceInstanceExtended service, Cloud return newMetadata != null; } - private CloudServiceInstanceExtended prepareServiceParameters(ProcessContext context, CloudServiceInstanceExtended service) - throws FileStorageException { + private CloudServiceInstanceExtended prepareServiceParameters(ProcessContext context, CloudServiceInstanceExtended service) { MtaArchiveElements mtaArchiveElements = context.getVariable(Variables.MTA_ARCHIVE_ELEMENTS); String fileName = mtaArchiveElements.getResourceFileName(service.getResourceName()); if (fileName != null) { @@ -206,17 +220,39 @@ private CloudServiceInstanceExtended prepareServiceParameters(ProcessContext con return service; } - private CloudServiceInstanceExtended setServiceParameters(ProcessContext context, CloudServiceInstanceExtended service, String fileName) - throws FileStorageException { + private CloudServiceInstanceExtended setServiceParameters(ProcessContext context, CloudServiceInstanceExtended service, + String fileName) { String appArchiveId = context.getRequiredVariable(Variables.APP_ARCHIVE_ID); String spaceGuid = context.getVariable(Variables.SPACE_GUID); - return fileService.processFileContent(spaceGuid, appArchiveId, appArchiveStream -> { - try (InputStream is = ArchiveHandler.getInputStream(appArchiveStream, fileName, configuration.getMaxManifestSize())) { - return mergeCredentials(service, is); - } catch (IOException e) { - throw new SLException(e, Messages.ERROR_RETRIEVING_MTA_RESOURCE_CONTENT, fileName); + + // TODO: backwards compatibility for one tact + List archiveEntriesWithStreamPositions = context.getVariable(Variables.ARCHIVE_ENTRIES_POSITIONS); + if (archiveEntriesWithStreamPositions == null) { + try { + return fileService.processFileContent(spaceGuid, appArchiveId, appArchiveStream -> { + InputStream fileStream = ArchiveHandler.getInputStream(appArchiveStream, fileName, configuration.getMaxManifestSize()); + return mergeCredentials(service, fileStream); + }); + } catch (FileStorageException e) { + throw new SLException(e, e.getMessage()); } - }); + } + // TODO: backwards compatibility for one tact + + ArchiveEntryWithStreamPositions serviceBindingParametersEntry = ArchiveEntryExtractorUtil.findEntry(fileName, + context.getVariable(Variables.ARCHIVE_ENTRIES_POSITIONS)); + byte[] serviceBindingsParametersContent = archiveEntryExtractor.extractEntryBytes(ImmutableFileEntryProperties.builder() + .guid(appArchiveId) + .name(serviceBindingParametersEntry.getName()) + .spaceGuid(spaceGuid) + .maxFileSizeInBytes(configuration.getMaxManifestSize()) + .build(), + serviceBindingParametersEntry); + try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(serviceBindingsParametersContent)) { + return mergeCredentials(service, byteArrayInputStream); + } catch (IOException e) { + throw new SLException(e, Messages.ERROR_RETRIEVING_MTA_RESOURCE_CONTENT, fileName); + } } private CloudServiceInstanceExtended mergeCredentials(CloudServiceInstanceExtended service, InputStream credentialsJson) { diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/IncrementalAppInstancesUpdateStep.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/IncrementalAppInstancesUpdateStep.java index 9d6fcd48ba..4d9361d390 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/IncrementalAppInstancesUpdateStep.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/IncrementalAppInstancesUpdateStep.java @@ -1,11 +1,14 @@ package org.cloudfoundry.multiapps.controller.process.steps; -import com.sap.cloudfoundry.client.facade.CloudControllerClient; -import com.sap.cloudfoundry.client.facade.domain.CloudApplication; -import com.sap.cloudfoundry.client.facade.domain.InstanceInfo; -import com.sap.cloudfoundry.client.facade.domain.InstanceState; -import jakarta.inject.Inject; -import jakarta.inject.Named; +import static org.cloudfoundry.multiapps.controller.process.steps.StepsUtil.disableAutoscaling; +import static org.cloudfoundry.multiapps.controller.process.steps.StepsUtil.enableAutoscaling; + +import java.text.MessageFormat; +import java.time.Duration; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + import org.cloudfoundry.multiapps.controller.client.lib.domain.CloudApplicationExtended; import org.cloudfoundry.multiapps.controller.core.cf.CloudControllerClientFactory; import org.cloudfoundry.multiapps.controller.core.model.DeployedMta; @@ -20,14 +23,13 @@ import org.springframework.beans.factory.config.BeanDefinition; import org.springframework.context.annotation.Scope; -import java.text.MessageFormat; -import java.time.Duration; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.TimeUnit; +import com.sap.cloudfoundry.client.facade.CloudControllerClient; +import com.sap.cloudfoundry.client.facade.domain.CloudApplication; +import com.sap.cloudfoundry.client.facade.domain.InstanceInfo; +import com.sap.cloudfoundry.client.facade.domain.InstanceState; -import static org.cloudfoundry.multiapps.controller.process.steps.StepsUtil.disableAutoscaling; -import static org.cloudfoundry.multiapps.controller.process.steps.StepsUtil.enableAutoscaling; +import jakarta.inject.Inject; +import jakarta.inject.Named; @Named("incrementalAppInstancesUpdateStep") @Scope(BeanDefinition.SCOPE_PROTOTYPE) @@ -64,10 +66,8 @@ protected StepPhase executeAsyncStep(ProcessContext context) throws Exception { List idleApplicationInstances = client.getApplicationInstances(applicationId) .getInstances(); var incrementalAppInstanceUpdateConfigurationBuilder = ImmutableIncrementalAppInstanceUpdateConfiguration.builder() - .newApplication( - application) - .newApplicationInstanceCount( - idleApplicationInstances.size()); + .newApplication(application) + .newApplicationInstanceCount(idleApplicationInstances.size()); int oldApplicationInstanceCount = client.getApplicationInstances(oldApplication) .getInstances() @@ -95,8 +95,7 @@ private StepPhase scaleUpNewAppToTheRequiredInstances(ProcessContext context, Cl client.updateApplicationInstances(application.getName(), application.getInstances()); incrementalAppInstanceUpdateConfigurationBuilder = ImmutableIncrementalAppInstanceUpdateConfiguration.builder() .newApplication(application) - .newApplicationInstanceCount( - application.getInstances()); + .newApplicationInstanceCount(application.getInstances()); context.setVariable(Variables.INCREMENTAL_APP_INSTANCE_UPDATE_CONFIGURATION, incrementalAppInstanceUpdateConfigurationBuilder.build()); setExecutionIndexForPollingNewAppInstances(context); @@ -116,8 +115,7 @@ private DeployedMtaApplication getOldApplication(ProcessContext context, CloudAp DeployedMtaApplication deployedMtaApplication = deployedMta.getApplications() .stream() .filter(deployedApplication -> deployedApplication.getModuleName() - .equals( - currentApplication.getModuleName())) + .equals(currentApplication.getModuleName())) .findFirst() .orElse(null); if (deployedMtaApplication == null) { diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/ProcessMtaArchiveStep.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/ProcessMtaArchiveStep.java index 075667a035..7b970c2f9c 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/ProcessMtaArchiveStep.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/ProcessMtaArchiveStep.java @@ -1,78 +1,94 @@ package org.cloudfoundry.multiapps.controller.process.steps; +import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; +import java.util.List; import java.util.Map; import java.util.function.Function; import java.util.jar.Manifest; -import jakarta.inject.Inject; - +import org.cloudfoundry.multiapps.common.SLException; import org.cloudfoundry.multiapps.controller.core.helpers.DescriptorParserFacadeFactory; import org.cloudfoundry.multiapps.controller.core.helpers.MtaArchiveElements; import org.cloudfoundry.multiapps.controller.core.helpers.MtaArchiveHelper; -import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; import org.cloudfoundry.multiapps.controller.persistence.services.OperationService; import org.cloudfoundry.multiapps.controller.process.Messages; +import org.cloudfoundry.multiapps.controller.process.util.ArchiveEntryExtractor; +import org.cloudfoundry.multiapps.controller.process.util.ArchiveEntryExtractorUtil; +import org.cloudfoundry.multiapps.controller.process.util.ArchiveEntryStreamWithStreamPositionsDeterminer; +import org.cloudfoundry.multiapps.controller.process.util.ArchiveEntryWithStreamPositions; +import org.cloudfoundry.multiapps.controller.process.util.ImmutableFileEntryProperties; import org.cloudfoundry.multiapps.controller.process.util.ProcessConflictPreventer; import org.cloudfoundry.multiapps.controller.process.variables.Variables; import org.cloudfoundry.multiapps.mta.handlers.ArchiveHandler; import org.cloudfoundry.multiapps.mta.handlers.DescriptorParserFacade; import org.cloudfoundry.multiapps.mta.model.DeploymentDescriptor; +import jakarta.inject.Inject; + public class ProcessMtaArchiveStep extends SyncFlowableStep { @Inject private OperationService operationService; @Inject protected DescriptorParserFacadeFactory descriptorParserFactory; + @Inject + private ArchiveEntryExtractor archiveEntryExtractor; + @Inject + protected ArchiveEntryStreamWithStreamPositionsDeterminer archiveEntryStreamWithStreamPositionsDeterminer; protected Function conflictPreventerSupplier = ProcessConflictPreventer::new; @Override - protected StepPhase executeStep(ProcessContext context) throws FileStorageException { + protected StepPhase executeStep(ProcessContext context) { getStepLogger().debug(Messages.PROCESSING_MTA_ARCHIVE); - String appArchiveId = context.getRequiredVariable(Variables.APP_ARCHIVE_ID); - getStepLogger().debug("MTA Archive ID: {0}", appArchiveId); + getStepLogger().debug(Messages.MTA_ARCHIVE_ID_0_MESSAGE, appArchiveId); processApplicationArchive(context, appArchiveId); setMtaIdForProcess(context); acquireOperationLock(context); - getStepLogger().debug(Messages.MTA_ARCHIVE_PROCESSED); return StepPhase.DONE; } - @Override - protected String getStepErrorMessage(ProcessContext context) { - return Messages.ERROR_PROCESSING_MTA_ARCHIVE; - } - - private void processApplicationArchive(ProcessContext context, String appArchiveId) throws FileStorageException { - DeploymentDescriptor descriptor = fileService.processFileContent(context.getVariable(Variables.SPACE_GUID), appArchiveId, - this::extractDeploymentDescriptor); - context.setVariable(Variables.DEPLOYMENT_DESCRIPTOR, descriptor); - - MtaArchiveHelper helper = fileService.processFileContent(context.getVariable(Variables.SPACE_GUID), appArchiveId, - this::createMtaArchiveHelperFromManifest); + private void processApplicationArchive(ProcessContext context, String appArchiveId) { + List archiveEntriesWithStreamPositions = archiveEntryStreamWithStreamPositionsDeterminer.determineArchiveEntries(context.getRequiredVariable(Variables.SPACE_GUID), + appArchiveId); + context.setVariable(Variables.ARCHIVE_ENTRIES_POSITIONS, archiveEntriesWithStreamPositions); + MtaArchiveHelper helper = createMtaArchiveHelperFromManifest(context, appArchiveId, archiveEntriesWithStreamPositions); MtaArchiveElements mtaArchiveElements = new MtaArchiveElements(); addMtaArchiveModulesInMtaArchiveElements(context, helper, mtaArchiveElements); addMtaRequiredDependenciesInMtaArchiveElements(helper, mtaArchiveElements); addMtaArchiveResourcesInMtaArchiveElements(helper, mtaArchiveElements); context.setVariable(Variables.MTA_ARCHIVE_ELEMENTS, mtaArchiveElements); + DeploymentDescriptor deploymentDescriptor = extractDeploymentDescriptor(context, appArchiveId, archiveEntriesWithStreamPositions); + context.setVariable(Variables.DEPLOYMENT_DESCRIPTOR, deploymentDescriptor); } - private DeploymentDescriptor extractDeploymentDescriptor(InputStream appArchiveStream) { - String descriptorString = ArchiveHandler.getDescriptor(appArchiveStream, configuration.getMaxMtaDescriptorSize()); - getStepLogger().debug("MTA Descriptor length: {0}", descriptorString.length()); - DescriptorParserFacade descriptorParserFacade = descriptorParserFactory.getInstance(); - return descriptorParserFacade.parseDeploymentDescriptor(descriptorString); + private MtaArchiveHelper createMtaArchiveHelperFromManifest(ProcessContext context, String appArchiveId, + List archiveEntriesWithStreamPositions) { + ArchiveEntryWithStreamPositions mtaManifestEntry = ArchiveEntryExtractorUtil.findEntry(ArchiveHandler.MTA_MANIFEST_NAME, + archiveEntriesWithStreamPositions); + byte[] inflatedManifestFile = readEntry(context, appArchiveId, mtaManifestEntry); + try (InputStream inputStream = new ByteArrayInputStream(inflatedManifestFile)) { + Manifest manifest = new Manifest(inputStream); + MtaArchiveHelper helper = getHelper(manifest); + helper.init(); + return helper; + } catch (IOException e) { + throw new SLException(e, e.getMessage()); + } } - private MtaArchiveHelper createMtaArchiveHelperFromManifest(InputStream appArchiveStream) { - Manifest manifest = ArchiveHandler.getManifest(appArchiveStream, configuration.getMaxManifestSize()); - MtaArchiveHelper helper = getHelper(manifest); - helper.init(); - return helper; + private byte[] readEntry(ProcessContext context, String appArchiveId, ArchiveEntryWithStreamPositions mtaManifestEntry) { + return archiveEntryExtractor.extractEntryBytes(ImmutableFileEntryProperties.builder() + .guid(appArchiveId) + .name(mtaManifestEntry.getName()) + .spaceGuid(context.getRequiredVariable(Variables.SPACE_GUID)) + .maxFileSizeInBytes(configuration.getMaxMtaDescriptorSize()) + .build(), + mtaManifestEntry); } protected MtaArchiveHelper getHelper(Manifest manifest) { @@ -83,26 +99,37 @@ private void addMtaArchiveModulesInMtaArchiveElements(ProcessContext context, Mt MtaArchiveElements mtaArchiveElements) { Map mtaArchiveModules = helper.getMtaArchiveModules(); mtaArchiveModules.forEach(mtaArchiveElements::addModuleFileName); - getStepLogger().debug("MTA Archive Modules: {0}", mtaArchiveModules.keySet()); + getStepLogger().debug(Messages.MTA_ARCHIVE_MODULES_0_MESSAGE, mtaArchiveModules.keySet()); context.setVariable(Variables.MTA_ARCHIVE_MODULES, mtaArchiveModules.keySet()); } private void addMtaRequiredDependenciesInMtaArchiveElements(MtaArchiveHelper helper, MtaArchiveElements mtaArchiveElements) { Map mtaArchiveRequiresDependencies = helper.getMtaRequiresDependencies(); mtaArchiveRequiresDependencies.forEach(mtaArchiveElements::addRequiredDependencyFileName); - getStepLogger().debug("MTA Archive Requires: {0}", mtaArchiveRequiresDependencies.keySet()); + getStepLogger().debug(Messages.MTA_ARCHIVE_REQUIRES_0_MESSAGE, mtaArchiveRequiresDependencies.keySet()); } private void addMtaArchiveResourcesInMtaArchiveElements(MtaArchiveHelper helper, MtaArchiveElements mtaArchiveElements) { Map mtaArchiveResources = helper.getMtaArchiveResources(); mtaArchiveResources.forEach(mtaArchiveElements::addResourceFileName); - getStepLogger().debug("MTA Archive Resources: {0}", mtaArchiveResources.keySet()); + getStepLogger().debug(Messages.MTA_ARCHIVE_RESOURCES_0_MESSAGE, mtaArchiveResources.keySet()); + } + + private DeploymentDescriptor extractDeploymentDescriptor(ProcessContext context, String appArchiveId, + List archiveEntriesWithStreamPositions) { + + ArchiveEntryWithStreamPositions deploymentDescriptorEntry = ArchiveEntryExtractorUtil.findEntry(ArchiveHandler.MTA_DEPLOYMENT_DESCRIPTOR_NAME, + archiveEntriesWithStreamPositions); + byte[] inflatedDeploymentDescriptor = readEntry(context, appArchiveId, deploymentDescriptorEntry); + DescriptorParserFacade descriptorParserFacade = descriptorParserFactory.getInstance(); + DeploymentDescriptor deploymentDescriptor = descriptorParserFacade.parseDeploymentDescriptor(new String(inflatedDeploymentDescriptor)); + getStepLogger().debug(Messages.MTA_DESCRIPTOR_LENGTH_0_MESSAGE, inflatedDeploymentDescriptor.length); + return deploymentDescriptor; } private void setMtaIdForProcess(ProcessContext context) { DeploymentDescriptor deploymentDescriptor = context.getVariable(Variables.DEPLOYMENT_DESCRIPTOR); String mtaId = deploymentDescriptor.getId(); - context.setVariable(Variables.MTA_ID, mtaId); } @@ -110,9 +137,14 @@ private void acquireOperationLock(ProcessContext context) { DeploymentDescriptor deploymentDescriptor = context.getVariable(Variables.DEPLOYMENT_DESCRIPTOR); String mtaId = deploymentDescriptor.getId(); String namespace = context.getVariable(Variables.MTA_NAMESPACE); - conflictPreventerSupplier.apply(operationService) .acquireLock(mtaId, namespace, context.getVariable(Variables.SPACE_GUID), context.getVariable(Variables.CORRELATION_ID)); } + + @Override + protected String getStepErrorMessage(ProcessContext context) { + return Messages.ERROR_PROCESSING_MTA_ARCHIVE; + } + } diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java index dafcfc97b3..5bfca316cc 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java @@ -2,7 +2,6 @@ import static java.text.MessageFormat.format; -import java.io.InputStream; import java.nio.file.Path; import java.text.MessageFormat; import java.time.Duration; @@ -22,8 +21,6 @@ import org.cloudfoundry.multiapps.controller.core.helpers.MtaArchiveElements; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.util.FileUtils; -import org.cloudfoundry.multiapps.controller.persistence.services.FileService; -import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; import org.cloudfoundry.multiapps.controller.persistence.services.ProcessLoggerPersister; import org.cloudfoundry.multiapps.controller.process.Messages; import org.cloudfoundry.multiapps.controller.process.context.ApplicationToUploadContext; @@ -44,16 +41,13 @@ public class UploadAppAsyncExecution implements AsyncExecution { private static final Logger LOGGER = LoggerFactory.getLogger(UploadAppAsyncExecution.class); - private final FileService fileService; private final ApplicationZipBuilder applicationZipBuilder; private final ProcessLoggerPersister processLoggerPersister; private final ApplicationConfiguration applicationConfiguration; private final ExecutorService appUploaderThreadPool; - public UploadAppAsyncExecution(FileService fileService, ApplicationZipBuilder applicationZipBuilder, - ProcessLoggerPersister processLoggerPersister, ApplicationConfiguration applicationConfiguration, - ExecutorService appUploaderThreadPool) { - this.fileService = fileService; + public UploadAppAsyncExecution(ApplicationZipBuilder applicationZipBuilder, ProcessLoggerPersister processLoggerPersister, + ApplicationConfiguration applicationConfiguration, ExecutorService appUploaderThreadPool) { this.applicationZipBuilder = applicationZipBuilder; this.processLoggerPersister = processLoggerPersister; this.applicationConfiguration = applicationConfiguration; @@ -79,8 +73,8 @@ public AsyncExecutionState execute(ProcessContext context) { } catch (RejectedExecutionException rejectedExecutionException) { LOGGER.warn(rejectedExecutionException.getMessage(), rejectedExecutionException); context.getStepLogger() - .warnWithoutProgressMessage(Messages.UPLOAD_OF_APPLICATION_0_WAS_NOT_ACCEPTED_BY_INSTANCE_1, - applicationToProcess.getName(), applicationConfiguration.getApplicationInstanceIndex()); + .warn(Messages.UPLOAD_OF_APPLICATION_0_WAS_NOT_ACCEPTED_BY_INSTANCE_1, applicationToProcess.getName(), + applicationConfiguration.getApplicationInstanceIndex()); return AsyncExecutionState.RUNNING; } CloudPackage cloudPackage; @@ -108,23 +102,19 @@ private ApplicationToUploadContext buildApplicationToUploadContext(ProcessContex .taskId(context.getVariable(Variables.TASK_ID)) .appArchiveId(context.getRequiredVariable(Variables.APP_ARCHIVE_ID)) .stepLogger(context.getStepLogger()) + .archiveEntries(context.getVariable(Variables.ARCHIVE_ENTRIES_POSITIONS)) .build(); } private CloudPackage doUpload(ProcessContext context, CloudApplicationExtended applicationToProcess, ApplicationToUploadContext applicationToUploadContext) { context.getStepLogger() - .infoWithoutProgressMessage(Messages.UPLOAD_OF_APPLICATION_0_STARTED_ON_INSTANCE_1, applicationToProcess.getName(), - applicationConfiguration.getApplicationInstanceIndex()); - try { - return proceedWithUpload(context.getControllerClient(), applicationToUploadContext); - } catch (FileStorageException e) { - throw new SLException(e, e.getMessage()); - } + .debug(Messages.UPLOAD_OF_APPLICATION_0_STARTED_ON_INSTANCE_1, applicationToProcess.getName(), + applicationConfiguration.getApplicationInstanceIndex()); + return proceedWithUpload(context.getControllerClient(), applicationToUploadContext); } - private CloudPackage proceedWithUpload(CloudControllerClient client, ApplicationToUploadContext applicationToUploadContext) - throws FileStorageException { + private CloudPackage proceedWithUpload(CloudControllerClient client, ApplicationToUploadContext applicationToUploadContext) { applicationToUploadContext.getStepLogger() .debug(Messages.UPLOADING_FILE_0_FOR_APP_1, applicationToUploadContext.getModuleFileName(), applicationToUploadContext.getApplication() @@ -137,8 +127,7 @@ private CloudPackage proceedWithUpload(CloudControllerClient client, Application return cloudPackage; } - private CloudPackage asyncUploadFiles(CloudControllerClient client, ApplicationToUploadContext applicationToUploadContext) - throws FileStorageException { + private CloudPackage asyncUploadFiles(CloudControllerClient client, ApplicationToUploadContext applicationToUploadContext) { Path extractedAppPath = extractApplicationFromArchive(applicationToUploadContext); LOGGER.debug(MessageFormat.format(Messages.APPLICATION_WITH_NAME_0_SAVED_TO_1, applicationToUploadContext.getApplication() .getName(), @@ -151,13 +140,10 @@ private CloudPackage asyncUploadFiles(CloudControllerClient client, ApplicationT return upload(client, applicationToUploadContext, extractedAppPath); } - private Path extractApplicationFromArchive(ApplicationToUploadContext applicationToUploadContext) throws FileStorageException { + private Path extractApplicationFromArchive(ApplicationToUploadContext applicationToUploadContext) { LocalDateTime startTime = LocalDateTime.now(); - Path extractedAppPath = fileService.processFileContent(applicationToUploadContext.getSpaceGuid(), - applicationToUploadContext.getAppArchiveId(), - appArchiveStream -> extractFromMtar(createApplicationArchiveContext(appArchiveStream, - applicationToUploadContext.getModuleFileName(), - applicationConfiguration.getMaxResourceFileSize()))); + Path extractedAppPath = extractFromMtar(createApplicationArchiveContext(applicationToUploadContext, + applicationConfiguration.getMaxResourceFileSize())); long timeElapsedForUpload = Duration.between(startTime, LocalDateTime.now()) .toMillis(); applicationToUploadContext.getStepLogger() @@ -166,8 +152,13 @@ private Path extractApplicationFromArchive(ApplicationToUploadContext applicatio return extractedAppPath; } - protected ApplicationArchiveContext createApplicationArchiveContext(InputStream appArchiveStream, String fileName, long maxSize) { - return new ApplicationArchiveContext(appArchiveStream, fileName, maxSize); + protected ApplicationArchiveContext createApplicationArchiveContext(ApplicationToUploadContext applicationToUploadContext, + long maxSize) { + return new ApplicationArchiveContext(applicationToUploadContext.getModuleFileName(), + maxSize, + applicationToUploadContext.getArchiveEntries(), + applicationToUploadContext.getSpaceGuid(), + applicationToUploadContext.getAppArchiveId()); } protected Path extractFromMtar(ApplicationArchiveContext applicationArchiveContext) { diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStep.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStep.java index 0d84486442..0e1a963fe9 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStep.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStep.java @@ -2,7 +2,6 @@ import static java.text.MessageFormat.format; -import java.io.InputStream; import java.time.Duration; import java.util.List; import java.util.Map; @@ -11,22 +10,19 @@ import java.util.UUID; import java.util.concurrent.ExecutorService; -import jakarta.inject.Inject; -import jakarta.inject.Named; - import org.cloudfoundry.multiapps.common.util.JsonUtil; import org.cloudfoundry.multiapps.controller.client.lib.domain.CloudApplicationExtended; import org.cloudfoundry.multiapps.controller.core.Constants; import org.cloudfoundry.multiapps.controller.core.helpers.ApplicationFileDigestDetector; import org.cloudfoundry.multiapps.controller.core.helpers.MtaArchiveElements; import org.cloudfoundry.multiapps.controller.core.security.serialization.SecureSerialization; -import org.cloudfoundry.multiapps.controller.persistence.services.FileContentProcessor; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; import org.cloudfoundry.multiapps.controller.process.Messages; import org.cloudfoundry.multiapps.controller.process.util.ApplicationArchiveContext; -import org.cloudfoundry.multiapps.controller.process.util.ApplicationArchiveReader; +import org.cloudfoundry.multiapps.controller.process.util.ApplicationDigestCalculator; import org.cloudfoundry.multiapps.controller.process.util.ApplicationStager; import org.cloudfoundry.multiapps.controller.process.util.ApplicationZipBuilder; +import org.cloudfoundry.multiapps.controller.process.util.ArchiveEntryWithStreamPositions; import org.cloudfoundry.multiapps.controller.process.util.CloudPackagesGetter; import org.cloudfoundry.multiapps.controller.process.util.TimeoutType; import org.cloudfoundry.multiapps.controller.process.variables.Variables; @@ -41,12 +37,17 @@ import com.sap.cloudfoundry.client.facade.domain.CloudPackage; import com.sap.cloudfoundry.client.facade.domain.Status; +import jakarta.inject.Inject; +import jakarta.inject.Named; + @Named("uploadAppStep") @Scope(BeanDefinition.SCOPE_PROTOTYPE) public class UploadAppStep extends TimeoutAsyncFlowableStep { + private static final Logger LOGGER = LoggerFactory.getLogger(UploadAppStep.class); + @Inject - protected ApplicationArchiveReader applicationArchiveReader; + protected ApplicationDigestCalculator applicationDigestCalculator; @Inject protected ApplicationZipBuilder applicationZipBuilder; @Inject @@ -124,22 +125,19 @@ private CloudPackage createDockerPackage(CloudControllerClient client, CloudAppl return client.createDockerPackage(applicationGuid, application.getDockerInfo()); } - private String getNewApplicationDigest(ProcessContext context, String fileName) throws FileStorageException { - return fileService.processFileContent(context.getVariable(Variables.SPACE_GUID), - context.getRequiredVariable(Variables.APP_ARCHIVE_ID), - createDigestCalculatorFileContentProcessor(fileName)); - } - - private FileContentProcessor createDigestCalculatorFileContentProcessor(String fileName) { - return appArchiveStream -> { - long maxSize = configuration.getMaxResourceFileSize(); - ApplicationArchiveContext applicationArchiveContext = createApplicationArchiveContext(appArchiveStream, fileName, maxSize); - return applicationArchiveReader.calculateApplicationDigest(applicationArchiveContext); - }; + private String getNewApplicationDigest(ProcessContext context, String fileName) { + ApplicationArchiveContext applicationArchiveContext = createApplicationArchiveContext(context, fileName); + return applicationDigestCalculator.calculateApplicationDigest(applicationArchiveContext); } - protected ApplicationArchiveContext createApplicationArchiveContext(InputStream appArchiveStream, String fileName, long maxSize) { - return new ApplicationArchiveContext(appArchiveStream, fileName, maxSize); + protected ApplicationArchiveContext createApplicationArchiveContext(ProcessContext context, String fileName) { + long maxSize = configuration.getMaxResourceFileSize(); + List archiveEntryWithStreamPositions = context.getVariable(Variables.ARCHIVE_ENTRIES_POSITIONS); + return new ApplicationArchiveContext(fileName, + maxSize, + archiveEntryWithStreamPositions, + context.getRequiredVariable(Variables.SPACE_GUID), + context.getRequiredVariable(Variables.APP_ARCHIVE_ID)); } private boolean detectApplicationFileDigestChanges(Map appEnv, String newApplicationDigest) { @@ -185,11 +183,7 @@ private void removeApplicationDigestIfSet(ProcessContext context, Map getAsyncStepExecutions(ProcessContext context) { - return List.of(new UploadAppAsyncExecution(fileService, - applicationZipBuilder, - getProcessLogsPersister(), - configuration, - appUploaderThreadPool), + return List.of(new UploadAppAsyncExecution(applicationZipBuilder, getProcessLogsPersister(), configuration, appUploaderThreadPool), new PollUploadAppStatusExecution()); } diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/ValidateDeployParametersStep.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/ValidateDeployParametersStep.java index 2360c7264a..256bc33faf 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/ValidateDeployParametersStep.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/ValidateDeployParametersStep.java @@ -216,7 +216,7 @@ private FileEntry persistArchive(ArchiveStreamWithName archiveStreamWithName, Pr () -> doPersistArchive(archiveStreamWithName, context, size))) .get(); } catch (ExecutionException | InterruptedException e) { - throw new SLException(e.getMessage(), e); + throw new SLException(e, e.getMessage()); } } diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/stream/DefaultLimitedInputStream.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/stream/DefaultLimitedInputStream.java new file mode 100644 index 0000000000..ddfa090281 --- /dev/null +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/stream/DefaultLimitedInputStream.java @@ -0,0 +1,22 @@ +package org.cloudfoundry.multiapps.controller.process.stream; + +import java.io.InputStream; + +import org.cloudfoundry.multiapps.common.ContentException; +import org.cloudfoundry.multiapps.mta.Messages; +import org.cloudfoundry.multiapps.mta.util.LimitedInputStream; + +public class DefaultLimitedInputStream extends LimitedInputStream { + + private final String fileName; + + public DefaultLimitedInputStream(InputStream in, String fileName, long maxSize) { + super(in, maxSize); + this.fileName = fileName; + } + + @Override + protected void raiseError(long maxSize, long currentSize) { + throw new ContentException(Messages.ERROR_SIZE_OF_FILE_EXCEEDS_CONFIGURED_MAX_SIZE_LIMIT, currentSize, fileName, maxSize); + } +} diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationArchiveContext.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationArchiveContext.java index 7d85207937..87c53a6892 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationArchiveContext.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationArchiveContext.java @@ -1,26 +1,32 @@ package org.cloudfoundry.multiapps.controller.process.util; -import java.io.InputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.HashSet; +import java.util.List; import java.util.Set; -import java.util.zip.ZipInputStream; import org.cloudfoundry.multiapps.controller.persistence.Constants; public class ApplicationArchiveContext { - private final ZipInputStream zipInputStream; + private final String moduleFileName; private final long maxSizeInBytes; + private final List archiveEntryWithStreamPositions; + private final String spaceId; + private final String appArchiveId; private long currentSizeInBytes; private DigestCalculator applicationDigestCalculator; private Set alreadyUploadedFiles; - public ApplicationArchiveContext(InputStream inputStream, String moduleFileName, long maxSizeInBytes) { - this.zipInputStream = new ZipInputStream(inputStream); + public ApplicationArchiveContext(String moduleFileName, long maxSizeInBytes, + List archiveEntryWithStreamPositions, String spaceId, + String appArchiveId) { this.moduleFileName = moduleFileName; this.maxSizeInBytes = maxSizeInBytes; + this.archiveEntryWithStreamPositions = archiveEntryWithStreamPositions; + this.spaceId = spaceId; + this.appArchiveId = appArchiveId; createDigestCalculator(Constants.DIGEST_ALGORITHM); } @@ -40,10 +46,6 @@ public void calculateCurrentSizeInBytes(long sizeInBytes) { currentSizeInBytes += sizeInBytes; } - public ZipInputStream getZipInputStream() { - return zipInputStream; - } - public String getModuleFileName() { return moduleFileName; } @@ -52,7 +54,7 @@ public long getMaxSizeInBytes() { return maxSizeInBytes; } - public DigestCalculator getApplicationDigestCalculator() { + public DigestCalculator getDigestCalculator() { return applicationDigestCalculator; } @@ -67,4 +69,15 @@ public void setAlreadyUploadedFiles(Set alreadyUploadedFiles) { this.alreadyUploadedFiles = alreadyUploadedFiles; } + public List getArchiveEntryWithStreamPositions() { + return archiveEntryWithStreamPositions; + } + + public String getSpaceId() { + return spaceId; + } + + public String getAppArchiveId() { + return appArchiveId; + } } diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationArchiveIterator.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationArchiveIterator.java new file mode 100644 index 0000000000..2bef9aa23f --- /dev/null +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationArchiveIterator.java @@ -0,0 +1,38 @@ +package org.cloudfoundry.multiapps.controller.process.util; + +import java.io.IOException; + +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; +import org.cloudfoundry.multiapps.common.ContentException; +import org.cloudfoundry.multiapps.controller.core.util.FileUtils; + +import jakarta.inject.Named; + +@Named +public class ApplicationArchiveIterator { + + public ZipArchiveEntry getFirstZipEntry(String moduleFileName, ZipArchiveInputStream zipArchiveInputStream) throws IOException { + ZipArchiveEntry zipEntry = getNextEntryByName(moduleFileName, zipArchiveInputStream); + if (zipEntry == null) { + throw new ContentException(org.cloudfoundry.multiapps.mta.Messages.CANNOT_FIND_ARCHIVE_ENTRY, moduleFileName); + } + return zipEntry; + } + + public ZipArchiveEntry getNextEntryByName(String name, ZipArchiveInputStream zipArchiveInputStream) throws IOException { + for (ZipArchiveEntry zipEntry; (zipEntry = zipArchiveInputStream.getNextEntry()) != null;) { + if (zipEntry.getName() + .startsWith(name)) { + validateEntry(zipEntry); + return zipEntry; + } + } + return null; + } + + protected void validateEntry(ZipArchiveEntry entry) { + FileUtils.validatePath(entry.getName()); + } + +} diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationArchiveReader.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationArchiveReader.java deleted file mode 100644 index 8205c05c2a..0000000000 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationArchiveReader.java +++ /dev/null @@ -1,80 +0,0 @@ -package org.cloudfoundry.multiapps.controller.process.util; - -import java.io.IOException; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - -import jakarta.inject.Named; - -import org.cloudfoundry.multiapps.common.ContentException; -import org.cloudfoundry.multiapps.common.SLException; -import org.cloudfoundry.multiapps.controller.core.util.FileUtils; -import org.cloudfoundry.multiapps.controller.process.Messages; - -@Named -public class ApplicationArchiveReader { - protected static final int BUFFER_SIZE = 4 * 1024; // 4KB - - public String calculateApplicationDigest(ApplicationArchiveContext applicationArchiveContext) { - try { - iterateApplicationArchive(applicationArchiveContext); - return applicationArchiveContext.getApplicationDigestCalculator() - .getDigest(); - } catch (IOException e) { - throw new SLException(e, Messages.ERROR_RETRIEVING_MTA_MODULE_CONTENT, applicationArchiveContext.getModuleFileName()); - } - } - - private void iterateApplicationArchive(ApplicationArchiveContext applicationArchiveContext) throws IOException { - String moduleFileName = applicationArchiveContext.getModuleFileName(); - ZipEntry zipEntry = getFirstZipEntry(applicationArchiveContext); - do { - if (!zipEntry.isDirectory()) { - calculateDigestFromArchive(applicationArchiveContext); - } - } while ((zipEntry = getNextEntryByName(moduleFileName, applicationArchiveContext)) != null); - } - - public ZipEntry getFirstZipEntry(ApplicationArchiveContext applicationArchiveContext) throws IOException { - String moduleFileName = applicationArchiveContext.getModuleFileName(); - ZipEntry zipEntry = getNextEntryByName(moduleFileName, applicationArchiveContext); - if (zipEntry == null) { - throw new ContentException(org.cloudfoundry.multiapps.mta.Messages.CANNOT_FIND_ARCHIVE_ENTRY, moduleFileName); - } - return zipEntry; - } - - protected void calculateDigestFromArchive(ApplicationArchiveContext applicationArchiveContext) throws IOException { - byte[] buffer = new byte[BUFFER_SIZE]; - int numberOfReadBytes = 0; - ZipInputStream zipInputStream = applicationArchiveContext.getZipInputStream(); - long maxSizeInBytes = applicationArchiveContext.getMaxSizeInBytes(); - DigestCalculator applicationDigestCalculator = applicationArchiveContext.getApplicationDigestCalculator(); - - while ((numberOfReadBytes = zipInputStream.read(buffer)) != -1) { - long currentSizeInBytes = applicationArchiveContext.getCurrentSizeInBytes(); - if (currentSizeInBytes + numberOfReadBytes > maxSizeInBytes) { - throw new ContentException(Messages.SIZE_OF_APP_EXCEEDS_MAX_SIZE_LIMIT, maxSizeInBytes); - } - applicationArchiveContext.calculateCurrentSizeInBytes(numberOfReadBytes); - applicationDigestCalculator.updateDigest(buffer, 0, numberOfReadBytes); - } - } - - public ZipEntry getNextEntryByName(String name, ApplicationArchiveContext applicationArchiveContext) throws IOException { - ZipInputStream zipInputStream = applicationArchiveContext.getZipInputStream(); - for (ZipEntry zipEntry; (zipEntry = zipInputStream.getNextEntry()) != null;) { - if (zipEntry.getName() - .startsWith(name)) { - validateEntry(zipEntry); - return zipEntry; - } - } - return null; - } - - protected void validateEntry(ZipEntry entry) { - FileUtils.validatePath(entry.getName()); - } - -} \ No newline at end of file diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationDigestCalculator.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationDigestCalculator.java new file mode 100644 index 0000000000..f10fa2cedd --- /dev/null +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationDigestCalculator.java @@ -0,0 +1,106 @@ +package org.cloudfoundry.multiapps.controller.process.util; + +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.ZipEntry; + +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; +import org.cloudfoundry.multiapps.common.ContentException; +import org.cloudfoundry.multiapps.common.SLException; +import org.cloudfoundry.multiapps.controller.persistence.services.FileService; +import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; +import org.cloudfoundry.multiapps.controller.process.Messages; + +import jakarta.inject.Inject; +import jakarta.inject.Named; + +@Named +public class ApplicationDigestCalculator { + + protected static final int BUFFER_SIZE = 4 * 1024; // 4KB + + private final FileService fileService; + private final ApplicationArchiveIterator applicationArchiveIterator; + private final ArchiveEntryExtractor archiveEntryExtractor; + + @Inject + public ApplicationDigestCalculator(FileService fileService, ApplicationArchiveIterator applicationArchiveIterator, + ArchiveEntryExtractor archiveEntryExtractor) { + this.fileService = fileService; + this.applicationArchiveIterator = applicationArchiveIterator; + this.archiveEntryExtractor = archiveEntryExtractor; + } + + public String calculateApplicationDigest(ApplicationArchiveContext applicationArchiveContext) { + try { + iterateApplicationArchive(applicationArchiveContext); + return applicationArchiveContext.getDigestCalculator() + .getDigest(); + } catch (FileStorageException e) { + throw new SLException(e, Messages.ERROR_RETRIEVING_MTA_MODULE_CONTENT, applicationArchiveContext.getModuleFileName()); + } + } + + private void iterateApplicationArchive(ApplicationArchiveContext applicationArchiveContext) throws FileStorageException { + // TODO: backwards compatibility for one tact + if (applicationArchiveContext.getArchiveEntryWithStreamPositions() == null) { + fileService.consumeFileContent(applicationArchiveContext.getSpaceId(), applicationArchiveContext.getAppArchiveId(), + archiveStream -> calculateDigestFromDirectory(applicationArchiveContext, archiveStream)); + return; + } + // TODO: backwards compatibility for one tact + if (ArchiveEntryExtractorUtil.containsDirectory(applicationArchiveContext.getModuleFileName(), + applicationArchiveContext.getArchiveEntryWithStreamPositions())) { + fileService.consumeFileContent(applicationArchiveContext.getSpaceId(), applicationArchiveContext.getAppArchiveId(), + archiveStream -> calculateDigestFromDirectory(applicationArchiveContext, archiveStream)); + } else { + ArchiveEntryWithStreamPositions archiveEntryWithStreamPositions = ArchiveEntryExtractorUtil.findEntry(applicationArchiveContext.getModuleFileName(), + applicationArchiveContext.getArchiveEntryWithStreamPositions()); + DigestCalculator applicationDigestCalculator = applicationArchiveContext.getDigestCalculator(); + archiveEntryExtractor.processFileEntryBytes(ImmutableFileEntryProperties.builder() + .guid(applicationArchiveContext.getAppArchiveId()) + .name(archiveEntryWithStreamPositions.getName()) + .spaceGuid(applicationArchiveContext.getSpaceId()) + .maxFileSizeInBytes(applicationArchiveContext.getMaxSizeInBytes()) + .build(), + archiveEntryWithStreamPositions, (bytesBuffer, bytesRead) -> { + applicationArchiveContext.calculateCurrentSizeInBytes(bytesRead); + applicationDigestCalculator.updateDigest(bytesBuffer, 0, bytesRead); + }); + } + } + + private void calculateDigestFromDirectory(ApplicationArchiveContext applicationArchiveContext, InputStream archiveStream) + throws IOException { + try (ZipArchiveInputStream zipArchiveInputStream = new ZipArchiveInputStream(archiveStream)) { + String moduleFileName = applicationArchiveContext.getModuleFileName(); + ZipEntry zipEntry = applicationArchiveIterator.getFirstZipEntry(applicationArchiveContext.getModuleFileName(), + zipArchiveInputStream); + do { + if (!zipEntry.isDirectory()) { + calculateDigestFromArchive(applicationArchiveContext, zipArchiveInputStream); + } + } while ((zipEntry = applicationArchiveIterator.getNextEntryByName(moduleFileName, zipArchiveInputStream)) != null); + } + } + + private void calculateDigestFromArchive(ApplicationArchiveContext applicationArchiveContext, InputStream inputStream) + throws IOException { + byte[] buffer = new byte[BUFFER_SIZE]; + int numberOfReadBytes = 0; + long maxSizeInBytes = applicationArchiveContext.getMaxSizeInBytes(); + DigestCalculator applicationDigestCalculator = applicationArchiveContext.getDigestCalculator(); + while ((numberOfReadBytes = inputStream.read(buffer)) != -1) { + long currentSizeInBytes = applicationArchiveContext.getCurrentSizeInBytes(); + if (currentSizeInBytes + numberOfReadBytes > maxSizeInBytes) { + throw new ContentException(org.cloudfoundry.multiapps.mta.Messages.ERROR_SIZE_OF_FILE_EXCEEDS_CONFIGURED_MAX_SIZE_LIMIT, + currentSizeInBytes + numberOfReadBytes, + applicationArchiveContext.getModuleFileName(), + maxSizeInBytes); + } + applicationArchiveContext.calculateCurrentSizeInBytes(numberOfReadBytes); + applicationDigestCalculator.updateDigest(buffer, 0, numberOfReadBytes); + } + } + +} \ No newline at end of file diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationZipBuilder.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationZipBuilder.java index db7ef2c054..cb99d76ddc 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationZipBuilder.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationZipBuilder.java @@ -5,37 +5,59 @@ import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.text.MessageFormat; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; -import jakarta.inject.Inject; -import jakarta.inject.Named; - +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; import org.apache.commons.io.FilenameUtils; import org.cloudfoundry.multiapps.common.ContentException; import org.cloudfoundry.multiapps.common.SLException; import org.cloudfoundry.multiapps.controller.core.util.FileUtils; +import org.cloudfoundry.multiapps.controller.persistence.services.FileService; +import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; import org.cloudfoundry.multiapps.controller.process.Messages; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import jakarta.inject.Inject; +import jakarta.inject.Named; + @Named public class ApplicationZipBuilder { private static final Logger LOGGER = LoggerFactory.getLogger(ApplicationZipBuilder.class); private static final int BUFFER_SIZE = 4 * 1024; // 4KB - private final ApplicationArchiveReader applicationArchiveReader; + + private final FileService fileService; + private final ApplicationArchiveIterator applicationArchiveIterator; + private final ArchiveEntryExtractor archiveEntryExtractor; @Inject - public ApplicationZipBuilder(ApplicationArchiveReader applicationArchiveReader) { - this.applicationArchiveReader = applicationArchiveReader; + public ApplicationZipBuilder(FileService fileService, ApplicationArchiveIterator applicationArchiveIterator, + ArchiveEntryExtractor archiveEntryExtractor) { + this.fileService = fileService; + this.applicationArchiveIterator = applicationArchiveIterator; + this.archiveEntryExtractor = archiveEntryExtractor; } public Path extractApplicationInNewArchive(ApplicationArchiveContext applicationArchiveContext) { Path appPath = null; try { appPath = createTempFile(); - saveAllEntries(appPath, applicationArchiveContext); + // TODO: backwards compatibility for one tact + if (applicationArchiveContext.getArchiveEntryWithStreamPositions() == null) { + extractDirectoryContent(applicationArchiveContext, appPath); + return appPath; + } + // TODO: backwards compatibility for one tact + if (ArchiveEntryExtractorUtil.containsDirectory(applicationArchiveContext.getModuleFileName(), + applicationArchiveContext.getArchiveEntryWithStreamPositions())) { + LOGGER.info(MessageFormat.format(Messages.MODULE_0_CONTENT_IS_A_DIRECTORY, applicationArchiveContext.getModuleFileName())); + extractDirectoryContent(applicationArchiveContext, appPath); + } else { + extractModuleContent(applicationArchiveContext, appPath); + } return appPath; } catch (Exception e) { FileUtils.cleanUp(appPath, LOGGER); @@ -43,51 +65,66 @@ public Path extractApplicationInNewArchive(ApplicationArchiveContext application } } - private void saveAllEntries(Path dirPath, ApplicationArchiveContext applicationArchiveContext) throws IOException { + protected Path createTempFile() { + try { + return Files.createTempFile(null, getFileExtension()); + } catch (IOException e) { + throw new SLException(e, e.getMessage()); + } + } + + private String getFileExtension() { + return FilenameUtils.EXTENSION_SEPARATOR_STR + "zip"; + } + + private void extractDirectoryContent(ApplicationArchiveContext applicationArchiveContext, Path applicationPath) + throws FileStorageException { + fileService.consumeFileContent(applicationArchiveContext.getSpaceId(), applicationArchiveContext.getAppArchiveId(), + archiveStream -> { + try (ZipArchiveInputStream zipArchiveInputStream = new ZipArchiveInputStream(archiveStream)) { + saveAllEntries(applicationPath, applicationArchiveContext, zipArchiveInputStream); + } + }); + } + + private void saveAllEntries(Path dirPath, ApplicationArchiveContext applicationArchiveContext, + ZipArchiveInputStream zipArchiveInputStream) + throws IOException { try (OutputStream fileOutputStream = Files.newOutputStream(dirPath)) { - ZipEntry zipEntry = applicationArchiveReader.getFirstZipEntry(applicationArchiveContext); + ZipEntry zipEntry = applicationArchiveIterator.getFirstZipEntry(applicationArchiveContext.getModuleFileName(), + zipArchiveInputStream); if (zipEntry.isDirectory()) { - saveAsZip(fileOutputStream, applicationArchiveContext, zipEntry); + saveAsZip(fileOutputStream, applicationArchiveContext, zipEntry, zipArchiveInputStream); } else { - saveToFile(fileOutputStream, applicationArchiveContext, zipEntry); + saveToFile(fileOutputStream, applicationArchiveContext, zipEntry, zipArchiveInputStream); } } } - private void saveAsZip(OutputStream fileOutputStream, ApplicationArchiveContext applicationArchiveContext, ZipEntry zipEntry) + private void saveAsZip(OutputStream fileOutputStream, ApplicationArchiveContext applicationArchiveContext, ZipEntry zipEntry, + ZipArchiveInputStream zipArchiveInputStream) throws IOException { try (ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream)) { String moduleFileName = applicationArchiveContext.getModuleFileName(); do { if (!isAlreadyUploaded(zipEntry.getName(), applicationArchiveContext) && !zipEntry.isDirectory()) { zipOutputStream.putNextEntry(createNewZipEntry(zipEntry.getName(), moduleFileName)); - copy(applicationArchiveContext.getZipInputStream(), zipOutputStream, applicationArchiveContext); + copy(zipArchiveInputStream, zipOutputStream, applicationArchiveContext); zipOutputStream.closeEntry(); - } - } while ((zipEntry = applicationArchiveReader.getNextEntryByName(moduleFileName, applicationArchiveContext)) != null); + } while ((zipEntry = applicationArchiveIterator.getNextEntryByName(moduleFileName, zipArchiveInputStream)) != null); } } - private ZipEntry createNewZipEntry(String zipEntryName, String moduleFileName) { - return new UtcAdjustedZipEntry(FileUtils.getRelativePath(moduleFileName, zipEntryName)); - } - - private void saveToFile(OutputStream fileOutputStream, ApplicationArchiveContext applicationArchiveContext, ZipEntry zipEntry) - throws IOException { - String moduleFileName = applicationArchiveContext.getModuleFileName(); - do { - if (!isAlreadyUploaded(zipEntry.getName(), applicationArchiveContext)) { - copy(applicationArchiveContext.getZipInputStream(), fileOutputStream, applicationArchiveContext); - } - } while ((zipEntry = applicationArchiveReader.getNextEntryByName(moduleFileName, applicationArchiveContext)) != null); - } - private boolean isAlreadyUploaded(String zipEntryName, ApplicationArchiveContext applicationArchiveContext) { return applicationArchiveContext.getAlreadyUploadedFiles() .contains(zipEntryName); } + private ZipEntry createNewZipEntry(String zipEntryName, String moduleFileName) { + return new ZipEntry(FileUtils.getRelativePath(moduleFileName, zipEntryName)); + } + protected void copy(InputStream input, OutputStream output, ApplicationArchiveContext applicationArchiveContext) throws IOException { byte[] buffer = new byte[BUFFER_SIZE]; int numberOfReadBytes = 0; @@ -95,19 +132,49 @@ protected void copy(InputStream input, OutputStream output, ApplicationArchiveCo while ((numberOfReadBytes = input.read(buffer)) != -1) { long currentSizeInBytes = applicationArchiveContext.getCurrentSizeInBytes(); if (currentSizeInBytes + numberOfReadBytes > maxSizeInBytes) { - throw new ContentException(Messages.SIZE_OF_APP_EXCEEDS_MAX_SIZE_LIMIT, maxSizeInBytes); + throw new ContentException(org.cloudfoundry.multiapps.mta.Messages.ERROR_SIZE_OF_FILE_EXCEEDS_CONFIGURED_MAX_SIZE_LIMIT, + currentSizeInBytes + numberOfReadBytes, + applicationArchiveContext.getModuleFileName(), + maxSizeInBytes); } output.write(buffer, 0, numberOfReadBytes); applicationArchiveContext.calculateCurrentSizeInBytes(numberOfReadBytes); } } - protected Path createTempFile() throws IOException { - return Files.createTempFile(null, getFileExtension()); + private void saveToFile(OutputStream fileOutputStream, ApplicationArchiveContext applicationArchiveContext, ZipEntry zipEntry, + ZipArchiveInputStream zipArchiveInputStream) + throws IOException { + String moduleFileName = applicationArchiveContext.getModuleFileName(); + do { + if (!isAlreadyUploaded(zipEntry.getName(), applicationArchiveContext)) { + copy(zipArchiveInputStream, fileOutputStream, applicationArchiveContext); + } + } while ((zipEntry = applicationArchiveIterator.getNextEntryByName(moduleFileName, zipArchiveInputStream)) != null); } - private String getFileExtension() { - return FilenameUtils.EXTENSION_SEPARATOR_STR + "zip"; + private void extractModuleContent(ApplicationArchiveContext applicationArchiveContext, Path appPath) throws IOException { + try (OutputStream fileOutputStream = Files.newOutputStream(appPath)) { + ArchiveEntryWithStreamPositions archiveEntryWithStreamPositions = ArchiveEntryExtractorUtil.findEntry(applicationArchiveContext.getModuleFileName(), + applicationArchiveContext.getArchiveEntryWithStreamPositions()); + archiveEntryExtractor.processFileEntryBytes(ImmutableFileEntryProperties.builder() + .guid(applicationArchiveContext.getAppArchiveId()) + .name(archiveEntryWithStreamPositions.getName()) + .spaceGuid(applicationArchiveContext.getSpaceId()) + .maxFileSizeInBytes(applicationArchiveContext.getMaxSizeInBytes()) + .build(), + archiveEntryWithStreamPositions, + (bytesBuffer, bytesRead) -> writeModuleContent(bytesBuffer, bytesRead, + fileOutputStream)); + } + } + + private void writeModuleContent(byte[] bytesBuffer, Integer bytesRead, OutputStream fileOutputStream) { + try { + fileOutputStream.write(bytesBuffer, 0, bytesRead); + } catch (IOException e) { + throw new SLException(e, e.getMessage()); + } } } diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ArchiveEntryExtractor.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ArchiveEntryExtractor.java new file mode 100644 index 0000000000..ce7a66fd9d --- /dev/null +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ArchiveEntryExtractor.java @@ -0,0 +1,117 @@ +package org.cloudfoundry.multiapps.controller.process.util; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.function.ObjIntConsumer; + +import org.cloudfoundry.multiapps.common.SLException; +import org.cloudfoundry.multiapps.controller.persistence.services.FileContentToProcess; +import org.cloudfoundry.multiapps.controller.persistence.services.FileService; +import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; +import org.cloudfoundry.multiapps.controller.persistence.services.ImmutableFileContentToProcess; +import org.cloudfoundry.multiapps.controller.process.stream.DefaultLimitedInputStream; +import org.cloudfoundry.multiapps.mta.util.EntryToInflate; +import org.cloudfoundry.multiapps.mta.util.InflatorUtil; + +import jakarta.inject.Inject; +import jakarta.inject.Named; + +@Named +public class ArchiveEntryExtractor { + + private static final int BUFFER_SIZE = 4 * 1024; // 4KB + + private final FileService fileService; + + @Inject + public ArchiveEntryExtractor(FileService fileService) { + this.fileService = fileService; + } + + public byte[] extractEntryBytes(FileEntryProperties fileEntryProperties, + ArchiveEntryWithStreamPositions archiveEntryWithStreamPositions) { + try { + return fileService.processFileContentWithOffset(toFileContentToProcess(fileEntryProperties, archiveEntryWithStreamPositions), + fileEntryStream -> processArchiveEntryStream(fileEntryProperties, + archiveEntryWithStreamPositions, + fileEntryStream)); + } catch (FileStorageException e) { + throw new SLException(e, e.getMessage()); + } + } + + private FileContentToProcess toFileContentToProcess(FileEntryProperties fileEntryProperties, + ArchiveEntryWithStreamPositions archiveEntryWithStreamPositions) { + return ImmutableFileContentToProcess.builder() + .guid(fileEntryProperties.getGuid()) + .spaceGuid(fileEntryProperties.getSpaceGuid()) + .startOffset(archiveEntryWithStreamPositions.getStartPosition()) + .endOffset(archiveEntryWithStreamPositions.getEndPosition()) + .build(); + } + + private byte[] processArchiveEntryStream(FileEntryProperties fileEntryProperties, + ArchiveEntryWithStreamPositions archiveEntryWithStreamPositions, InputStream fileEntryStream) + throws IOException { + if (archiveEntryWithStreamPositions.getCompressionMethod() == ArchiveEntryWithStreamPositions.CompressionMethod.STORED) { + return new DefaultLimitedInputStream(fileEntryStream, + fileEntryProperties.getName(), + fileEntryProperties.getMaxFileSizeInBytes()).readAllBytes(); + } + return inflateFileContent(fileEntryProperties, fileEntryStream); + } + + private byte[] inflateFileContent(FileEntryProperties fileEntryProperties, InputStream fileEntryStream) throws IOException { + try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { + InflatorUtil.inflate(new EntryToInflate(fileEntryProperties.getName(), + fileEntryProperties.getMaxFileSizeInBytes(), + fileEntryStream), + (bytesBuffer, bytesRead) -> byteArrayOutputStream.write(bytesBuffer, 0, bytesRead)); + return byteArrayOutputStream.toByteArray(); + } + } + + public void processFileEntryBytes(FileEntryProperties fileEntryProperties, + ArchiveEntryWithStreamPositions archiveEntryWithStreamPositions, + ObjIntConsumer decompressedBytesConsumer) { + try { + if (archiveEntryWithStreamPositions.getCompressionMethod() == ArchiveEntryWithStreamPositions.CompressionMethod.STORED) { + fileService.consumeFileContentWithOffset(toFileContentToProcess(fileEntryProperties, archiveEntryWithStreamPositions), + fileEntryStream -> processStoredEntryStream(fileEntryProperties, + decompressedBytesConsumer, + fileEntryStream)); + } else { + fileService.consumeFileContentWithOffset(toFileContentToProcess(fileEntryProperties, archiveEntryWithStreamPositions), + fileEntryStream -> processInflatedEntryStream(fileEntryProperties, + decompressedBytesConsumer, + fileEntryStream)); + } + } catch (FileStorageException e) { + throw new SLException(e, e.getMessage()); + } + } + + private void processStoredEntryStream(FileEntryProperties fileEntryProperties, ObjIntConsumer decompressedBytesConsumer, + InputStream fileEntryStream) + throws IOException { + byte[] buffer = new byte[BUFFER_SIZE]; + int bytesRead; + try (InputStream inputStream = new DefaultLimitedInputStream(fileEntryStream, + fileEntryProperties.getName(), + fileEntryProperties.getMaxFileSizeInBytes())) { + while ((bytesRead = inputStream.read(buffer)) != -1) { + decompressedBytesConsumer.accept(buffer, bytesRead); + } + } + } + + private void processInflatedEntryStream(FileEntryProperties fileEntryProperties, ObjIntConsumer decompressedBytesConsumer, + InputStream fileEntryStream) { + InflatorUtil.inflate(new EntryToInflate(fileEntryProperties.getName(), + fileEntryProperties.getMaxFileSizeInBytes(), + fileEntryStream), + decompressedBytesConsumer); + } + +} diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ArchiveEntryExtractorUtil.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ArchiveEntryExtractorUtil.java new file mode 100644 index 0000000000..ae9b61a640 --- /dev/null +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ArchiveEntryExtractorUtil.java @@ -0,0 +1,30 @@ +package org.cloudfoundry.multiapps.controller.process.util; + +import java.util.List; + +import org.cloudfoundry.multiapps.common.SLException; +import org.cloudfoundry.multiapps.mta.Messages; + +public class ArchiveEntryExtractorUtil { + + private ArchiveEntryExtractorUtil() { + + } + + public static ArchiveEntryWithStreamPositions findEntry(String entryName, + List archiveEntriesWithStreamPositions) { + return archiveEntriesWithStreamPositions.stream() + .filter(e -> e.getName() + .startsWith(entryName)) + .findFirst() + .orElseThrow(() -> new SLException(Messages.CANNOT_FIND_ARCHIVE_ENTRY, entryName)); + } + + public static boolean containsDirectory(String entryName, List archiveEntriesWithStreamPositions) { + return archiveEntriesWithStreamPositions.stream() + .filter(entry -> entry.getName() + .startsWith(entryName)) + .anyMatch(ArchiveEntryWithStreamPositions::isDirectory); + } + +} diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ArchiveEntryStreamWithStreamPositionsDeterminer.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ArchiveEntryStreamWithStreamPositionsDeterminer.java new file mode 100644 index 0000000000..86adf6d41c --- /dev/null +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ArchiveEntryStreamWithStreamPositionsDeterminer.java @@ -0,0 +1,67 @@ +package org.cloudfoundry.multiapps.controller.process.util; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.zip.ZipEntry; + +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; +import org.cloudfoundry.multiapps.common.SLException; +import org.cloudfoundry.multiapps.controller.core.util.FileUtils; +import org.cloudfoundry.multiapps.controller.persistence.services.FileService; +import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; + +import jakarta.inject.Inject; +import jakarta.inject.Named; + +@Named +public class ArchiveEntryStreamWithStreamPositionsDeterminer { + + public static final int BUFFER_SIZE = 4 * 1024; // 4KB + + private final FileService fileService; + + @Inject + public ArchiveEntryStreamWithStreamPositionsDeterminer(FileService fileService) { + this.fileService = fileService; + } + + public List determineArchiveEntries(String spaceGuid, String appArchiveId) { + try { + return fileService.processFileContent(spaceGuid, appArchiveId, archiveStream -> { + List archiveEntriesWithPositions = new ArrayList<>(); + try ( + ZipArchiveInputStream zipStream = new ZipArchiveInputStream(archiveStream, StandardCharsets.UTF_8.name(), true, true)) { + ZipArchiveEntry entry = zipStream.getNextEntry(); + while (entry != null) { + validateEntry(entry); + long startOffset = entry.getDataOffset(); + long endOffset = startOffset; + byte[] buffer = new byte[BUFFER_SIZE]; + while (zipStream.read(buffer, 0, buffer.length) != -1) { + // read the entry, to calculate the compressed size + } + endOffset += zipStream.getCompressedCount(); + archiveEntriesWithPositions.add(ImmutableArchiveEntryWithStreamPositions.builder() + .name(entry.getName()) + .startPosition(startOffset) + .endPosition(endOffset) + .compressionMethod(ArchiveEntryWithStreamPositions.CompressionMethod.parseValue(entry.getMethod())) + .isDirectory(entry.isDirectory()) + .build()); + entry = zipStream.getNextEntry(); + } + } + return archiveEntriesWithPositions; + }); + } catch (FileStorageException e) { + throw new SLException(e, e.getMessage()); + } + } + + protected void validateEntry(ZipEntry entry) { + FileUtils.validatePath(entry.getName()); + } + +} diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ArchiveEntryWithStreamPositions.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ArchiveEntryWithStreamPositions.java new file mode 100644 index 0000000000..bb2d544ba0 --- /dev/null +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ArchiveEntryWithStreamPositions.java @@ -0,0 +1,43 @@ +package org.cloudfoundry.multiapps.controller.process.util; + +import java.util.Arrays; + +import org.cloudfoundry.multiapps.common.SLException; +import org.cloudfoundry.multiapps.controller.process.Messages; +import org.immutables.value.Value; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +@Value.Immutable +@JsonSerialize(as = ImmutableArchiveEntryWithStreamPositions.class) +@JsonDeserialize(as = ImmutableArchiveEntryWithStreamPositions.class) +public interface ArchiveEntryWithStreamPositions { + + String getName(); + + long getStartPosition(); + + long getEndPosition(); + + CompressionMethod getCompressionMethod(); + + boolean isDirectory(); + + enum CompressionMethod { + STORED(0), DEFLATED(8); + + private final int value; + + CompressionMethod(int value) { + this.value = value; + } + + public static CompressionMethod parseValue(int value) { + return Arrays.stream(values()) + .filter(entry -> entry.value == value) + .findFirst() + .orElseThrow(() -> new SLException(Messages.COMPRESSION_METHOD_WITH_VALUE_0_NOT_FOUND, value)); + } + } +} diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/FileEntryProperties.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/FileEntryProperties.java new file mode 100644 index 0000000000..3441e98ead --- /dev/null +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/FileEntryProperties.java @@ -0,0 +1,15 @@ +package org.cloudfoundry.multiapps.controller.process.util; + +import org.immutables.value.Value; + +@Value.Immutable +public interface FileEntryProperties { + + String getGuid(); + + String getName(); + + String getSpaceGuid(); + + long getMaxFileSizeInBytes(); +} diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ServiceBindingParametersGetter.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ServiceBindingParametersGetter.java index 1122d6b99b..3db44c5874 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ServiceBindingParametersGetter.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/ServiceBindingParametersGetter.java @@ -1,5 +1,6 @@ package org.cloudfoundry.multiapps.controller.process.util; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Collections; @@ -15,7 +16,6 @@ import org.cloudfoundry.multiapps.controller.client.lib.domain.CloudServiceInstanceExtended; import org.cloudfoundry.multiapps.controller.core.helpers.MtaArchiveElements; import org.cloudfoundry.multiapps.controller.core.security.serialization.SecureSerialization; -import org.cloudfoundry.multiapps.controller.persistence.services.FileContentProcessor; import org.cloudfoundry.multiapps.controller.persistence.services.FileService; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; import org.cloudfoundry.multiapps.controller.process.Messages; @@ -34,17 +34,19 @@ public class ServiceBindingParametersGetter { private final ProcessContext context; - private final FileService fileService; + private final ArchiveEntryExtractor archiveEntryExtractor; private final long maxManifestSize; + private final FileService fileService; - public ServiceBindingParametersGetter(ProcessContext context, FileService fileService, long maxManifestSize) { + public ServiceBindingParametersGetter(ProcessContext context, ArchiveEntryExtractor archiveEntryExtractor, long maxManifestSize, + FileService fileService) { this.context = context; - this.fileService = fileService; + this.archiveEntryExtractor = archiveEntryExtractor; this.maxManifestSize = maxManifestSize; + this.fileService = fileService; } - public Map getServiceBindingParametersFromMta(CloudApplicationExtended app, String serviceName) - throws FileStorageException { + public Map getServiceBindingParametersFromMta(CloudApplicationExtended app, String serviceName) { Optional service = getService(context.getVariable(Variables.SERVICES_TO_BIND), serviceName); if (service.isEmpty()) { return Collections.emptyMap(); @@ -66,30 +68,49 @@ private Optional getService(List getFileProvidedBindingParameters(String moduleName, CloudServiceInstanceExtended service) - throws FileStorageException { - + private Map getFileProvidedBindingParameters(String moduleName, CloudServiceInstanceExtended service) { String requiredDependencyName = NameUtil.getPrefixedName(moduleName, service.getResourceName(), org.cloudfoundry.multiapps.controller.core.Constants.MTA_ELEMENT_SEPARATOR); return getFileProvidedBindingParameters(requiredDependencyName); - } - private Map getFileProvidedBindingParameters(String requiredDependencyName) throws FileStorageException { - String archiveId = context.getRequiredVariable(Variables.APP_ARCHIVE_ID); + private Map getFileProvidedBindingParameters(String requiredDependencyName) { MtaArchiveElements mtaArchiveElements = context.getVariable(Variables.MTA_ARCHIVE_ELEMENTS); String fileName = mtaArchiveElements.getRequiredDependencyFileName(requiredDependencyName); if (fileName == null) { return Collections.emptyMap(); } - FileContentProcessor> fileProcessor = archive -> { - try (InputStream file = ArchiveHandler.getInputStream(archive, fileName, maxManifestSize)) { - return JsonUtil.convertJsonToMap(file); - } catch (IOException e) { - throw new SLException(e, Messages.ERROR_RETRIEVING_MTA_REQUIRED_DEPENDENCY_CONTENT, fileName); + String spaceGuid = context.getRequiredVariable(Variables.SPACE_GUID); + String appArchiveId = context.getRequiredVariable(Variables.APP_ARCHIVE_ID); + + // TODO: backwards compatibility for one tact + List archiveEntriesWithStreamPositions = context.getVariable(Variables.ARCHIVE_ENTRIES_POSITIONS); + if (archiveEntriesWithStreamPositions == null) { + try { + return fileService.processFileContent(spaceGuid, appArchiveId, appArchiveStream -> { + InputStream fileStream = ArchiveHandler.getInputStream(appArchiveStream, fileName, maxManifestSize); + return JsonUtil.convertJsonToMap(fileStream); + }); + } catch (FileStorageException e) { + throw new SLException(e, e.getMessage()); } - }; - return fileService.processFileContent(context.getVariable(Variables.SPACE_GUID), archiveId, fileProcessor); + } + // TODO: backwards compatibility for one tact + + ArchiveEntryWithStreamPositions archiveEntryWithStreamPositions = ArchiveEntryExtractorUtil.findEntry(fileName, + context.getVariable(Variables.ARCHIVE_ENTRIES_POSITIONS)); + byte[] serviceBindingParametersFileContent = archiveEntryExtractor.extractEntryBytes(ImmutableFileEntryProperties.builder() + .guid(appArchiveId) + .name(archiveEntryWithStreamPositions.getName()) + .spaceGuid(context.getRequiredVariable(Variables.SPACE_GUID)) + .maxFileSizeInBytes(maxManifestSize) + .build(), + archiveEntryWithStreamPositions); + try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(serviceBindingParametersFileContent)) { + return JsonUtil.convertJsonToMap(byteArrayInputStream); + } catch (IOException e) { + throw new SLException(e, Messages.ERROR_RETRIEVING_MTA_REQUIRED_DEPENDENCY_CONTENT, fileName); + } } private Map getDescriptorProvidedBindingParameters(CloudApplicationExtended app, CloudServiceInstanceExtended service) { diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/UtcAdjustedZipEntry.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/UtcAdjustedZipEntry.java deleted file mode 100644 index 317fa2c7cb..0000000000 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/UtcAdjustedZipEntry.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.cloudfoundry.multiapps.controller.process.util; - -import java.time.OffsetDateTime; -import java.util.zip.ZipEntry; - -public class UtcAdjustedZipEntry extends ZipEntry { - - public UtcAdjustedZipEntry(String name) { - super(name); - setTime(OffsetDateTime.now() - .toEpochSecond()); - } - - public UtcAdjustedZipEntry(ZipEntry e) { - super(e); - setTime(OffsetDateTime.now() - .toEpochSecond()); - } -} \ No newline at end of file diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/variables/Variables.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/variables/Variables.java index 7887ee0187..8b838de76d 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/variables/Variables.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/variables/Variables.java @@ -28,6 +28,7 @@ import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.process.DeployStrategy; import org.cloudfoundry.multiapps.controller.process.steps.StepPhase; +import org.cloudfoundry.multiapps.controller.process.util.ArchiveEntryWithStreamPositions; import org.cloudfoundry.multiapps.controller.process.util.ServiceAction; import org.cloudfoundry.multiapps.controller.process.util.ServiceDeletionActions; import org.cloudfoundry.multiapps.mta.model.DeploymentDescriptor; @@ -847,4 +848,10 @@ public Serializer> getSerializer() { .name("shouldSkipApplicationUpload") .defaultValue(false) .build(); + + Variable> ARCHIVE_ENTRIES_POSITIONS = ImmutableJsonStringListVariable. builder() + .name("archiveEntriesPositions") + .type(new TypeReference<>() { + }) + .build(); } diff --git a/multiapps-controller-process/src/main/resources/org/cloudfoundry/multiapps/controller/process/process-batches-sequentially.bpmn b/multiapps-controller-process/src/main/resources/org/cloudfoundry/multiapps/controller/process/process-batches-sequentially.bpmn index 0dc4f89bad..608c109710 100644 --- a/multiapps-controller-process/src/main/resources/org/cloudfoundry/multiapps/controller/process/process-batches-sequentially.bpmn +++ b/multiapps-controller-process/src/main/resources/org/cloudfoundry/multiapps/controller/process/process-batches-sequentially.bpmn @@ -1,5 +1,5 @@ - + @@ -34,6 +34,7 @@ + @@ -56,20 +57,24 @@ - + - + + + - - - + + + + + - + diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/DetermineServiceCreateUpdateServiceActionsStepTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/DetermineServiceCreateUpdateServiceActionsStepTest.java index c8dfb8cc56..7800704780 100644 --- a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/DetermineServiceCreateUpdateServiceActionsStepTest.java +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/DetermineServiceCreateUpdateServiceActionsStepTest.java @@ -23,6 +23,7 @@ import org.cloudfoundry.multiapps.controller.core.model.DynamicResolvableParameter; import org.cloudfoundry.multiapps.controller.core.model.ImmutableDynamicResolvableParameter; import org.cloudfoundry.multiapps.controller.process.Messages; +import org.cloudfoundry.multiapps.controller.process.util.ArchiveEntryExtractor; import org.cloudfoundry.multiapps.controller.process.util.ServiceAction; import org.cloudfoundry.multiapps.controller.process.variables.Variables; import org.junit.jupiter.api.Test; @@ -43,6 +44,8 @@ class DetermineServiceCreateUpdateServiceActionsStepTest extends SyncFlowableSte private static final String SERVICE_NAME = "service"; + private ArchiveEntryExtractor archiveEntryExtractor; + public static Stream testExecute() { return Stream.of( // @formatter:off @@ -330,7 +333,8 @@ private CloudServiceInstanceExtended createMockUserProvidedServiceInstance(Cloud @Override protected DetermineServiceCreateUpdateServiceActionsStep createStep() { - return new DetermineServiceCreateUpdateServiceActionsStep(); + archiveEntryExtractor = Mockito.mock(ArchiveEntryExtractor.class); + return new DetermineServiceCreateUpdateServiceActionsStep(archiveEntryExtractor); } private static class StepInput { diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/ProcessMtaArchiveStepTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/ProcessMtaArchiveStepTest.java index 2ebf97f152..ed225acb7f 100644 --- a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/ProcessMtaArchiveStepTest.java +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/ProcessMtaArchiveStepTest.java @@ -5,20 +5,26 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; +import java.io.IOException; +import java.io.InputStream; import java.util.List; import java.util.Set; import java.util.jar.Manifest; import java.util.stream.Collectors; import org.cloudfoundry.multiapps.common.ParsingException; +import org.cloudfoundry.multiapps.common.SLException; import org.cloudfoundry.multiapps.common.test.TestUtil; import org.cloudfoundry.multiapps.common.util.JsonUtil; import org.cloudfoundry.multiapps.controller.core.helpers.DescriptorParserFacadeFactory; import org.cloudfoundry.multiapps.controller.core.helpers.MtaArchiveHelper; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.persistence.services.FileContentProcessor; +import org.cloudfoundry.multiapps.controller.process.util.ArchiveEntryExtractor; +import org.cloudfoundry.multiapps.controller.process.util.ArchiveEntryStreamWithStreamPositionsDeterminer; import org.cloudfoundry.multiapps.controller.process.util.ProcessConflictPreventer; import org.cloudfoundry.multiapps.controller.process.variables.Variables; import org.cloudfoundry.multiapps.mta.handlers.DescriptorParserFacade; @@ -36,6 +42,8 @@ class ProcessMtaArchiveStepTest extends SyncFlowableStepTest archiveFileLocations; diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecutionTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecutionTest.java index f7b4d75acd..e0cabad88e 100644 --- a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecutionTest.java +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecutionTest.java @@ -6,7 +6,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; @@ -15,7 +14,7 @@ import static org.mockito.Mockito.when; import java.io.File; -import java.io.InputStream; +import java.io.FileInputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -32,11 +31,13 @@ import org.cloudfoundry.multiapps.controller.client.lib.domain.ImmutableCloudApplicationExtended; import org.cloudfoundry.multiapps.controller.core.helpers.MtaArchiveElements; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; -import org.cloudfoundry.multiapps.controller.persistence.services.FileContentProcessor; -import org.cloudfoundry.multiapps.controller.process.util.ApplicationArchiveContext; -import org.cloudfoundry.multiapps.controller.process.util.ApplicationArchiveReader; +import org.cloudfoundry.multiapps.controller.persistence.services.FileContentConsumer; +import org.cloudfoundry.multiapps.controller.persistence.services.FileService; +import org.cloudfoundry.multiapps.controller.process.util.ApplicationArchiveIterator; import org.cloudfoundry.multiapps.controller.process.util.ApplicationZipBuilder; -import org.cloudfoundry.multiapps.controller.process.util.CloudPackagesGetter; +import org.cloudfoundry.multiapps.controller.process.util.ArchiveEntryExtractor; +import org.cloudfoundry.multiapps.controller.process.util.ArchiveEntryWithStreamPositions; +import org.cloudfoundry.multiapps.controller.process.util.ImmutableArchiveEntryWithStreamPositions; import org.cloudfoundry.multiapps.controller.process.variables.Variables; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -55,6 +56,13 @@ class UploadAppAsyncExecutionTest extends AsyncStepOperationTest private static final String APP_NAME = "sample-app-backend"; private static final String APP_FILE = "web.zip"; + private static final ArchiveEntryWithStreamPositions ARCHIVE_ENTRY_WITH_STREAM_POSITIONS = ImmutableArchiveEntryWithStreamPositions.builder() + .name(APP_FILE) + .startPosition(37) + .endPosition(5012) + .compressionMethod(ArchiveEntryWithStreamPositions.CompressionMethod.DEFLATED) + .isDirectory(false) + .build(); private static final String SPACE = "space"; private static final String APP_ARCHIVE = "sample-app.mtar"; private static final CloudOperationException CO_EXCEPTION = new CloudOperationException(HttpStatus.BAD_REQUEST); @@ -72,7 +80,6 @@ class UploadAppAsyncExecutionTest extends AsyncStepOperationTest .status(Status.AWAITING_UPLOAD) .build(); private final MtaArchiveElements mtaArchiveElements = new MtaArchiveElements(); - private final CloudPackagesGetter cloudPackagesGetter = mock(CloudPackagesGetter.class); private final ExecutorService appUploaderThreadPool = mock(ExecutorService.class); @TempDir @@ -85,6 +92,9 @@ class UploadAppAsyncExecutionTest extends AsyncStepOperationTest public void setUp() throws Exception { prepareFileService(); prepareContext(); + step.applicationZipBuilder = spy(new ApplicationZipBuilderMock(fileService, + new ApplicationArchiveIterator(), + new ArchiveEntryExtractor(fileService))); } @SuppressWarnings("rawtypes") @@ -95,10 +105,11 @@ private void prepareFileService() throws Exception { Files.createFile(appFile); } doAnswer(invocation -> { - FileContentProcessor contentProcessor = invocation.getArgument(2); - return contentProcessor.process(null); + FileContentConsumer fileContentConsumer = invocation.getArgument(1); + fileContentConsumer.consume(new FileInputStream(appFile.toFile())); + return null; }).when(fileService) - .processFileContent(anyString(), anyString(), any()); + .consumeFileContentWithOffset(any(), any()); } private void prepareContext() { @@ -123,6 +134,7 @@ private void prepareContext() { @Test void testFailedUploadWithException() { prepareExecutorService(); + context.setVariable(Variables.ARCHIVE_ENTRIES_POSITIONS, List.of(ARCHIVE_ENTRY_WITH_STREAM_POSITIONS)); when(client.asyncUploadApplicationWithExponentialBackoff(eq(APP_NAME), eq(appFile), any(UploadStatusCallback.class), any())).thenThrow(CO_EXCEPTION); expectedStatus = AsyncExecutionState.ERROR; @@ -163,6 +175,7 @@ void testUploadExecutorCapacityIsFull() { @Test void testSuccessfulUpload() { prepareExecutorService(); + context.setVariable(Variables.ARCHIVE_ENTRIES_POSITIONS, List.of(ARCHIVE_ENTRY_WITH_STREAM_POSITIONS)); when(client.asyncUploadApplicationWithExponentialBackoff(eq(APP_NAME), eq(appFile), any(UploadStatusCallback.class), any())).thenReturn(CLOUD_PACKAGE); expectedStatus = AsyncExecutionState.FINISHED; @@ -195,39 +208,28 @@ protected UploadAppStep createStep() { private class UploadAppStepMock extends UploadAppStep { - public UploadAppStepMock() { - applicationArchiveReader = getApplicationArchiveReader(); - applicationZipBuilder = spy(getApplicationZipBuilder(applicationArchiveReader)); - cloudPackagesGetter = UploadAppAsyncExecutionTest.this.cloudPackagesGetter; - } - - private ApplicationArchiveReader getApplicationArchiveReader() { - return new ApplicationArchiveReader(); - } - - private ApplicationZipBuilder getApplicationZipBuilder(ApplicationArchiveReader applicationArchiveReader) { - return new ApplicationZipBuilder(applicationArchiveReader) { - @Override - protected Path createTempFile() { - return appFile; - } - }; - } - @Override protected List getAsyncStepExecutions(ProcessContext context) { - - return List.of(new UploadAppAsyncExecution(fileService, - applicationZipBuilder, + return List.of(new UploadAppAsyncExecution(applicationZipBuilder, getProcessLogsPersister(), configuration, appUploaderThreadPool) { - @Override - protected ApplicationArchiveContext createApplicationArchiveContext(InputStream appArchiveStream, String fileName, - long maxSize) { - return super.createApplicationArchiveContext(getClass().getResourceAsStream(APP_ARCHIVE), fileName, maxSize); - } + }); } } + + private class ApplicationZipBuilderMock extends ApplicationZipBuilder { + + public ApplicationZipBuilderMock(FileService fileService, ApplicationArchiveIterator applicationArchiveIterator, + ArchiveEntryExtractor archiveEntryExtractor) { + super(fileService, applicationArchiveIterator, archiveEntryExtractor); + } + + @Override + protected Path createTempFile() { + return appFile; + } + } + } diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStepGeneralTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStepGeneralTest.java index e69105cd2f..e8e5c521d6 100644 --- a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStepGeneralTest.java +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStepGeneralTest.java @@ -3,7 +3,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -11,7 +10,6 @@ import static org.mockito.Mockito.when; import java.io.File; -import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -30,11 +28,15 @@ import org.cloudfoundry.multiapps.controller.core.Constants; import org.cloudfoundry.multiapps.controller.core.helpers.MtaArchiveElements; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; -import org.cloudfoundry.multiapps.controller.persistence.services.FileContentProcessor; -import org.cloudfoundry.multiapps.controller.process.util.ApplicationArchiveContext; -import org.cloudfoundry.multiapps.controller.process.util.ApplicationArchiveReader; +import org.cloudfoundry.multiapps.controller.persistence.services.FileContentConsumer; +import org.cloudfoundry.multiapps.controller.persistence.services.FileService; +import org.cloudfoundry.multiapps.controller.process.util.ApplicationArchiveIterator; +import org.cloudfoundry.multiapps.controller.process.util.ApplicationDigestCalculator; import org.cloudfoundry.multiapps.controller.process.util.ApplicationZipBuilder; +import org.cloudfoundry.multiapps.controller.process.util.ArchiveEntryExtractor; +import org.cloudfoundry.multiapps.controller.process.util.ArchiveEntryWithStreamPositions; import org.cloudfoundry.multiapps.controller.process.util.CloudPackagesGetter; +import org.cloudfoundry.multiapps.controller.process.util.ImmutableArchiveEntryWithStreamPositions; import org.cloudfoundry.multiapps.controller.process.variables.Variables; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -59,6 +61,13 @@ class UploadAppStepGeneralTest extends SyncFlowableStepTest { private static final String APP_NAME = "sample-app-backend"; private static final String APP_FILE = "web.zip"; + private static final ArchiveEntryWithStreamPositions ARCHIVE_ENTRY_WITH_STREAM_POSITIONS = ImmutableArchiveEntryWithStreamPositions.builder() + .name(APP_FILE) + .startPosition(37) + .endPosition(5012) + .compressionMethod(ArchiveEntryWithStreamPositions.CompressionMethod.DEFLATED) + .isDirectory(false) + .build(); private static final String SPACE = "space"; private static final String APP_ARCHIVE = "sample-app.mtar"; private static final String CURRENT_MODULE_DIGEST = "439B99DFFD0583200D5D21F4CD1BF035"; @@ -121,6 +130,10 @@ private static Stream testWithBuildStates() { public void setUp() throws Exception { prepareFileService(); prepareContext(); + step.applicationZipBuilder = spy(new ApplicationZipBuilderMock(fileService, + new ApplicationArchiveIterator(), + new ArchiveEntryExtractor(fileService))); + step.applicationDigestCalculator = mock(ApplicationDigestCalculator.class); } @SuppressWarnings("rawtypes") @@ -131,10 +144,11 @@ private void prepareFileService() throws Exception { Files.createFile(appFile); } doAnswer(invocation -> { - FileContentProcessor contentProcessor = invocation.getArgument(2); - return contentProcessor.process(null); + FileContentConsumer fileContentConsumer = invocation.getArgument(1); + fileContentConsumer.consume(null); + return null; }).when(fileService) - .processFileContent(anyString(), anyString(), any()); + .consumeFileContentWithOffset(any(), any()); } private void prepareContext() { @@ -154,6 +168,7 @@ private void prepareContext() { context.setVariable(Variables.VCAP_APP_PROPERTIES_CHANGED, false); when(configuration.getMaxResourceFileSize()).thenReturn(ApplicationConfiguration.DEFAULT_MAX_RESOURCE_FILE_SIZE); context.setVariable(Variables.DEPLOYMENT_DESCRIPTOR, descriptor); + context.setVariable(Variables.ARCHIVE_ENTRIES_POSITIONS, List.of(ARCHIVE_ENTRY_WITH_STREAM_POSITIONS)); } @AfterEach @@ -251,6 +266,7 @@ private void prepareClients(String applicationDigest) { CloudApplicationExtended application = createApplication(applicationDigest); when(client.getApplicationEnvironment(APP_GUID)).thenReturn(application.getEnv()); when(client.getApplication(APP_NAME)).thenReturn(application); + when(step.applicationDigestCalculator.calculateApplicationDigest(any())).thenReturn(applicationDigest); } private CloudApplicationExtended createApplication(String digest) { @@ -268,35 +284,20 @@ private CloudApplicationExtended createApplication(String digest) { @Override protected UploadAppStep createStep() { - return new UploadAppStepMock(); + return new UploadAppStep(); } - private class UploadAppStepMock extends UploadAppStep { + private class ApplicationZipBuilderMock extends ApplicationZipBuilder { - public UploadAppStepMock() { - applicationArchiveReader = getApplicationArchiveReader(); - applicationZipBuilder = spy(getApplicationZipBuilder(applicationArchiveReader)); - cloudPackagesGetter = UploadAppStepGeneralTest.this.cloudPackagesGetter; - } - - private ApplicationArchiveReader getApplicationArchiveReader() { - return new ApplicationArchiveReader(); - } - - private ApplicationZipBuilder getApplicationZipBuilder(ApplicationArchiveReader applicationArchiveReader) { - return new ApplicationZipBuilder(applicationArchiveReader) { - @Override - protected Path createTempFile() { - return appFile; - } - }; + public ApplicationZipBuilderMock(FileService fileService, ApplicationArchiveIterator applicationArchiveIterator, + ArchiveEntryExtractor archiveEntryExtractor) { + super(fileService, applicationArchiveIterator, archiveEntryExtractor); } @Override - protected ApplicationArchiveContext createApplicationArchiveContext(InputStream appArchiveStream, String fileName, long maxSize) { - return super.createApplicationArchiveContext(getClass().getResourceAsStream(APP_ARCHIVE), fileName, maxSize); + protected Path createTempFile() { + return appFile; } - } } diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationArchiveIteratorTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationArchiveIteratorTest.java new file mode 100644 index 0000000000..3e16fff234 --- /dev/null +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationArchiveIteratorTest.java @@ -0,0 +1,91 @@ +package org.cloudfoundry.multiapps.controller.process.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Paths; +import java.text.MessageFormat; +import java.util.stream.Stream; + +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; +import org.apache.commons.io.FilenameUtils; +import org.cloudfoundry.multiapps.controller.core.util.FileUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.MockitoAnnotations; + +class ApplicationArchiveIteratorTest { + + private static final String SAMPLE_FLAT_MTAR = "com.sap.mta.sample-1.2.1-beta-flat.mtar"; + private static final String SAMPLE_MTAR_WITH_JAR_ENTRY_ABSOLUTE_PATH = "archive-entry-with-absolute-path.mtar"; + private static final String SAMPLE_MTAR_WITH_JAR_ENTRY_NOT_NORMALIZED_PATH = "archive-entry-with-not-normalized-path.mtar"; + + @BeforeEach + void setUp() throws Exception { + MockitoAnnotations.openMocks(this) + .close(); + } + + static Stream testFailingCalculateDigest() { + return Stream.of(Arguments.of(SAMPLE_FLAT_MTAR, "xxx/", + MessageFormat.format(org.cloudfoundry.multiapps.mta.Messages.CANNOT_FIND_ARCHIVE_ENTRY, "xxx/")), + Arguments.of(SAMPLE_MTAR_WITH_JAR_ENTRY_NOT_NORMALIZED_PATH, "web/../asd", + MessageFormat.format(FileUtils.PATH_SHOULD_BE_NORMALIZED, "web/../asd"))); + } + + @ParameterizedTest + @MethodSource + void testFailingCalculateDigest(String mtar, String fileName, String expectedException) { + ApplicationArchiveIterator applicationArchiveIterator = new ApplicationArchiveIterator(); + try (InputStream inputStream = getClass().getResourceAsStream(mtar); + ZipArchiveInputStream zipArchiveInputStream = new ZipArchiveInputStream(inputStream)) { + Exception exception = assertThrows(Exception.class, + () -> applicationArchiveIterator.getFirstZipEntry(fileName, zipArchiveInputStream)); + assertEquals(expectedException, exception.getMessage()); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + + @Test + void testBadAbsolutePathRead() { + String mtar = SAMPLE_MTAR_WITH_JAR_ENTRY_ABSOLUTE_PATH; + String fileName = "/web/"; + String expectedException = MessageFormat.format(FileUtils.PATH_SHOULD_NOT_BE_ABSOLUTE, "/web/"); + ApplicationArchiveIterator applicationArchiveIterator = getApplicationArchiveReaderForAbsolutePath(); + try (InputStream inputStream = getClass().getResourceAsStream(mtar); + ZipArchiveInputStream zipArchiveInputStream = new ZipArchiveInputStream(inputStream)) { + Exception exception = Assertions.assertThrows(Exception.class, + () -> applicationArchiveIterator.getFirstZipEntry(fileName, + zipArchiveInputStream)); + assertEquals(expectedException, exception.getMessage()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private ApplicationArchiveIterator getApplicationArchiveReaderForAbsolutePath() { + return new ApplicationArchiveIterator() { + @Override + protected void validateEntry(ZipArchiveEntry entry) { + String path = entry.getName(); + if (!path.equals(FilenameUtils.normalize(path, true))) { + throw new IllegalArgumentException(MessageFormat.format(FileUtils.PATH_SHOULD_BE_NORMALIZED, path)); + } + if (FilenameUtils.getPrefixLength(path) != 0 || Paths.get(path) + .isAbsolute()) { + throw new IllegalArgumentException(MessageFormat.format(FileUtils.PATH_SHOULD_NOT_BE_ABSOLUTE, path)); + } + } + }; + } + +} diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationArchiveReaderTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationArchiveReaderTest.java deleted file mode 100644 index bea83dbb9a..0000000000 --- a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationArchiveReaderTest.java +++ /dev/null @@ -1,86 +0,0 @@ -package org.cloudfoundry.multiapps.controller.process.util; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import java.io.InputStream; -import java.nio.file.Paths; -import java.text.MessageFormat; -import java.util.stream.Stream; -import java.util.zip.ZipEntry; - -import org.apache.commons.io.FilenameUtils; -import org.cloudfoundry.multiapps.controller.core.util.FileUtils; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; - -class ApplicationArchiveReaderTest { - - private static final String ERROR_SIZE_OF_APP_EXCEEDS_MAX_SIZE_LIMIT = "The size of the application exceeds max size limit \"{0}\""; - private static final String SAMPLE_MTAR = "com.sap.mta.sample-1.2.1-beta.mtar"; - private static final String SAMPLE_FLAT_MTAR = "com.sap.mta.sample-1.2.1-beta-flat.mtar"; - private static final String SAMPLE_MTAR_WITH_JAR_ENTRY_ABSOLUTE_PATH = "archive-entry-with-absolute-path.mtar"; - private static final String SAMPLE_MTAR_WITH_JAR_ENTRY_NOT_NORMALIZED_PATH = "archive-entry-with-not-normalized-path.mtar"; - private static final long MAX_UPLOAD_FILE_SIZE = 1024 * 1024 * 1024L; // 1gb - - static Stream testFailingCalculateDigest() { - return Stream.of(Arguments.of(SAMPLE_FLAT_MTAR, "xxx/", - MessageFormat.format(org.cloudfoundry.multiapps.mta.Messages.CANNOT_FIND_ARCHIVE_ENTRY, "xxx/"), - MAX_UPLOAD_FILE_SIZE), - Arguments.of(SAMPLE_MTAR_WITH_JAR_ENTRY_NOT_NORMALIZED_PATH, "web/", - MessageFormat.format(FileUtils.PATH_SHOULD_BE_NORMALIZED, "web/../asd"), MAX_UPLOAD_FILE_SIZE), - Arguments.of(SAMPLE_MTAR, "db/", MessageFormat.format(ERROR_SIZE_OF_APP_EXCEEDS_MAX_SIZE_LIMIT, 200), 200L), - Arguments.of(SAMPLE_MTAR, "web/web-server.zip", - MessageFormat.format(ERROR_SIZE_OF_APP_EXCEEDS_MAX_SIZE_LIMIT, 200), 200)); - } - - @ParameterizedTest - @MethodSource - void testFailingCalculateDigest(String mtar, String fileName, String expectedException, long maxFileUploadSize) { - ApplicationArchiveContext applicationArchiveContext = getApplicationArchiveContext(mtar, fileName, maxFileUploadSize); - ApplicationArchiveReader reader = getApplicationArchiveReader(); - Exception exception = Assertions.assertThrows(Exception.class, () -> reader.calculateApplicationDigest(applicationArchiveContext)); - assertEquals(expectedException, exception.getMessage()); - } - - @Test - void testBadAbsolutePathRead() { - String mtar = SAMPLE_MTAR_WITH_JAR_ENTRY_ABSOLUTE_PATH; - String fileName = "/web/"; - String expectedException = MessageFormat.format(FileUtils.PATH_SHOULD_NOT_BE_ABSOLUTE, "/web/"); - long maxFileUploadSize = MAX_UPLOAD_FILE_SIZE; - - ApplicationArchiveContext applicationArchiveContext = getApplicationArchiveContext(mtar, fileName, maxFileUploadSize); - ApplicationArchiveReader reader = getApplicationArchiveReaderForAbsolutePath(); - Exception exception = Assertions.assertThrows(Exception.class, () -> reader.calculateApplicationDigest(applicationArchiveContext)); - assertEquals(expectedException, exception.getMessage()); - } - - private ApplicationArchiveContext getApplicationArchiveContext(String mtar, String fileName, long maxFileUploadSize) { - InputStream mtarInputStream = getClass().getResourceAsStream(mtar); - return new ApplicationArchiveContext(mtarInputStream, fileName, maxFileUploadSize); - } - - private ApplicationArchiveReader getApplicationArchiveReader() { - return new ApplicationArchiveReader(); - } - - private ApplicationArchiveReader getApplicationArchiveReaderForAbsolutePath() { - return new ApplicationArchiveReader() { - @Override - protected void validateEntry(ZipEntry entry) { - String path = entry.getName(); - if (!path.equals(FilenameUtils.normalize(path, true))) { - throw new IllegalArgumentException(MessageFormat.format(FileUtils.PATH_SHOULD_BE_NORMALIZED, path)); - } - if (FilenameUtils.getPrefixLength(path) != 0 || Paths.get(path) - .isAbsolute()) { - throw new IllegalArgumentException(MessageFormat.format(FileUtils.PATH_SHOULD_NOT_BE_ABSOLUTE, path)); - } - } - }; - } - -} diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationDigestCalculatorTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationDigestCalculatorTest.java new file mode 100644 index 0000000000..4193f6ad19 --- /dev/null +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationDigestCalculatorTest.java @@ -0,0 +1,157 @@ +package org.cloudfoundry.multiapps.controller.process.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; + +import java.io.InputStream; +import java.util.List; +import java.util.function.ObjIntConsumer; + +import org.apache.commons.io.input.BoundedInputStream; +import org.cloudfoundry.multiapps.common.SLException; +import org.cloudfoundry.multiapps.controller.persistence.services.FileContentConsumer; +import org.cloudfoundry.multiapps.controller.persistence.services.FileService; +import org.cloudfoundry.multiapps.mta.util.EntryToInflate; +import org.cloudfoundry.multiapps.mta.util.InflatorUtil; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +class ApplicationDigestCalculatorTest { + + private static final String SPACE_GUID = "123"; + private static final String APP_ARCHIVE_ID = "132"; + private static final String DB_DIRECTORY_MODULE_DIGEST = "71017C6429E2E1FA4ED2AD97ABF321A0"; + private static final String WEB_SERVER_MODULE_DIGEST = "4C64A36CDC073B5D07947005F630DACC"; + + @Mock + private FileService fileService; + @Mock + private ArchiveEntryExtractor archiveEntryExtractor; + + private ApplicationDigestCalculator applicationDigestCalculator; + + @BeforeEach + void setUp() throws Exception { + MockitoAnnotations.openMocks(this) + .close(); + doAnswer(answer -> { + FileContentConsumer fileContentConsumer = answer.getArgument(2); + fileContentConsumer.consume(getClass().getResourceAsStream("com.sap.mta.sample-1.2.1-beta.mtar")); + return null; + }).when(fileService) + .consumeFileContent(any(), any(), any()); + doAnswer(answer -> { + FileEntryProperties fileEntryProperties = answer.getArgument(0); + ArchiveEntryWithStreamPositions archiveEntryWithStreamPositions = answer.getArgument(1); + ObjIntConsumer consumer = answer.getArgument(2); + InputStream resourceAsStream = getClass().getResourceAsStream("com.sap.mta.sample-1.2.1-beta.mtar"); + resourceAsStream.skip(archiveEntryWithStreamPositions.getStartPosition()); + BoundedInputStream boundedInputStream = new BoundedInputStream.Builder().setInputStream(resourceAsStream) + .setCount(archiveEntryWithStreamPositions.getStartPosition()) + .setMaxCount(archiveEntryWithStreamPositions.getEndPosition()) + .get(); + InflatorUtil.inflate(new EntryToInflate("web/web-server.zip", fileEntryProperties.getMaxFileSizeInBytes(), boundedInputStream), + consumer); + return null; + }).when(archiveEntryExtractor) + .processFileEntryBytes(any(), any(), any()); + applicationDigestCalculator = new ApplicationDigestCalculator(fileService, new ApplicationArchiveIterator(), archiveEntryExtractor); + } + + @Test + void testDigestCalculationWhenModuleIsDirectory() { + ArchiveEntryWithStreamPositions directoryModuleEntry = buildDbModule(); + List archiveEntriesWithStreamPositions = List.of(directoryModuleEntry); + ApplicationArchiveContext applicationArchiveContext = new ApplicationArchiveContext("db/", + Integer.MAX_VALUE, + archiveEntriesWithStreamPositions, + SPACE_GUID, + APP_ARCHIVE_ID); + String appDigest = applicationDigestCalculator.calculateApplicationDigest(applicationArchiveContext); + assertEquals(DB_DIRECTORY_MODULE_DIGEST, appDigest); + } + + private ArchiveEntryWithStreamPositions buildDbModule() { + return ImmutableArchiveEntryWithStreamPositions.builder() + .name("db/") + .startPosition(0) + .endPosition(0) + .compressionMethod(ArchiveEntryWithStreamPositions.CompressionMethod.DEFLATED) + .isDirectory(true) + .build(); + } + + @Test + void testDigestCalculationWhenModuleIsDirectoryAndExceedsMaxSizeLimit() { + ArchiveEntryWithStreamPositions directoryModuleEntry = buildDbModule(); + List archiveEntriesWithStreamPositions = List.of(directoryModuleEntry); + ApplicationArchiveContext applicationArchiveContext = new ApplicationArchiveContext("db/", + 1, + archiveEntriesWithStreamPositions, + SPACE_GUID, + APP_ARCHIVE_ID); + Exception exception = assertThrows(Exception.class, + () -> applicationDigestCalculator.calculateApplicationDigest(applicationArchiveContext)); + assertEquals("The size \"201\" of mta file \"db/\" exceeds the configured max size limit \"1\"", exception.getMessage()); + } + + @Test + void testDigestCalculationWhenModuleIsZip() { + ArchiveEntryWithStreamPositions zipModuleEntry = buildWebServerModule(); + List archiveEntriesWithStreamPositions = List.of(zipModuleEntry); + ApplicationArchiveContext applicationArchiveContext = new ApplicationArchiveContext("web/web-server.zip", + Integer.MAX_VALUE, + archiveEntriesWithStreamPositions, + SPACE_GUID, + APP_ARCHIVE_ID); + String appDigest = applicationDigestCalculator.calculateApplicationDigest(applicationArchiveContext); + assertEquals(WEB_SERVER_MODULE_DIGEST, appDigest); + } + + private ArchiveEntryWithStreamPositions buildWebServerModule() { + return ImmutableArchiveEntryWithStreamPositions.builder() + .name("web/web-server.zip") + .startPosition(531) + .endPosition(678) + .compressionMethod(ArchiveEntryWithStreamPositions.CompressionMethod.DEFLATED) + .isDirectory(false) + .build(); + } + + @Test + void testDigestCalculationWhenModuleIsZipAndExceedsMaxSizeLimit() { + ArchiveEntryWithStreamPositions zipModuleEntry = buildWebServerModule(); + List archiveEntriesWithStreamPositions = List.of(zipModuleEntry); + ApplicationArchiveContext applicationArchiveContext = new ApplicationArchiveContext("web/web-server.zip", + 1, + archiveEntriesWithStreamPositions, + SPACE_GUID, + APP_ARCHIVE_ID); + Exception exception = assertThrows(Exception.class, + () -> applicationDigestCalculator.calculateApplicationDigest(applicationArchiveContext)); + assertEquals("The size \"203\" of mta file \"web/web-server.zip\" exceeds the configured max size limit \"1\"", + exception.getMessage()); + } + + @Test + void testDigestCalculationWhenExceptionIsThrown() { + ArchiveEntryWithStreamPositions zipModuleEntry = buildWebServerModule(); + List archiveEntriesWithStreamPositions = List.of(zipModuleEntry); + ApplicationArchiveContext applicationArchiveContext = new ApplicationArchiveContext("web/web-server.zip", + 1, + archiveEntriesWithStreamPositions, + SPACE_GUID, + APP_ARCHIVE_ID); + doThrow(new SLException("Cannot calculate digest")).when(archiveEntryExtractor) + .processFileEntryBytes(any(), any(), any()); + Exception exception = assertThrows(Exception.class, + () -> applicationDigestCalculator.calculateApplicationDigest(applicationArchiveContext)); + assertEquals("Cannot calculate digest", exception.getMessage()); + } + +} diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationZipBuilderTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationZipBuilderTest.java index dc991966d9..4b28c5c30c 100644 --- a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationZipBuilderTest.java +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/ApplicationZipBuilderTest.java @@ -2,6 +2,8 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import java.io.IOException; import java.io.InputStream; @@ -11,14 +13,21 @@ import java.text.MessageFormat; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +import org.apache.commons.io.input.BoundedInputStream; import org.cloudfoundry.multiapps.common.SLException; import org.cloudfoundry.multiapps.controller.core.util.FileUtils; +import org.cloudfoundry.multiapps.controller.persistence.services.FileContentConsumer; +import org.cloudfoundry.multiapps.controller.persistence.services.FileContentProcessor; +import org.cloudfoundry.multiapps.controller.persistence.services.FileContentToProcess; +import org.cloudfoundry.multiapps.controller.persistence.services.FileService; +import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -26,6 +35,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; import org.mockito.MockitoAnnotations; class ApplicationZipBuilderTest { @@ -34,6 +44,9 @@ class ApplicationZipBuilderTest { private static final String SAMPLE_FLAT_MTAR = "com.sap.mta.sample-1.2.1-beta-flat.mtar"; private static final long MAX_UPLOAD_FILE_SIZE = 1024 * 1024 * 1024L; // 1gb + @Mock + private FileService fileService; + private Path appPath = null; static Stream testCreateNewZip() { @@ -68,8 +81,9 @@ void tearDown() throws IOException { @MethodSource void testCreateNewZip(String mtar, String fileName) throws Exception { ApplicationArchiveContext applicationArchiveContext = getApplicationArchiveContext(mtar, fileName); - ApplicationArchiveReader reader = new ApplicationArchiveReader(); - ApplicationZipBuilder zipBuilder = new ApplicationZipBuilder(reader); + ApplicationZipBuilder zipBuilder = new ApplicationZipBuilder(fileService, + new ApplicationArchiveIterator(), + new ArchiveEntryExtractor(fileService)); appPath = zipBuilder.extractApplicationInNewArchive(applicationArchiveContext); assertTrue(Files.exists(appPath)); try (InputStream zipStream = Files.newInputStream(appPath)) { @@ -78,16 +92,68 @@ void testCreateNewZip(String mtar, String fileName) throws Exception { } } - private ApplicationArchiveContext getApplicationArchiveContext(String mtar, String fileName) { - return new ApplicationArchiveContext(getClass().getResourceAsStream(mtar), fileName, MAX_UPLOAD_FILE_SIZE); + private ApplicationArchiveContext getApplicationArchiveContext(String mtar, String fileName) throws FileStorageException { + mockProcessingOfFileContent(mtar); + mockConsumptionOfFileContent(mtar); + mockConsumptionOfFileContentWithOffset(mtar); + ArchiveEntryStreamWithStreamPositionsDeterminer archiveEntryStreamWithStreamPositionsDeterminer = new ArchiveEntryStreamWithStreamPositionsDeterminer(fileService); + List archiveEntriesWithStreamPositions = archiveEntryStreamWithStreamPositionsDeterminer.determineArchiveEntries("123", + "123"); + return new ApplicationArchiveContext(fileName, MAX_UPLOAD_FILE_SIZE, archiveEntriesWithStreamPositions, "123", "123"); + } + + private void mockProcessingOfFileContent(String mtar) throws FileStorageException { + doAnswer(answer -> { + try (InputStream inputStream = getClass().getResourceAsStream(mtar)) { + FileContentProcessor fileContentProcessor = answer.getArgument(2); + return fileContentProcessor.process(inputStream); + } catch (IOException e) { + throw new SLException(e, e.getMessage()); + } + }).when(fileService) + .processFileContent(any(), any(), any()); + } + + private void mockConsumptionOfFileContent(String mtar) throws FileStorageException { + doAnswer(answer -> { + try (InputStream inputStream = getClass().getResourceAsStream(mtar)) { + FileContentConsumer fileContentConsumer = answer.getArgument(2); + fileContentConsumer.consume(inputStream); + return null; + } catch (IOException e) { + throw new SLException(e, e.getMessage()); + } + }).when(fileService) + .consumeFileContent(any(), any(), any()); + } + + private void mockConsumptionOfFileContentWithOffset(String mtar) throws FileStorageException { + doAnswer(answer -> { + try (InputStream inputStream = getClass().getResourceAsStream(mtar)) { + FileContentToProcess fileContentToProcess = answer.getArgument(0); + inputStream.skip(fileContentToProcess.getStartOffset()); + BoundedInputStream boundedInputStream = new BoundedInputStream.Builder().setInputStream(inputStream) + .setCount(fileContentToProcess.getStartOffset()) + .setMaxCount(fileContentToProcess.getEndOffset()) + .get(); + FileContentConsumer fileContentConsumer = answer.getArgument(1); + fileContentConsumer.consume(boundedInputStream); + return null; + } catch (IOException e) { + throw new SLException(e, e.getMessage()); + } + }).when(fileService) + .consumeFileContentWithOffset(any(), any()); } @ParameterizedTest @MethodSource - void testCreateZipOnlyWithMissingResources(String mtar, String fileName, Set alreadyUploadedFiles) throws IOException { + void testCreateZipOnlyWithMissingResources(String mtar, String fileName, Set alreadyUploadedFiles) + throws IOException, FileStorageException { ApplicationArchiveContext applicationArchiveContext = getApplicationArchiveContext(mtar, fileName); - ApplicationArchiveReader reader = new ApplicationArchiveReader(); - ApplicationZipBuilder zipBuilder = new ApplicationZipBuilder(reader); + ApplicationZipBuilder zipBuilder = new ApplicationZipBuilder(fileService, + new ApplicationArchiveIterator(), + new ArchiveEntryExtractor(fileService)); appPath = zipBuilder.extractApplicationInNewArchive(applicationArchiveContext); assertTrue(Files.exists(appPath)); Set relativizedFilePaths = relativizeUploadedFilesPaths(fileName, alreadyUploadedFiles); @@ -117,16 +183,16 @@ private Set getZipEntriesName(InputStream inputStream) throws IOExceptio } @Test - void testFailToCreateZip() { + void testFailToCreateZip() throws FileStorageException { String fileName = "db/"; - ApplicationArchiveReader reader = new ApplicationArchiveReader(); - ApplicationZipBuilder zipBuilder = new ApplicationZipBuilder(reader) { + ApplicationZipBuilder zipBuilder = new ApplicationZipBuilder(fileService, + new ApplicationArchiveIterator(), + new ArchiveEntryExtractor(fileService)) { @Override protected void copy(InputStream input, OutputStream output, ApplicationArchiveContext applicationArchiveContext) throws IOException { throw new IOException(); } - }; ApplicationArchiveContext applicationArchiveContext = getApplicationArchiveContext(SAMPLE_MTAR, fileName); Assertions.assertThrows(SLException.class, () -> appPath = zipBuilder.extractApplicationInNewArchive(applicationArchiveContext)); diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/ArchiveEntryExtractorTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/ArchiveEntryExtractorTest.java new file mode 100644 index 0000000000..c689963ee1 --- /dev/null +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/ArchiveEntryExtractorTest.java @@ -0,0 +1,154 @@ +package org.cloudfoundry.multiapps.controller.process.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.UUID; +import java.util.stream.Stream; + +import org.apache.commons.io.input.BoundedInputStream; +import org.cloudfoundry.multiapps.common.ContentException; +import org.cloudfoundry.multiapps.controller.persistence.services.FileContentConsumer; +import org.cloudfoundry.multiapps.controller.persistence.services.FileContentProcessor; +import org.cloudfoundry.multiapps.controller.persistence.services.FileService; +import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +class ArchiveEntryExtractorTest { + + private static final ArchiveEntryWithStreamPositions DEFLATED_DEPLOYMENT_DESCRIPTOR_ENTRY = ImmutableArchiveEntryWithStreamPositions.builder() + .name("META-INF/mtad.yaml") + .startPosition(271) + .endPosition(315) + .compressionMethod(ArchiveEntryWithStreamPositions.CompressionMethod.DEFLATED) + .isDirectory(false) + .build(); + + private static final ArchiveEntryWithStreamPositions STORED_DEPLOYMENT_DESCRIPTOR_ENTRY = ImmutableArchiveEntryWithStreamPositions.builder() + .name("META-INF/mtad.yaml") + .startPosition(271) + .endPosition(320) + .compressionMethod(ArchiveEntryWithStreamPositions.CompressionMethod.STORED) + .isDirectory(false) + .build(); + + @Mock + private FileService fileService; + + private ArchiveEntryExtractor archiveEntryExtractor; + + @BeforeEach + void setUp() throws Exception { + MockitoAnnotations.openMocks(this) + .close(); + archiveEntryExtractor = new ArchiveEntryExtractor(fileService); + } + + static Stream readFullDeploymentDescriptorFile() { + return Stream.of(Arguments.of("stored-mta.mtar", STORED_DEPLOYMENT_DESCRIPTOR_ENTRY), + Arguments.of("deflated-mta.mtar", DEFLATED_DEPLOYMENT_DESCRIPTOR_ENTRY)); + } + + @ParameterizedTest + @MethodSource + void readFullDeploymentDescriptorFile(String mtarFileName, ArchiveEntryWithStreamPositions deploymentDescriptorEntry) + throws FileStorageException, IOException { + prepareFileService(getClass().getResourceAsStream(mtarFileName), deploymentDescriptorEntry.getStartPosition(), + deploymentDescriptorEntry.getEndPosition()); + byte[] bytesRead = archiveEntryExtractor.extractEntryBytes(buildFileEntryProperty(Integer.MAX_VALUE), deploymentDescriptorEntry); + assertDeploymentDescriptorAreEqual(new String(bytesRead)); + } + + private FileEntryProperties buildFileEntryProperty(long maxFileSizeInBytes) { + return ImmutableFileEntryProperties.builder() + .guid(UUID.randomUUID() + .toString()) + .name("archive-entry") + .maxFileSizeInBytes(maxFileSizeInBytes) + .spaceGuid(UUID.randomUUID() + .toString()) + .build(); + } + + private void assertDeploymentDescriptorAreEqual(String actualDeploymentDescriptor) throws IOException { + String expectedDeploymentDescriptor = new String(getClass().getResourceAsStream("expected-mtad.yaml") + .readAllBytes()); + assertEquals(expectedDeploymentDescriptor, actualDeploymentDescriptor); + } + + static Stream processFileEntryBytes() { + return Stream.of(Arguments.of("stored-mta.mtar", STORED_DEPLOYMENT_DESCRIPTOR_ENTRY), + Arguments.of("deflated-mta.mtar", DEFLATED_DEPLOYMENT_DESCRIPTOR_ENTRY)); + } + + @ParameterizedTest + @MethodSource + void processFileEntryBytes(String mtarFileName, ArchiveEntryWithStreamPositions deploymentDescriptorEntry) + throws FileStorageException, IOException { + prepareFileService(getClass().getResourceAsStream(mtarFileName), deploymentDescriptorEntry.getStartPosition(), + deploymentDescriptorEntry.getEndPosition()); + StringBuilder actualDeploymentDescriptorContent = new StringBuilder(); + archiveEntryExtractor.processFileEntryBytes(buildFileEntryProperty(Integer.MAX_VALUE), deploymentDescriptorEntry, + (byteBuffer, + bytesRead) -> actualDeploymentDescriptorContent.append(new String(Arrays.copyOfRange(byteBuffer, + 0, + bytesRead)))); + assertDeploymentDescriptorAreEqual(actualDeploymentDescriptorContent.toString()); + } + + static Stream processFileEntryContentWithExceedingSizeEntry() { + return Stream.of(Arguments.of("stored-mta.mtar", STORED_DEPLOYMENT_DESCRIPTOR_ENTRY), + Arguments.of("deflated-mta.mtar", DEFLATED_DEPLOYMENT_DESCRIPTOR_ENTRY)); + } + + @ParameterizedTest + @MethodSource + void processFileEntryContentWithExceedingSizeEntry(String mtarFileName, ArchiveEntryWithStreamPositions deploymentDescriptorEntry) + throws FileStorageException { + prepareFileService(getClass().getResourceAsStream(mtarFileName), deploymentDescriptorEntry.getStartPosition(), + deploymentDescriptorEntry.getEndPosition()); + Exception exception = assertThrows(ContentException.class, + () -> archiveEntryExtractor.processFileEntryBytes(buildFileEntryProperty(2), + deploymentDescriptorEntry, + (byteBuffer, bytesRead) -> { + })); + assertEquals("The size \"49\" of mta file \"archive-entry\" exceeds the configured max size limit \"2\"", exception.getMessage()); + } + + private void prepareFileService(InputStream fileEntryInputStream, long startPosition, long endPosition) throws FileStorageException { + doAnswer(answer -> { + FileContentProcessor fileContentProcessor = answer.getArgument(1); + fileEntryInputStream.skip(startPosition); + BoundedInputStream boundedInputStream = new BoundedInputStream.Builder().setInputStream(fileEntryInputStream) + .setCount(startPosition) + .setMaxCount(endPosition) + .setPropagateClose(true) + .get(); + return fileContentProcessor.process(boundedInputStream); + }).when(fileService) + .processFileContentWithOffset(any(), any()); + doAnswer(answer -> { + FileContentConsumer fileContentConsumer = answer.getArgument(1); + fileEntryInputStream.skip(startPosition); + BoundedInputStream boundedInputStream = new BoundedInputStream.Builder().setInputStream(fileEntryInputStream) + .setCount(startPosition) + .setMaxCount(endPosition) + .setPropagateClose(true) + .get(); + fileContentConsumer.consume(boundedInputStream); + return null; + }).when(fileService) + .consumeFileContentWithOffset(any(), any()); + } + +} diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/ServiceBindingParametersGetterTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/ServiceBindingParametersGetterTest.java index f6491c715a..0ae79eb6ba 100644 --- a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/ServiceBindingParametersGetterTest.java +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/ServiceBindingParametersGetterTest.java @@ -10,11 +10,12 @@ import static org.mockito.Mockito.when; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.UUID; import java.util.stream.Stream; -import com.sap.cloudfoundry.client.facade.domain.ServiceCredentialBindingOperation; +import org.cloudfoundry.multiapps.common.util.JsonUtil; import org.cloudfoundry.multiapps.controller.client.lib.domain.CloudApplicationExtended; import org.cloudfoundry.multiapps.controller.client.lib.domain.CloudServiceInstanceExtended; import org.cloudfoundry.multiapps.controller.client.lib.domain.ImmutableBindingDetails; @@ -22,7 +23,6 @@ import org.cloudfoundry.multiapps.controller.client.lib.domain.ImmutableCloudServiceInstanceExtended; import org.cloudfoundry.multiapps.controller.core.helpers.MtaArchiveElements; import org.cloudfoundry.multiapps.controller.persistence.services.FileService; -import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; import org.cloudfoundry.multiapps.controller.process.steps.ProcessContext; import org.cloudfoundry.multiapps.controller.process.variables.Variables; import org.junit.jupiter.api.BeforeEach; @@ -41,6 +41,7 @@ import com.sap.cloudfoundry.client.facade.domain.ImmutableCloudMetadata; import com.sap.cloudfoundry.client.facade.domain.ImmutableCloudServiceBinding; import com.sap.cloudfoundry.client.facade.domain.ImmutableServiceCredentialBindingOperation; +import com.sap.cloudfoundry.client.facade.domain.ServiceCredentialBindingOperation; class ServiceBindingParametersGetterTest { @@ -49,24 +50,34 @@ class ServiceBindingParametersGetterTest { private static final String APP_ARCHIVE_ID = "test_archive_id"; private static final String SERVICE_BINDING_PARAMETERS_FILENAME = "test_binding_parameters.json"; private static final UUID RANDOM_GUID = UUID.randomUUID(); + private static final ArchiveEntryWithStreamPositions ARCHIVE_ENTRY_WITH_STREAM_POSITIONS = ImmutableArchiveEntryWithStreamPositions.builder() + .name(SERVICE_BINDING_PARAMETERS_FILENAME) + .startPosition(37) + .endPosition(5012) + .compressionMethod(ArchiveEntryWithStreamPositions.CompressionMethod.DEFLATED) + .isDirectory(false) + .build(); + private static final String TEST_SPACE_GUID = "test_space_guid"; @Mock private ProcessContext context; @Mock private StepLogger stepLogger; @Mock - private FileService fileService; + private ArchiveEntryExtractor archiveEntryExtractor; @Mock private MtaArchiveElements mtaArchiveElements; @Mock private CloudControllerClient client; + @Mock + private FileService fileService; private ServiceBindingParametersGetter serviceBindingParametersGetter; @BeforeEach void setUp() { MockitoAnnotations.openMocks(this); - serviceBindingParametersGetter = new ServiceBindingParametersGetter(context, fileService, 0); + serviceBindingParametersGetter = new ServiceBindingParametersGetter(context, archiveEntryExtractor, 0, fileService); } static Stream testGetServiceBindingParametersFromMta() { @@ -89,8 +100,7 @@ static Stream testGetServiceBindingParametersFromMta() { @ParameterizedTest @MethodSource void testGetServiceBindingParametersFromMta(Map descriptorParameters, Map filedProvidedParameters, - Map expectedParameters) - throws FileStorageException { + Map expectedParameters) { CloudApplicationExtended application = buildApplication(descriptorParameters); CloudServiceInstanceExtended serviceInstance = buildServiceInstance(); prepareMtaArchiveElements(filedProvidedParameters); @@ -104,7 +114,7 @@ void testGetServiceBindingParametersFromMta(Map descriptorParame } @Test - void testGetServiceBindingOfMissingService() throws FileStorageException { + void testGetServiceBindingOfMissingService() { CloudApplicationExtended application = buildApplication(null); when(context.getVariable(Variables.SERVICES_TO_BIND)).thenReturn(Collections.emptyList()); @@ -202,11 +212,15 @@ private void prepareContext(CloudServiceInstanceExtended serviceInstance) { when(context.getVariable(Variables.SERVICES_TO_BIND)).thenReturn(Collections.singletonList(serviceInstance)); when(context.getRequiredVariable(Variables.APP_ARCHIVE_ID)).thenReturn(APP_ARCHIVE_ID); when(context.getVariable(Variables.MTA_ARCHIVE_ELEMENTS)).thenReturn(mtaArchiveElements); + when(context.getVariable(Variables.ARCHIVE_ENTRIES_POSITIONS)).thenReturn(List.of(ARCHIVE_ENTRY_WITH_STREAM_POSITIONS)); + when(context.getVariable(Variables.SPACE_GUID)).thenReturn(TEST_SPACE_GUID); + when(context.getRequiredVariable(Variables.SPACE_GUID)).thenReturn(TEST_SPACE_GUID); when(context.getControllerClient()).thenReturn(client); } - private void prepareFileService(Map filedProvidedParameters) throws FileStorageException { - when(fileService.processFileContent(any(), any(), any())).thenReturn(filedProvidedParameters); + private void prepareFileService(Map fileProvidedParameters) { + String fileProvidedParametersJson = JsonUtil.toJson(fileProvidedParameters); + when(archiveEntryExtractor.extractEntryBytes(any(), any())).thenReturn(fileProvidedParametersJson.getBytes()); } private void prepareClient(Map bindingParameters, boolean serviceBindingExist) { @@ -220,9 +234,9 @@ private void prepareClient(Map bindingParameters, boolean servic .guid(RANDOM_GUID) .build()) .serviceBindingOperation(ImmutableServiceCredentialBindingOperation.builder() - .type(ServiceCredentialBindingOperation.Type.CREATE) - .state(ServiceCredentialBindingOperation.State.SUCCEEDED) - .build()) + .type(ServiceCredentialBindingOperation.Type.CREATE) + .state(ServiceCredentialBindingOperation.State.SUCCEEDED) + .build()) .build(); when(client.getServiceBindingForApplication(RANDOM_GUID, RANDOM_GUID)).thenReturn(serviceBinding); return; diff --git a/multiapps-controller-process/src/test/resources/org/cloudfoundry/multiapps/controller/process/steps/test-manifest-1.MF b/multiapps-controller-process/src/test/resources/org/cloudfoundry/multiapps/controller/process/steps/test-manifest-1.MF new file mode 100644 index 0000000000..d94085803f --- /dev/null +++ b/multiapps-controller-process/src/test/resources/org/cloudfoundry/multiapps/controller/process/steps/test-manifest-1.MF @@ -0,0 +1,18 @@ +Manifest-Version: 1.0 + +Name: test-module1 +MTA-Module: app1 + +Name: test-module2 +MTA-Module: app2 + +Name: test-resource1 +MTA-Resource: test-resource1 + +Name: test-dependency1 +MTA-Requires: test-module1/test-resource1 + +Name: test-dependency1 +MTA-Requires: test-module2/test-resource2 + +Name: META-INF/mtad.yaml diff --git a/multiapps-controller-process/src/test/resources/org/cloudfoundry/multiapps/controller/process/steps/test-mtad-1.yaml b/multiapps-controller-process/src/test/resources/org/cloudfoundry/multiapps/controller/process/steps/test-mtad-1.yaml new file mode 100644 index 0000000000..ad3f8a18e4 --- /dev/null +++ b/multiapps-controller-process/src/test/resources/org/cloudfoundry/multiapps/controller/process/steps/test-mtad-1.yaml @@ -0,0 +1,18 @@ +_schema-version: "3.1.0" +ID: mta +version: 6.0.0 + +modules: + - name: test-module1 + type: application + requires: + - name: test-resource1 + + - name: test-module2 + type: application + requires: + - name: test-resource1 + +resources: + - name: test-resource1 + type: com.sap.xs.uaa \ No newline at end of file diff --git a/multiapps-controller-process/src/test/resources/org/cloudfoundry/multiapps/controller/process/util/deflated-mta.mtar b/multiapps-controller-process/src/test/resources/org/cloudfoundry/multiapps/controller/process/util/deflated-mta.mtar new file mode 100644 index 0000000000..2d26210424 Binary files /dev/null and b/multiapps-controller-process/src/test/resources/org/cloudfoundry/multiapps/controller/process/util/deflated-mta.mtar differ diff --git a/multiapps-controller-process/src/test/resources/org/cloudfoundry/multiapps/controller/process/util/expected-mtad.yaml b/multiapps-controller-process/src/test/resources/org/cloudfoundry/multiapps/controller/process/util/expected-mtad.yaml new file mode 100644 index 0000000000..453b653fc5 --- /dev/null +++ b/multiapps-controller-process/src/test/resources/org/cloudfoundry/multiapps/controller/process/util/expected-mtad.yaml @@ -0,0 +1,3 @@ +_schema-version: "3.1.0" +ID: test +version: 6.0.0 diff --git a/multiapps-controller-process/src/test/resources/org/cloudfoundry/multiapps/controller/process/util/stored-mta.mtar b/multiapps-controller-process/src/test/resources/org/cloudfoundry/multiapps/controller/process/util/stored-mta.mtar new file mode 100644 index 0000000000..87028eb5b5 Binary files /dev/null and b/multiapps-controller-process/src/test/resources/org/cloudfoundry/multiapps/controller/process/util/stored-mta.mtar differ diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/security/AuthenticationLoaderFilter.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/security/AuthenticationLoaderFilter.java index 140cf4193d..404bdf26b5 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/security/AuthenticationLoaderFilter.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/security/AuthenticationLoaderFilter.java @@ -2,13 +2,6 @@ import java.io.IOException; -import jakarta.inject.Inject; -import jakarta.inject.Named; -import jakarta.servlet.FilterChain; -import jakarta.servlet.ServletException; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; - import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.util.SSLUtil; import org.cloudfoundry.multiapps.controller.core.util.SecurityUtil; @@ -27,6 +20,13 @@ import com.sap.cloudfoundry.client.facade.oauth2.OAuth2AccessTokenWithAdditionalInfo; +import jakarta.inject.Inject; +import jakarta.inject.Named; +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + @Named public class AuthenticationLoaderFilter extends OncePerRequestFilter { diff --git a/pom.xml b/pom.xml index c9c584465c..216b79c7ad 100644 --- a/pom.xml +++ b/pom.xml @@ -58,6 +58,7 @@ 2.0.1 3.0.0 1.78 + 1.27.1 multiapps-controller-client @@ -284,6 +285,11 @@ + + org.apache.commons + commons-compress + ${apache.compress.version} + org.immutables @@ -338,6 +344,12 @@ javax.xml.bind jaxb-api 2.3.1 + + + javax.activation + javax.activation-api + +