Skip to content

Commit 403c257

Browse files
IvanBorislavovDimitrovIvanBorislavovDimitrov
committed
Fix comments Kris
1 parent f2cb578 commit 403c257

File tree

9 files changed

+511
-559
lines changed

9 files changed

+511
-559
lines changed

‎multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/util/ApplicationConfiguration.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ public class ApplicationConfiguration {
152152
public static final String DEFAULT_GLOBAL_AUDITOR_ORIGIN = "uaa";
153153
public static final int DEFAULT_SPRING_SCHEDULER_TASK_EXECUTOR_THREADS = 3;
154154
public static final int DEFAULT_FILES_ASYNC_UPLOAD_EXECUTOR_MAX_THREADS = 6;
155-
public static final int DEFAULT_DEPLOY_FROM_URL_EXECUTOR_MAX_THREADS = 50;
155+
public static final int DEFAULT_DEPLOY_FROM_URL_EXECUTOR_MAX_THREADS = 30;
156156
public static final boolean DEFAULT_ENABLE_ON_START_FILES_WITHOUT_CONTENT_CLEANER = false;
157157
public static final int DEFAULT_THREADS_FOR_FILE_UPLOAD_TO_CONTROLLER = 6;
158158
public static final int DEFAULT_THREADS_FOR_FILE_STORAGE_UPLOAD = 7;

multiapps-controller-persistence/src/main/resources/org/cloudfoundry/multiapps/controller/persistence/db/changelog/db-changelog-1.199.0-persistence.xml renamed to multiapps-controller-persistence/src/main/resources/org/cloudfoundry/multiapps/controller/persistence/db/changelog/db-changelog-1.200.0-persistence.xml

File renamed without changes.

multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/upload/CountingInputStream.java renamed to multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/stream/CountingInputStream.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package org.cloudfoundry.multiapps.controller.web.upload;
1+
package org.cloudfoundry.multiapps.controller.process.stream;
22

33
import java.io.InputStream;
44
import java.util.concurrent.atomic.AtomicLong;

‎multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/FilesApiServiceImpl.java‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -118,15 +118,17 @@ public ResponseEntity<Void> startUploadFromUrl(String spaceGuid, String namespac
118118
LOGGER.trace(Messages.RECEIVED_UPLOAD_FROM_URL_REQUEST, urlWithoutUserInfo);
119119
filesApiServiceAuditLog.logStartUploadFromUrl(SecurityContextUtil.getUsername(), spaceGuid, decodedUrl);
120120
var existingJob = getExistingJob(spaceGuid, namespace, urlWithoutUserInfo);
121-
if (existingJob != null && hasJobStuck(existingJob)) {
121+
if (existingJob == null) {
122+
return triggerUploadFromUrl(spaceGuid, namespace, urlWithoutUserInfo, decodedUrl, fileUrl.getUserCredentials());
123+
}
124+
if (hasJobStuck(existingJob)) {
122125
deleteAsyncJobEntry(existingJob);
123-
} else if (existingJob != null) {
124-
LOGGER.info(Messages.ASYNC_UPLOAD_JOB_EXISTS, urlWithoutUserInfo, existingJob);
125-
return ResponseEntity.status(HttpStatus.SEE_OTHER)
126-
.header(HttpHeaders.LOCATION, getLocationHeader(spaceGuid, existingJob.getId()))
127-
.build();
126+
return triggerUploadFromUrl(spaceGuid, namespace, urlWithoutUserInfo, decodedUrl, fileUrl.getUserCredentials());
128127
}
129-
return triggerUploadFromUrl(spaceGuid, namespace, urlWithoutUserInfo, decodedUrl, fileUrl.getUserCredentials());
128+
LOGGER.info(Messages.ASYNC_UPLOAD_JOB_EXISTS, urlWithoutUserInfo, existingJob);
129+
return ResponseEntity.status(HttpStatus.SEE_OTHER)
130+
.header(HttpHeaders.LOCATION, getLocationHeader(spaceGuid, existingJob.getId()))
131+
.build();
130132
}
131133

132134
private boolean hasJobStuck(AsyncUploadJobEntry existingJob) {

‎multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/upload/AsyncUploadJobExecutor.java‎

Lines changed: 11 additions & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,20 @@
11
package org.cloudfoundry.multiapps.controller.web.upload;
22

33
import java.io.BufferedInputStream;
4-
import java.io.IOException;
54
import java.io.InputStream;
65
import java.math.BigInteger;
76
import java.net.URI;
8-
import java.net.URLDecoder;
9-
import java.net.http.HttpClient;
10-
import java.net.http.HttpRequest;
11-
import java.net.http.HttpResponse;
12-
import java.net.http.HttpResponse.BodyHandlers;
13-
import java.nio.charset.StandardCharsets;
147
import java.text.MessageFormat;
158
import java.time.Duration;
169
import java.time.LocalDateTime;
1710
import java.time.temporal.ChronoUnit;
18-
import java.util.Arrays;
19-
import java.util.Base64;
2011
import java.util.UUID;
2112
import java.util.concurrent.ExecutorService;
2213
import java.util.concurrent.atomic.AtomicLong;
2314
import java.util.concurrent.locks.Lock;
2415
import java.util.concurrent.locks.ReentrantLock;
2516
import jakarta.inject.Inject;
2617
import jakarta.inject.Named;
27-
import org.apache.commons.io.IOUtils;
2818
import org.cloudfoundry.multiapps.common.SLException;
2919
import org.cloudfoundry.multiapps.common.util.MiscUtil;
3020
import org.cloudfoundry.multiapps.controller.api.model.UserCredentials;
@@ -33,33 +23,28 @@
3323
import org.cloudfoundry.multiapps.controller.core.helpers.DescriptorParserFacadeFactory;
3424
import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration;
3525
import org.cloudfoundry.multiapps.controller.core.util.FileUtils;
36-
import org.cloudfoundry.multiapps.controller.core.util.UriUtil;
3726
import org.cloudfoundry.multiapps.controller.persistence.model.AsyncUploadJobEntry;
3827
import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry;
3928
import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableAsyncUploadJobEntry;
4029
import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry;
4130
import org.cloudfoundry.multiapps.controller.persistence.services.AsyncUploadJobService;
4231
import org.cloudfoundry.multiapps.controller.persistence.services.FileService;
43-
import org.cloudfoundry.multiapps.controller.web.Constants;
32+
import org.cloudfoundry.multiapps.controller.process.stream.CountingInputStream;
4433
import org.cloudfoundry.multiapps.controller.web.Messages;
34+
import org.cloudfoundry.multiapps.controller.web.upload.client.DeployFromUrlRemoteClient;
35+
import org.cloudfoundry.multiapps.controller.web.upload.client.FileFromUrlData;
4536
import org.cloudfoundry.multiapps.controller.web.util.SecurityContextUtil;
4637
import org.cloudfoundry.multiapps.mta.handlers.ArchiveHandler;
4738
import org.cloudfoundry.multiapps.mta.handlers.DescriptorParserFacade;
4839
import org.cloudfoundry.multiapps.mta.model.DeploymentDescriptor;
4940
import org.slf4j.Logger;
5041
import org.slf4j.LoggerFactory;
51-
import org.springframework.http.HttpHeaders;
52-
import org.springframework.http.HttpStatus;
5342

5443
@Named
5544
public class AsyncUploadJobExecutor {
5645

5746
private static final int INPUT_STREAM_BUFFER_SIZE = 16 * 1024;
5847

59-
private static final Duration HTTP_CONNECT_TIMEOUT = Duration.ofMinutes(10);
60-
private static final String USERNAME_PASSWORD_URL_FORMAT = "{0}:{1}";
61-
private static final int ERROR_RESPONSE_BODY_MAX_LENGTH = 4 * 1024;
62-
6348
private static final long WAIT_TIME_BETWEEN_ASYNC_JOB_UPDATES_IN_MILLIS = Duration.ofSeconds(3)
6449
.toMillis();
6550

@@ -72,19 +57,20 @@ public class AsyncUploadJobExecutor {
7257
private final AsyncUploadJobService asyncUploadJobService;
7358
private final FileService fileService;
7459
private final DescriptorParserFacadeFactory descriptorParserFactory;
75-
private final HttpClient httpClient;
60+
private final DeployFromUrlRemoteClient deployFromUrlRemoteClient;
7661

7762
@Inject
7863
public AsyncUploadJobExecutor(ExecutorService asyncFileUploadExecutor, ExecutorService deployFromUrlExecutor,
7964
ApplicationConfiguration applicationConfiguration, AsyncUploadJobService asyncUploadJobService,
80-
FileService fileService, DescriptorParserFacadeFactory descriptorParserFactory) {
65+
FileService fileService, DescriptorParserFacadeFactory descriptorParserFactory,
66+
DeployFromUrlRemoteClient deployFromUrlRemoteClient) {
8167
this.asyncFileUploadExecutor = asyncFileUploadExecutor;
8268
this.deployFromUrlExecutor = deployFromUrlExecutor;
8369
this.applicationConfiguration = applicationConfiguration;
8470
this.asyncUploadJobService = asyncUploadJobService;
8571
this.fileService = fileService;
8672
this.descriptorParserFactory = descriptorParserFactory;
87-
httpClient = buildHttpClient();
73+
this.deployFromUrlRemoteClient = deployFromUrlRemoteClient;
8874
}
8975

9076
public AsyncUploadJobEntry executeUploadFromUrl(String spaceGuid, String namespace, String urlWithoutUserInfo, String decodedUrl,
@@ -184,78 +170,28 @@ private void startSyncUploadFromUrlUpload(UploadFromUrlContext uploadFromUrlCont
184170
}
185171

186172
private FileEntry doUploadMtarFromUrl(UploadFromUrlContext uploadFromUrlContext, Lock lock) throws Exception {
187-
if (!UriUtil.isUrlSecure(uploadFromUrlContext.getFileUrl())) {
188-
throw new SLException(Messages.MTAR_ENDPOINT_NOT_SECURE);
189-
}
190-
UriUtil.validateUrl(uploadFromUrlContext.getFileUrl());
191-
192-
HttpResponse<InputStream> response = callRemoteEndpointWithRetry(uploadFromUrlContext.getFileUrl(),
193-
uploadFromUrlContext.getUserCredentials());
194-
long fileSize = response.headers()
195-
.firstValueAsLong(Constants.CONTENT_LENGTH)
196-
.orElseThrow(() -> new SLException(Messages.FILE_URL_RESPONSE_DID_NOT_RETURN_CONTENT_LENGTH));
197-
198-
long maxUploadSize = applicationConfiguration.getMaxUploadSize();
199-
if (fileSize > maxUploadSize) {
200-
throw new SLException(MessageFormat.format(Messages.MAX_UPLOAD_SIZE_EXCEEDED, maxUploadSize));
201-
}
202-
173+
FileFromUrlData fileFromUrlData = deployFromUrlRemoteClient.downloadFileFromUrl(uploadFromUrlContext);
203174
String fileName = extractFileName(uploadFromUrlContext.getFileUrl());
204175
FileUtils.validateFileHasExtension(fileName);
205176
resetCounterOnRetry(uploadFromUrlContext.getJobEntry()
206177
.getId(), lock);
207178
// Normal stream returned from the http response always returns 0 when InputStream::available() is executed which seems to break
208179
// JClods library: https://issues.apache.org/jira/browse/JCLOUDS-1623
209-
try (CountingInputStream source = new CountingInputStream(response.body(), uploadFromUrlContext.getCounterRef());
180+
try (CountingInputStream source = new CountingInputStream(fileFromUrlData.fileInputStream(), uploadFromUrlContext.getCounterRef());
210181
BufferedInputStream bufferedContent = new BufferedInputStream(source, INPUT_STREAM_BUFFER_SIZE)) {
211-
LOGGER.debug(Messages.UPLOADING_MTAR_STREAM_FROM_REMOTE_ENDPOINT, response.uri());
182+
LOGGER.debug(Messages.UPLOADING_MTAR_STREAM_FROM_REMOTE_ENDPOINT, fileFromUrlData.uri());
212183
return fileService.addFile(ImmutableFileEntry.builder()
213184
.space(uploadFromUrlContext.getJobEntry()
214185
.getSpaceGuid())
215186
.namespace(uploadFromUrlContext.getJobEntry()
216187
.getNamespace())
217188
.name(fileName)
218-
.size(BigInteger.valueOf(fileSize))
189+
.size(BigInteger.valueOf(fileFromUrlData.fileSize()))
219190
.build(), bufferedContent);
220191
}
221192
}
222193

223-
public HttpResponse<InputStream> callRemoteEndpointWithRetry(String decodedUrl, UserCredentials userCredentials)
224-
throws Exception {
225-
return resilientOperationExecutor.execute((CheckedSupplier<HttpResponse<InputStream>>) () -> {
226-
var request = buildFetchFileRequest(decodedUrl, userCredentials);
227-
LOGGER.debug(Messages.CALLING_REMOTE_MTAR_ENDPOINT, getMaskedUri(urlDecodeUrl(decodedUrl)));
228-
var response = httpClient.send(request, BodyHandlers.ofInputStream());
229-
if (response.statusCode() / 100 != 2) {
230-
String error = readErrorBodyFromResponse(response);
231-
LOGGER.error(error);
232-
if (response.statusCode() == HttpStatus.UNAUTHORIZED.value()) {
233-
String errorMessage = MessageFormat.format(Messages.DEPLOY_FROM_URL_WRONG_CREDENTIALS,
234-
UriUtil.stripUserInfo(decodedUrl));
235-
throw new SLException(errorMessage);
236-
}
237-
throw new SLException(MessageFormat.format(Messages.ERROR_FROM_REMOTE_MTAR_ENDPOINT, getMaskedUri(urlDecodeUrl(decodedUrl)),
238-
response.statusCode(), error));
239-
}
240-
return response;
241-
});
242-
}
243-
244-
private String getMaskedUri(String url) {
245-
if (url.contains("@")) {
246-
return url.substring(url.lastIndexOf("@"))
247-
.replace("@", "...");
248-
} else {
249-
return url;
250-
}
251-
}
252-
253-
private String urlDecodeUrl(String url) {
254-
return URLDecoder.decode(url, StandardCharsets.UTF_8);
255-
}
256-
257194
private void resetCounterOnRetry(String jobGuid, Lock lock) {
258-
259195
try {
260196
lock.lock();
261197
AsyncUploadJobEntry asyncUploadJobEntry = asyncUploadJobService.createQuery()
@@ -267,47 +203,6 @@ private void resetCounterOnRetry(String jobGuid, Lock lock) {
267203
} finally {
268204
lock.unlock();
269205
}
270-
271-
}
272-
273-
protected HttpClient buildHttpClient() {
274-
return HttpClient.newBuilder()
275-
.version(HttpClient.Version.HTTP_2)
276-
.connectTimeout(HTTP_CONNECT_TIMEOUT)
277-
.followRedirects(HttpClient.Redirect.NORMAL)
278-
.build();
279-
}
280-
281-
private HttpRequest buildFetchFileRequest(String decodedUrl, UserCredentials userCredentials) {
282-
var builder = HttpRequest.newBuilder()
283-
.GET()
284-
.timeout(Duration.ofMinutes(15));
285-
var uri = URI.create(decodedUrl);
286-
var userInfo = uri.getUserInfo();
287-
if (userCredentials != null) {
288-
builder.uri(uri);
289-
String userCredentialsUrlFormat = MessageFormat.format(USERNAME_PASSWORD_URL_FORMAT, userCredentials.getUsername(),
290-
userCredentials.getPassword());
291-
String encodedAuth = Base64.getEncoder()
292-
.encodeToString(userCredentialsUrlFormat.getBytes());
293-
builder.header(HttpHeaders.AUTHORIZATION, "Basic " + encodedAuth);
294-
} else if (userInfo != null) {
295-
builder.uri(URI.create(decodedUrl.replace(uri.getRawUserInfo() + "@", "")));
296-
String encodedAuth = Base64.getEncoder()
297-
.encodeToString(userInfo.getBytes());
298-
builder.header(HttpHeaders.AUTHORIZATION, "Basic " + encodedAuth);
299-
} else {
300-
builder.uri(uri);
301-
}
302-
return builder.build();
303-
}
304-
305-
private String readErrorBodyFromResponse(HttpResponse<InputStream> response) throws IOException {
306-
try (InputStream is = response.body()) {
307-
byte[] buffer = new byte[ERROR_RESPONSE_BODY_MAX_LENGTH];
308-
int read = IOUtils.read(is, buffer);
309-
return new String(Arrays.copyOf(buffer, read));
310-
}
311206
}
312207

313208
private String extractFileName(String url) {

0 commit comments

Comments
 (0)