diff --git a/launchable/jar/exe_deploy.jar b/launchable/jar/exe_deploy.jar index a9242212a..fea8b75d9 100755 Binary files a/launchable/jar/exe_deploy.jar and b/launchable/jar/exe_deploy.jar differ diff --git a/src/main/java/com/launchableinc/ingest/commits/CommitGraphCollector.java b/src/main/java/com/launchableinc/ingest/commits/CommitGraphCollector.java index 9b75bea2d..08ab9bb5a 100644 --- a/src/main/java/com/launchableinc/ingest/commits/CommitGraphCollector.java +++ b/src/main/java/com/launchableinc/ingest/commits/CommitGraphCollector.java @@ -89,6 +89,8 @@ public class CommitGraphCollector { private int maxDays; + private boolean reportAllFiles; + private boolean audit; private boolean dryRun; @@ -150,7 +152,7 @@ public void transfer(URL service, Authenticator authenticator, boolean enableTim } CloseableHttpResponse latestResponse = client.execute(new HttpGet(latestUrl.toExternalForm())); ImmutableList advertised = getAdvertisedRefs(handleError(latestUrl, latestResponse)); - honorMaxDaysHeader(latestResponse); + honorControlHeaders(latestResponse); // every time a new stream is needed, supply ByteArrayOutputStream, and when the data is all // written, turn around and ship that over @@ -301,16 +303,21 @@ public void accept(VirtualFile f) { } } - /** - * When a user incorrectly configures shallow clone, the incremental nature of commit collection - * makes it really hard for us and users to collaboratively reset and repopulate the commit data. - * This server-side override mechanism makes it easier. - */ - private void honorMaxDaysHeader(HttpResponse response) { + private void honorControlHeaders(HttpResponse response) { + // When a user incorrectly configures shallow clone, the incremental nature of commit collection + // makes it really hard for us and users to collaboratively reset and repopulate the commit data. + // This server-side override mechanism makes it easier. Header h = response.getFirstHeader("X-Max-Days"); if (h!=null) { maxDays = Integer.parseInt(h.getValue()); } + // File transfer is supposed to work incrementally, but we are investigating the problem where + // not all files get collected prior to commit collection, resulting in incomplete data on the server side. + // As a temporary mitigation, allow the server to request all files to be reported. + h = response.getFirstHeader("X-Report-All-Files"); + if (h!=null) { + reportAllFiles = true; + } } private ImmutableList getAdvertisedRefs(HttpResponse response) throws IOException { @@ -436,7 +443,9 @@ public void transfer(Collection advertised, Consumer commitR try { RevCommit c = walk.parseCommit(id); walk.markUninteresting(c); - treeWalk.addTree(c.getTree()); + if (!reportAllFiles) { + treeWalk.addTree(c.getTree()); + } } catch (MissingObjectException e) { // it's possible that the server advertises a commit we don't have. //