[LCHIB-666] allow the server to request reporting of all BLOBs

src/main/java/com/launchableinc/ingest/commits/CommitGraphCollector.java

@@ -89,6 +89,8 @@ public class CommitGraphCollector {
 
   private int maxDays;
 
+  private boolean reportAllFiles;
+
   private boolean audit;
 
   private boolean dryRun;
@@ -150,7 +152,7 @@ public void transfer(URL service, Authenticator authenticator, boolean enableTim
       }
       CloseableHttpResponse latestResponse = client.execute(new HttpGet(latestUrl.toExternalForm()));
       ImmutableList<ObjectId> advertised = getAdvertisedRefs(handleError(latestUrl, latestResponse));
-      honorMaxDaysHeader(latestResponse);
+      honorControlHeaders(latestResponse);
 
       // every time a new stream is needed, supply ByteArrayOutputStream, and when the data is all
       // written, turn around and ship that over
@@ -301,16 +303,21 @@ public void accept(VirtualFile f) {
     }
   }
 
-  /**
-   * When a user incorrectly configures shallow clone, the incremental nature of commit collection
-   * makes it really hard for us and users to collaboratively reset and repopulate the commit data.
-   * This server-side override mechanism makes it easier.
-   */
-  private void honorMaxDaysHeader(HttpResponse response) {
+  private void honorControlHeaders(HttpResponse response) {
+    // When a user incorrectly configures shallow clone, the incremental nature of commit collection
+    // makes it really hard for us and users to collaboratively reset and repopulate the commit data.
+    // This server-side override mechanism makes it easier.
     Header h = response.getFirstHeader("X-Max-Days");
     if (h!=null) {
       maxDays = Integer.parseInt(h.getValue());
     }
+    // File transfer is supposed to work incrementally, but we are investigating the problem where
+    // not all files get collected prior to commit collection, resulting in incomplete data on the server side.
+    // As a temporary mitigation, allow the server to request all files to be reported.
+    h = response.getFirstHeader("X-Report-All-Files");
+    if (h!=null) {
+      reportAllFiles = true;
+    }
   }
 
   private ImmutableList<ObjectId> getAdvertisedRefs(HttpResponse response) throws IOException {
@@ -436,7 +443,9 @@ public void transfer(Collection<ObjectId> advertised, Consumer<JSCommit> commitR
           try {
             RevCommit c = walk.parseCommit(id);
             walk.markUninteresting(c);
-            treeWalk.addTree(c.getTree());
+            if (!reportAllFiles) {
+              treeWalk.addTree(c.getTree());
+            }
           } catch (MissingObjectException e) {
             // it's possible that the server advertises a commit we don't have.
             //