JS: Extract HTML before TypeScript

Author: asgerf

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -26,6 +26,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
@@ -621,6 +622,13 @@ private void extractSource() throws IOException {
       dependencyInstallationResult = this.preparePackagesAndDependencies(filesToExtract);
     }
     Set<Path> extractedFiles = new LinkedHashSet<>();
+    
+    // Extract HTML files as they may contain TypeScript
+    CompletableFuture<?> htmlFuture = extractFiles(
+        filesToExtract, extractedFiles, extractors,
+        f -> extractors.fileType(f) == FileType.HTML);
+    
+    htmlFuture.join(); // Wait for HTML extraction to be finished.
 
     // extract TypeScript projects and files
     extractTypeScript(filesToExtract, extractedFiles,
@@ -634,21 +642,23 @@ private void extractSource() throws IOException {
         f -> !(hasTypeScriptFiles && isFileDerivedFromTypeScriptFile(f, extractedFiles)));
   }
 
-  private void extractFiles(
+  private CompletableFuture<?> extractFiles(
       Set<Path> filesToExtract,
       Set<Path> extractedFiles,
       FileExtractors extractors,
       Predicate<Path> shouldExtract) {
 
+    List<CompletableFuture<?>> futures = new ArrayList<>();
     for (Path f : filesToExtract) {
       if (extractedFiles.contains(f))
         continue;
       if (!shouldExtract.test(f)) {
         continue;
       }
       extractedFiles.add(f);
-      extract(extractors.forFile(f), f, null);
+      futures.add(extract(extractors.forFile(f), f, null));
     }
+    return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
   }
 
   /**
@@ -1164,10 +1174,13 @@ private SourceType getSourceType() {
    * <p>If the state is {@code null}, the extraction job will be submitted to the {@link
    * #threadPool}, otherwise extraction will happen on the main thread.
    */
-  protected void extract(FileExtractor extractor, Path file, ExtractorState state) {
-    if (state == null && threadPool != null)
-      threadPool.submit(() -> doExtract(extractor, file, state));
-    else doExtract(extractor, file, state);
+  protected CompletableFuture<?> extract(FileExtractor extractor, Path file, ExtractorState state) {
+    if (state == null && threadPool != null) {
+      return CompletableFuture.runAsync(() -> doExtract(extractor, file, state), threadPool);
+    } else {
+      doExtract(extractor, file, state);
+      return CompletableFuture.completedFuture(null);
+    }
   }
 
   private void doExtract(FileExtractor extractor, Path file, ExtractorState state) {

javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java

@@ -15,6 +15,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
 
 import org.junit.After;
 import org.junit.Assert;
@@ -109,11 +110,12 @@ private void runTest() throws IOException {
       Set<String> actual = new LinkedHashSet<>();
       new AutoBuild() {
         @Override
-        protected void extract(FileExtractor extractor, Path file, ExtractorState state) {
+        protected CompletableFuture<?> extract(FileExtractor extractor, Path file, ExtractorState state) {
           String extracted = file.toString();
           if (extractor.getConfig().hasFileType())
             extracted += ":" + extractor.getFileType(file.toFile());
           actual.add(extracted);
+          return CompletableFuture.completedFuture(null);
         }
 
         @Override