JS: Extract TypeScript in HTML files to a snippet in virtual source root

Author: asgerf

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -213,6 +213,7 @@ public class AutoBuild {
   private boolean installDependencies = false;
   private int installDependenciesTimeout;
   private final VirtualSourceRoot virtualSourceRoot;
+  private ExtractorState state;
 
   /** The default timeout when running <code>yarn</code>, in milliseconds. */
   public static final int INSTALL_DEPENDENCIES_DEFAULT_TIMEOUT = 10 * 60 * 1000; // 10 minutes
@@ -234,6 +235,7 @@ public AutoBuild() {
     setupFileTypes();
     setupXmlMode();
     setupMatchers();
+    this.state = new ExtractorState();
   }
 
   private String getEnvVar(String envVarName) {
@@ -534,7 +536,7 @@ public File lookup(String source, ExtractorConfig config, FileType type) {
           @Override
           public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
               throws IOException {
-            if (".js".equals(FileUtil.extension(file.toString()))) extract(extractor, file, null);
+            if (".js".equals(FileUtil.extension(file.toString()))) extract(extractor, file, true);
             return super.visitFile(file, attrs);
           }
         };
@@ -656,7 +658,7 @@ private CompletableFuture<?> extractFiles(
         continue;
       }
       extractedFiles.add(f);
-      futures.add(extract(extractors.forFile(f), f, null));
+      futures.add(extract(extractors.forFile(f), f, true));
     }
     return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
   }
@@ -980,9 +982,8 @@ private Set<Path> extractTypeScript(
       List<Path> tsconfig,
       DependencyInstallationResult deps) {
     if (hasTypeScriptFiles(files) || !tsconfig.isEmpty()) {
-      ExtractorState extractorState = new ExtractorState();
-      TypeScriptParser tsParser = extractorState.getTypeScriptParser();
-      verifyTypeScriptInstallation(extractorState);
+      TypeScriptParser tsParser = state.getTypeScriptParser();
+      verifyTypeScriptInstallation(state);
 
       // Collect all files included in a tsconfig.json inclusion pattern.
       // If a given file is referenced by multiple tsconfig files, we prefer to extract it using
@@ -1005,7 +1006,10 @@ private Set<Path> extractTypeScript(
         List<Path> typeScriptFiles = new ArrayList<Path>();
         for (File sourceFile : project.getAllFiles()) {
           Path sourcePath = sourceFile.toPath();
-          if (!files.contains(normalizePath(sourcePath))) continue;
+          Path normalizedFile = normalizePath(sourcePath);
+          if (!files.contains(normalizedFile) && !state.getSnippets().containsKey(normalizedFile)) {
+            continue;
+          }
           if (!project.getOwnFiles().contains(sourceFile) && explicitlyIncludedFiles.contains(sourceFile)) continue;
           if (extractors.fileType(sourcePath) != FileType.TYPESCRIPT) {
             // For the time being, skip non-TypeScript files, even if the TypeScript
@@ -1017,7 +1021,7 @@ private Set<Path> extractTypeScript(
           }
         }
         typeScriptFiles.sort(PATH_ORDERING);
-        extractTypeScriptFiles(typeScriptFiles, extractedFiles, extractors, extractorState);
+        extractTypeScriptFiles(typeScriptFiles, extractedFiles, extractors);
         tsParser.closeProject(projectFile);
       }
 
@@ -1036,7 +1040,7 @@ private Set<Path> extractTypeScript(
         }
       }
       if (!remainingTypeScriptFiles.isEmpty()) {
-        extractTypeScriptFiles(remainingTypeScriptFiles, extractedFiles, extractors, extractorState);
+        extractTypeScriptFiles(remainingTypeScriptFiles, extractedFiles, extractors);
       }
 
       // The TypeScript compiler instance is no longer needed.
@@ -1122,16 +1126,15 @@ public void verifyTypeScriptInstallation(ExtractorState extractorState) {
   public void extractTypeScriptFiles(
       List<Path> files,
       Set<Path> extractedFiles,
-      FileExtractors extractors,
-      ExtractorState extractorState) {
+      FileExtractors extractors) {
     List<File> list = files
         .stream()
         .sorted(PATH_ORDERING)
         .map(p -> p.toFile()).collect(Collectors.toList());
-    extractorState.getTypeScriptParser().prepareFiles(list);
+    state.getTypeScriptParser().prepareFiles(list);
     for (Path path : files) {
       extractedFiles.add(path);
-      extract(extractors.forFile(path), path, extractorState);
+      extract(extractors.forFile(path), path, false);
     }
   }
 
@@ -1174,8 +1177,8 @@ private SourceType getSourceType() {
    * <p>If the state is {@code null}, the extraction job will be submitted to the {@link
    * #threadPool}, otherwise extraction will happen on the main thread.
    */
-  protected CompletableFuture<?> extract(FileExtractor extractor, Path file, ExtractorState state) {
-    if (state == null && threadPool != null) {
+  protected CompletableFuture<?> extract(FileExtractor extractor, Path file, boolean concurrent) {
+    if (concurrent && threadPool != null) {
       return CompletableFuture.runAsync(() -> doExtract(extractor, file, state), threadPool);
     } else {
       doExtract(extractor, file, state);

javascript/extractor/src/com/semmle/js/extractor/ExtractorState.java

@@ -1,5 +1,8 @@
 package com.semmle.js.extractor;
 
+import java.nio.file.Path;
+import java.util.concurrent.ConcurrentHashMap;
+
 import com.semmle.js.parser.TypeScriptParser;
 
 /**
@@ -17,16 +20,28 @@
  */
 public class ExtractorState {
   private TypeScriptParser typeScriptParser = new TypeScriptParser();
+  
+  private final ConcurrentHashMap<Path, FileSnippet> snippets = new ConcurrentHashMap<>();
 
   public TypeScriptParser getTypeScriptParser() {
     return typeScriptParser;
   }
 
+  /**
+   * Returns the mapping that denotes where a snippet file originated from.
+   *
+   * <p>The map is thread-safe and may be mutated by the caller.
+   */
+  public ConcurrentHashMap<Path, FileSnippet> getSnippets() {
+    return snippets;
+  }
+
   /**
    * Makes this semantically equivalent to a fresh state, but may internally retain shared resources
    * that are expensive to reacquire.
    */
   public void reset() {
     typeScriptParser.reset();
+    snippets.clear();
   }
 }

javascript/extractor/src/com/semmle/js/extractor/FileExtractor.java

@@ -1,26 +1,28 @@
 package com.semmle.js.extractor;
 
-import com.semmle.js.extractor.ExtractionMetrics.ExtractionPhase;
-import com.semmle.js.extractor.trapcache.CachingTrapWriter;
-import com.semmle.js.extractor.trapcache.ITrapCache;
-import com.semmle.util.data.StringUtil;
-import com.semmle.util.exception.Exceptions;
-import com.semmle.util.extraction.ExtractorOutputConfig;
-import com.semmle.util.files.FileUtil;
-import com.semmle.util.io.WholeIO;
-import com.semmle.util.trap.TrapWriter;
-import com.semmle.util.trap.TrapWriter.Label;
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileReader;
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
 import java.util.LinkedHashSet;
 import java.util.Set;
 import java.util.regex.Pattern;
 
+import com.semmle.js.extractor.ExtractionMetrics.ExtractionPhase;
+import com.semmle.js.extractor.trapcache.CachingTrapWriter;
+import com.semmle.js.extractor.trapcache.ITrapCache;
+import com.semmle.util.data.StringUtil;
+import com.semmle.util.exception.Exceptions;
+import com.semmle.util.extraction.ExtractorOutputConfig;
+import com.semmle.util.files.FileUtil;
+import com.semmle.util.io.WholeIO;
+import com.semmle.util.trap.TrapWriter;
+import com.semmle.util.trap.TrapWriter.Label;
+
 /**
  * The file extractor extracts a single file and handles source archive population and TRAP caching;
  * it delegates to the appropriate {@link IExtractor} for extracting the contents of the file.
@@ -47,7 +49,7 @@ public static enum FileType {
     HTML(".htm", ".html", ".xhtm", ".xhtml", ".vue") {
       @Override
       public IExtractor mkExtractor(ExtractorConfig config, ExtractorState state) {
-        return new HTMLExtractor(config);
+        return new HTMLExtractor(config, state);
       }
 
       @Override
@@ -293,7 +295,7 @@ private boolean hasUnrecognizedShebang(byte[] bytes, int length) {
 
       @Override
       public IExtractor mkExtractor(ExtractorConfig config, ExtractorState state) {
-        return new TypeScriptExtractor(config, state.getTypeScriptParser());
+        return new TypeScriptExtractor(config, state);
       }
 
       @Override
@@ -398,6 +400,10 @@ public boolean supports(File f) {
 
   /** @return the number of lines of code extracted, or {@code null} if the file was cached */
   public Integer extract(File f, ExtractorState state) throws IOException {
+    FileSnippet snippet = state.getSnippets().get(f.toPath());
+    if (snippet != null) {
+      return this.extractSnippet(f.toPath(), snippet, state);
+    }
 
     // populate source archive
     String source = new WholeIO(config.getDefaultEncoding()).strictread(f);
@@ -414,6 +420,25 @@ public Integer extract(File f, ExtractorState state) throws IOException {
     return extractContents(f, fileLabel, source, locationManager, state);
   }
 
+  /**
+   * Extract the contents of a file that is a snippet from another file.
+   *
+   * <p>A trap file will be derived from the snippet file, but its file label, source locations, and
+   * source archive entry are based on the original file.
+   */
+  private Integer extractSnippet(Path file, FileSnippet origin, ExtractorState state) throws IOException {
+    TrapWriter trapwriter = outputConfig.getTrapWriterFactory().mkTrapWriter(file.toFile());
+
+    File originalFile = origin.getOriginalFile().toFile();
+    Label fileLabel = trapwriter.populateFile(originalFile);
+    LocationManager locationManager = new LocationManager(originalFile, trapwriter, fileLabel);
+    locationManager.setStart(origin.getLine(), origin.getColumn());
+
+    String source = new WholeIO(config.getDefaultEncoding()).strictread(file);
+
+    return extractContents(file.toFile(), fileLabel, source, locationManager, state);
+  }
+
   /**
    * Extract the contents of a file, potentially making use of cached information.
    *
@@ -436,20 +461,20 @@ public Integer extract(File f, ExtractorState state) throws IOException {
    * obviously, no caching is done in that scenario.
    */
   private Integer extractContents(
-      File f, Label fileLabel, String source, LocationManager locationManager, ExtractorState state)
+      File extractedFile, Label fileLabel, String source, LocationManager locationManager, ExtractorState state)
       throws IOException {
     ExtractionMetrics metrics = new ExtractionMetrics();
     metrics.startPhase(ExtractionPhase.FileExtractor_extractContents);
     metrics.setLength(source.length());
     metrics.setFileLabel(fileLabel);
     TrapWriter trapwriter = locationManager.getTrapWriter();
-    FileType fileType = getFileType(f);
+    FileType fileType = getFileType(extractedFile);
 
     File cacheFile = null, // the cache file for this extraction
         resultFile = null; // the final result TRAP file for this extraction
 
     if (bumpIdCounter(trapwriter)) {
-      resultFile = outputConfig.getTrapWriterFactory().getTrapFileFor(f);
+      resultFile = outputConfig.getTrapWriterFactory().getTrapFileFor(extractedFile);
     }
     // check whether we can perform caching
     if (resultFile != null && fileType.isTrapCachingAllowed()) {
@@ -475,7 +500,7 @@ private Integer extractContents(
       trapwriter = new CachingTrapWriter(cacheFile, resultFile);
       bumpIdCounter(trapwriter);
       // re-initialise the location manager, since it keeps a reference to the TRAP writer
-      locationManager = new LocationManager(f, trapwriter, locationManager.getFileLabel());
+      locationManager = new LocationManager(extractedFile, trapwriter, locationManager.getFileLabel());
     }
 
     // now do the extraction itself
@@ -484,7 +509,7 @@ private Integer extractContents(
       IExtractor extractor = fileType.mkExtractor(config, state);
       TextualExtractor textualExtractor =
           new TextualExtractor(
-              trapwriter, locationManager, source, config.getExtractLines(), metrics);
+              trapwriter, locationManager, source, config.getExtractLines(), metrics, extractedFile);
       LoCInfo loc = extractor.extract(textualExtractor);
       int numLines = textualExtractor.getNumLines();
       int linesOfCode = loc.getLinesOfCode(), linesOfComments = loc.getLinesOfComments();

javascript/extractor/src/com/semmle/js/extractor/FileSnippet.java

@@ -0,0 +1,36 @@
+package com.semmle.js.extractor;
+
+import java.nio.file.Path;
+
+/**
+ * Denotes where a code snippet originated from within a file.
+ */
+public class FileSnippet {
+  private Path originalFile;
+  private int line;
+  private int column;
+  private int topLevelKind;
+
+  public FileSnippet(Path originalFile, int line, int column, int topLevelKind) {
+    this.originalFile = originalFile;
+    this.line = line;
+    this.column = column;
+    this.topLevelKind = topLevelKind;
+  }
+
+  public Path getOriginalFile() {
+    return originalFile;
+  }
+
+  public int getLine() {
+    return line;
+  }
+
+  public int getColumn() {
+    return column;
+  }
+
+  public int getTopLevelKind() {
+    return topLevelKind;
+  }
+}

javascript/extractor/src/com/semmle/js/extractor/HTMLExtractor.java

@@ -1,11 +1,13 @@
 package com.semmle.js.extractor;
 
+import java.nio.file.Path;
 import java.util.regex.Pattern;
 
 import com.semmle.js.extractor.ExtractorConfig.Platform;
 import com.semmle.js.extractor.ExtractorConfig.SourceType;
 import com.semmle.js.parser.ParseError;
 import com.semmle.util.data.StringUtil;
+import com.semmle.util.io.WholeIO;
 import com.semmle.util.trap.TrapWriter;
 import com.semmle.util.trap.TrapWriter.Label;
 
@@ -28,9 +30,11 @@ public class HTMLExtractor implements IExtractor {
           Pattern.CASE_INSENSITIVE);
 
   private final ExtractorConfig config;
+  private final ExtractorState state;
 
-  public HTMLExtractor(ExtractorConfig config) {
+  public HTMLExtractor(ExtractorConfig config, ExtractorState state) {
     this.config = config.withPlatform(Platform.WEB);
+    this.state = state;
   }
 
   @Override
@@ -208,8 +212,25 @@ private LoCInfo extractSnippet(
       int line,
       int column,
       boolean isTypeScript) {
-    if (isTypeScript)
-      return null; // not supported right now
+    if (isTypeScript) {
+      Path file = textualExtractor.getExtractedFile().toPath();
+      FileSnippet snippet = new FileSnippet(file, line, column, toplevelKind);
+      VirtualSourceRoot vroot = config.getVirtualSourceRoot();
+      // Vue files are special in that they can be imported as modules, and may only contain one <script> tag.
+      // For .vue files we omit the usual snippet decoration to ensure the TypeScript compiler can find it.
+      Path virtualFile =
+          file.getFileName().toString().endsWith(".vue")
+          ? vroot.toVirtualFile(file.resolveSibling(file.getFileName() + ".ts"))
+          : vroot.getVirtualFileForSnippet(snippet, ".ts");
+      if (virtualFile != null) {
+        virtualFile = virtualFile.toAbsolutePath().normalize();
+        synchronized(vroot.getLock()) {
+          new WholeIO().strictwrite(virtualFile, source);
+        }
+        state.getSnippets().put(virtualFile, snippet);
+      }
+      return null; // LoC info is accounted for later
+    }
     TrapWriter trapwriter = textualExtractor.getTrapwriter();
     LocationManager locationManager = textualExtractor.getLocationManager();
     LocationManager scriptLocationManager =
@@ -224,7 +245,8 @@ private LoCInfo extractSnippet(
               scriptLocationManager,
               source,
               config.getExtractLines(),
-              textualExtractor.getMetrics());
+              textualExtractor.getMetrics(),
+              textualExtractor.getExtractedFile());
       return extractor.extract(tx, source, toplevelKind, scopeManager).snd();
     } catch (ParseError e) {
       e.setPosition(scriptLocationManager.translatePosition(e.getPosition()));