JS: Factor out VirtualSourceRoot

Author: asgerf

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -211,6 +211,7 @@ public class AutoBuild {
   private volatile boolean seenCode = false;
   private boolean installDependencies = false;
   private int installDependenciesTimeout;
+  private final VirtualSourceRoot virtualSourceRoot;
 
   /** The default timeout when running <code>yarn</code>, in milliseconds. */
   public static final int INSTALL_DEPENDENCIES_DEFAULT_TIMEOUT = 10 * 60 * 1000; // 10 minutes
@@ -228,6 +229,7 @@ public AutoBuild() {
         Env.systemEnv()
             .getInt(
                 "LGTM_INDEX_TYPESCRIPT_INSTALL_DEPS_TIMEOUT", INSTALL_DEPENDENCIES_DEFAULT_TIMEOUT);
+    this.virtualSourceRoot = new VirtualSourceRoot(LGTM_SRC, toRealPath(Paths.get(EnvironmentVariables.getScratchDir())));
     setupFileTypes();
     setupXmlMode();
     setupMatchers();
@@ -758,7 +760,6 @@ public static Path tryRelativize(Path from, Path to) {
    */
 protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path> filesToExtract) {
     final Path sourceRoot = LGTM_SRC;
-    final Path virtualSourceRoot = toRealPath(Paths.get(EnvironmentVariables.getScratchDir()));
 
     // Read all package.json files and index them by name.
     Map<Path, JsonObject> packageJsonFiles = new LinkedHashMap<>();
@@ -845,8 +846,7 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
 
     // Write the new package.json files to disk
     for (Path file : packageJsonFiles.keySet()) {
-      Path relativePath = sourceRoot.relativize(file);
-      Path virtualFile = virtualSourceRoot.resolve(relativePath);
+      Path virtualFile = virtualSourceRoot.toVirtualFile(file);
 
       try {
         Files.createDirectories(virtualFile.getParent());
@@ -861,7 +861,7 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
     // Install dependencies
     if (this.installDependencies && verifyYarnInstallation()) {
       for (Path file : packageJsonFiles.keySet()) {
-        Path virtualFile = virtualSourceRoot.resolve(sourceRoot.relativize(file));
+        Path virtualFile = virtualSourceRoot.toVirtualFile(file);
         System.out.println("Installing dependencies from " + virtualFile);
         ProcessBuilder pb =
             new ProcessBuilder(
@@ -887,7 +887,7 @@ protected DependencyInstallationResult preparePackagesAndDependencies(Set<Path>
       }
     }
 
-    return new DependencyInstallationResult(sourceRoot, virtualSourceRoot, packageMainFile, packagesInRepo);
+    return new DependencyInstallationResult(packageMainFile, packagesInRepo);
   }
 
   /**
@@ -958,6 +958,7 @@ private ExtractorConfig mkExtractorConfig() {
     ExtractorConfig config = new ExtractorConfig(true);
     config = config.withSourceType(getSourceType());
     config = config.withTypeScriptMode(typeScriptMode);
+    config = config.withVirtualSourceRoot(virtualSourceRoot);
     if (defaultEncoding != null) config = config.withDefaultEncoding(defaultEncoding);
     return config;
   }
@@ -979,15 +980,15 @@ private Set<Path> extractTypeScript(
       Set<File> explicitlyIncludedFiles = new LinkedHashSet<>();
       if (tsconfig.size() > 1) { // No prioritization needed if there's only one tsconfig.
         for (Path projectPath : tsconfig) {
-          explicitlyIncludedFiles.addAll(tsParser.getOwnFiles(projectPath.toFile(), deps));
+          explicitlyIncludedFiles.addAll(tsParser.getOwnFiles(projectPath.toFile(), deps, virtualSourceRoot));
         }
       }
 
       // Extract TypeScript projects
       for (Path projectPath : tsconfig) {
         File projectFile = projectPath.toFile();
         long start = logBeginProcess("Opening project " + projectFile);
-        ParsedProject project = tsParser.openProject(projectFile, deps);
+        ParsedProject project = tsParser.openProject(projectFile, deps, virtualSourceRoot);
         logEndProcess(start, "Done opening project " + projectFile);
         // Extract all files belonging to this project which are also matched
         // by our include/exclude filters.

javascript/extractor/src/com/semmle/js/extractor/DependencyInstallationResult.java

@@ -6,46 +6,19 @@
 
 /** Contains the results of installing dependencies. */
 public class DependencyInstallationResult {
-  private Path sourceRoot;
-  private Path virtualSourceRoot;
   private Map<String, Path> packageEntryPoints;
   private Map<String, Path> packageJsonFiles;
 
   public static final DependencyInstallationResult empty =
-      new DependencyInstallationResult(null, null, Collections.emptyMap(), Collections.emptyMap());
+      new DependencyInstallationResult(Collections.emptyMap(), Collections.emptyMap());
 
   public DependencyInstallationResult(
-      Path sourceRoot,
-      Path virtualSourceRoot,
       Map<String, Path> packageEntryPoints,
       Map<String, Path> packageJsonFiles) {
-    this.sourceRoot = sourceRoot;
-    this.virtualSourceRoot = virtualSourceRoot;
     this.packageEntryPoints = packageEntryPoints;
     this.packageJsonFiles = packageJsonFiles;
   }
 
-  /**
-   * Returns the source root mirrored by {@link #getVirtualSourceRoot()} or <code>null</code>
-   * if no virtual source root exists.
-   * <p>
-   * When invoked from the AutoBuilder, this corresponds to the source root. When invoked
-   * from ODASA, there is no notion of source root, so this is always <code>null</code> in that context.
-   */
-  public Path getSourceRoot() {
-    return sourceRoot;
-  }
-
-  /**
-   * Returns the virtual source root or <code>null</code> if no virtual source root exists.
-   * <p>
-   * The virtual source root is a directory hierarchy that mirrors the real source
-   * root, where dependencies are installed.
-   */
-  public Path getVirtualSourceRoot() {
-    return virtualSourceRoot;
-  }
-
   /**
    * Returns the mapping from package names to the TypeScript file that should
    * act as its main entry point.

javascript/extractor/src/com/semmle/js/extractor/ExtractorConfig.java

@@ -1,8 +1,5 @@
 package com.semmle.js.extractor;
 
-import com.semmle.js.parser.JcornWrapper;
-import com.semmle.util.data.StringUtil;
-import com.semmle.util.exception.UserError;
 import java.nio.charset.Charset;
 import java.nio.charset.IllegalCharsetNameException;
 import java.nio.charset.StandardCharsets;
@@ -12,6 +9,10 @@
 import java.util.LinkedHashSet;
 import java.util.Set;
 
+import com.semmle.js.parser.JcornWrapper;
+import com.semmle.util.data.StringUtil;
+import com.semmle.util.exception.UserError;
+
 /**
  * Configuration options that affect the behaviour of the extractor.
  *
@@ -235,6 +236,8 @@ public String toString() {
 
   /** The default character encoding to use for parsing source files. */
   private String defaultEncoding;
+  
+  private VirtualSourceRoot virtualSourceRoot;
 
   public ExtractorConfig(boolean experimental) {
     this.ecmaVersion = experimental ? ECMAVersion.ECMA2020 : ECMAVersion.ECMA2019;
@@ -252,6 +255,7 @@ public ExtractorConfig(boolean experimental) {
     this.typescriptMode = TypeScriptMode.NONE;
     this.e4x = experimental;
     this.defaultEncoding = StandardCharsets.UTF_8.name();
+    this.virtualSourceRoot = VirtualSourceRoot.none;
   }
 
   public ExtractorConfig(ExtractorConfig that) {
@@ -272,6 +276,7 @@ public ExtractorConfig(ExtractorConfig that) {
     this.typescriptMode = that.typescriptMode;
     this.typescriptRam = that.typescriptRam;
     this.defaultEncoding = that.defaultEncoding;
+    this.virtualSourceRoot = that.virtualSourceRoot;
   }
 
   public ECMAVersion getEcmaVersion() {
@@ -452,6 +457,16 @@ public ExtractorConfig withDefaultEncoding(String defaultEncoding) {
     return res;
   }
 
+  public VirtualSourceRoot getVirtualSourceRoot() {
+    return virtualSourceRoot;
+  }
+  
+  public ExtractorConfig withVirtualSourceRoot(VirtualSourceRoot virtualSourceRoot) {
+    ExtractorConfig res = new ExtractorConfig(this);
+    res.virtualSourceRoot = virtualSourceRoot;
+    return res;
+  }
+
   @Override
   public String toString() {
     return "ExtractorConfig [ecmaVersion="
@@ -486,6 +501,8 @@ public String toString() {
         + typescriptMode
         + ", defaultEncoding="
         + defaultEncoding
+        + ", virtualSourceRoot="
+        + virtualSourceRoot
         + "]";
   }
 }

javascript/extractor/src/com/semmle/js/extractor/Main.java

@@ -152,7 +152,7 @@ public void run(String[] args) {
     for (File projectFile : projectFiles) {
 
       long start = verboseLogStartTimer(ap, "Opening project " + projectFile);
-      ParsedProject project = tsParser.openProject(projectFile, DependencyInstallationResult.empty);
+      ParsedProject project = tsParser.openProject(projectFile, DependencyInstallationResult.empty, VirtualSourceRoot.none);
       verboseLogEndTimer(ap, start);
       // Extract all files belonging to this project which are also matched
       // by our include/exclude filters.

javascript/extractor/src/com/semmle/js/extractor/VirtualSourceRoot.java

@@ -0,0 +1,56 @@
+package com.semmle.js.extractor;
+
+import java.nio.file.Path;
+
+public class VirtualSourceRoot {
+  private Path sourceRoot;
+  private Path virtualSourceRoot;
+  
+  public static final VirtualSourceRoot none = new VirtualSourceRoot(null, null);
+
+  public VirtualSourceRoot(Path sourceRoot, Path virtualSourceRoot) {
+    this.sourceRoot = sourceRoot;
+    this.virtualSourceRoot = virtualSourceRoot;
+  }
+
+  /**
+   * Returns the source root mirrored by {@link #getVirtualSourceRoot()} or <code>null</code> if no
+   * virtual source root exists.
+   *
+   * <p>When invoked from the AutoBuilder, this corresponds to the source root. When invoked from
+   * ODASA, there is no notion of source root, so this is always <code>null</code> in that context.
+   */
+  public Path getSourceRoot() {
+    return sourceRoot;
+  }
+
+  /**
+   * Returns the virtual source root or <code>null</code> if no virtual source root exists.
+   *
+   * <p>The virtual source root is a directory hierarchy that mirrors the real source root, where
+   * dependencies are installed.
+   */
+  public Path getVirtualSourceRoot() {
+    return virtualSourceRoot;
+  }
+  
+  private static Path translate(Path oldRoot, Path newRoot, Path file) {
+    if (oldRoot == null || newRoot == null) return null;
+    Path relative = oldRoot.relativize(file);
+    if (relative.startsWith("..") || relative.isAbsolute()) return null;
+    return newRoot.resolve(relative);
+  }
+
+  public Path toVirtualFile(Path file) {
+    return translate(sourceRoot, virtualSourceRoot, file);
+  }
+
+  public Path fromVirtualFile(Path file) {
+    return translate(virtualSourceRoot, sourceRoot, file);
+  }
+
+  @Override
+  public String toString() {
+    return "[sourceRoot=" + sourceRoot + ", virtualSourceRoot=" + virtualSourceRoot + "]";
+  }
+}

javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java

@@ -31,6 +31,7 @@
 import com.semmle.js.extractor.DependencyInstallationResult;
 import com.semmle.js.extractor.EnvironmentVariables;
 import com.semmle.js.extractor.ExtractionMetrics;
+import com.semmle.js.extractor.VirtualSourceRoot;
 import com.semmle.js.parser.JSParser.Result;
 import com.semmle.ts.extractor.TypeTable;
 import com.semmle.util.data.StringUtil;
@@ -501,8 +502,8 @@ private static Set<File> getFilesFromJsonArray(JsonArray array) {
   /**
    * Returns the set of files included by the inclusion pattern in the given tsconfig.json file.
    */
-  public Set<File> getOwnFiles(File tsConfigFile, DependencyInstallationResult deps) {
-    JsonObject request = makeLoadCommand("get-own-files", tsConfigFile, deps);
+  public Set<File> getOwnFiles(File tsConfigFile, DependencyInstallationResult deps, VirtualSourceRoot vroot) {
+    JsonObject request = makeLoadCommand("get-own-files", tsConfigFile, deps, vroot);
     JsonObject response = talkToParserWrapper(request);
     try {
       checkResponseType(response, "file-list");
@@ -521,8 +522,8 @@ public Set<File> getOwnFiles(File tsConfigFile, DependencyInstallationResult dep
    *
    * <p>Only one project should be opened at once.
    */
-  public ParsedProject openProject(File tsConfigFile, DependencyInstallationResult deps) {
-    JsonObject request = makeLoadCommand("open-project", tsConfigFile, deps);
+  public ParsedProject openProject(File tsConfigFile, DependencyInstallationResult deps, VirtualSourceRoot vroot) {
+    JsonObject request = makeLoadCommand("open-project", tsConfigFile, deps, vroot);
     JsonObject response = talkToParserWrapper(request);
     try {
       checkResponseType(response, "project-opened");
@@ -536,18 +537,18 @@ public ParsedProject openProject(File tsConfigFile, DependencyInstallationResult
     }
   }
 
-  private JsonObject makeLoadCommand(String command, File tsConfigFile, DependencyInstallationResult deps) {
+  private JsonObject makeLoadCommand(String command, File tsConfigFile, DependencyInstallationResult deps, VirtualSourceRoot vroot) {
     JsonObject request = new JsonObject();
     request.add("command", new JsonPrimitive(command));
     request.add("tsConfig", new JsonPrimitive(tsConfigFile.getPath()));
     request.add("packageEntryPoints", mapToArray(deps.getPackageEntryPoints()));
     request.add("packageJsonFiles", mapToArray(deps.getPackageJsonFiles()));
-    request.add("sourceRoot", deps.getSourceRoot() == null
+    request.add("sourceRoot", vroot.getSourceRoot() == null
         ? JsonNull.INSTANCE
-        : new JsonPrimitive(deps.getSourceRoot().toString()));
-    request.add("virtualSourceRoot", deps.getVirtualSourceRoot() == null
+        : new JsonPrimitive(vroot.getSourceRoot().toString()));
+    request.add("virtualSourceRoot", vroot.getVirtualSourceRoot() == null
         ? JsonNull.INSTANCE
-        : new JsonPrimitive(deps.getVirtualSourceRoot().toString()));
+        : new JsonPrimitive(vroot.getVirtualSourceRoot().toString()));
     return request;
   }