JS: Prefer extracting file with tsconfig that included it

Author: asgerf

javascript/extractor/lib/typescript/src/main.ts

@@ -58,6 +58,9 @@ interface LoadCommand {
 interface OpenProjectCommand extends LoadCommand {
     command: "open-project";
 }
+interface GetOwnFilesCommand extends LoadCommand {
+    command: "get-own-files";
+}
 interface CloseProjectCommand {
     command: "close-project";
     tsConfig: string;
@@ -78,7 +81,7 @@ interface PrepareFilesCommand {
 interface GetMetadataCommand {
     command: "get-metadata";
 }
-type Command = ParseCommand | OpenProjectCommand | CloseProjectCommand
+type Command = ParseCommand | OpenProjectCommand | GetOwnFilesCommand | CloseProjectCommand
     | GetTypeTableCommand | ResetCommand | QuitCommand | PrepareFilesCommand | GetMetadataCommand;
 
 /** The state to be shared between commands. */
@@ -374,6 +377,7 @@ interface LoadedConfig {
     packageEntryPoints: Map<string, string>;
     packageJsonFiles: Map<string, string>;
     virtualSourceRoot: VirtualSourceRoot;
+    ownFiles: string[];
 }
 
 function loadTsConfig(command: LoadCommand): LoadedConfig {
@@ -428,11 +432,26 @@ function loadTsConfig(command: LoadCommand): LoadedConfig {
     };
     let config = ts.parseJsonConfigFileContent(tsConfig.config, parseConfigHost, basePath);
 
-    return { config, basePath, packageJsonFiles, packageEntryPoints, virtualSourceRoot };
+    let ownFiles = config.fileNames.map(file => pathlib.resolve(file));
+
+    return { config, basePath, packageJsonFiles, packageEntryPoints, virtualSourceRoot, ownFiles };
+}
+
+/**
+ * Returns the list of files included in the given tsconfig.json file's include pattern,
+ * (not including those only references through imports).
+ */
+function handleGetFileListCommand(command: GetOwnFilesCommand) {
+    let { config, ownFiles } = loadTsConfig(command);
+
+    console.log(JSON.stringify({
+        type: "file-list",
+        ownFiles,
+    }));
 }
 
 function handleOpenProjectCommand(command: OpenProjectCommand) {
-    let { config, packageEntryPoints, virtualSourceRoot, basePath } = loadTsConfig(command);
+    let { config, packageEntryPoints, virtualSourceRoot, basePath, ownFiles } = loadTsConfig(command);
 
     let project = new Project(command.tsConfig, config, state.typeTable, packageEntryPoints, virtualSourceRoot);
     project.load();
@@ -606,9 +625,14 @@ function handleOpenProjectCommand(command: OpenProjectCommand) {
         return symbol;
     }
 
+    // Unlike in the get-own-files command, this command gets all files we can possibly
+    // extract type information for, including files referenced outside the tsconfig's inclusion pattern.
+    let allFiles = program.getSourceFiles().map(sf => pathlib.resolve(sf.fileName));
+
     console.log(JSON.stringify({
         type: "project-opened",
-        files: config.fileNames.map(file => pathlib.resolve(file)),
+        ownFiles,
+        allFiles,
     }));
 }
 
@@ -704,6 +728,9 @@ function runReadLineInterface() {
         case "open-project":
             handleOpenProjectCommand(req);
             break;
+        case "get-own-files":
+            handleGetFileListCommand(req);
+            break;
         case "close-project":
             handleCloseProjectCommand(req);
             break;

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -949,6 +949,16 @@ private Set<Path> extractTypeScript(
       TypeScriptParser tsParser = extractorState.getTypeScriptParser();
       verifyTypeScriptInstallation(extractorState);
 
+      // Collect all files included in a tsconfig.json inclusion pattern.
+      // If a given file is referenced by multiple tsconfig files, we prefer to extract it using
+      // one that includes it rather than just references it.
+      Set<File> explicitlyIncludedFiles = new LinkedHashSet<>();
+      if (tsconfig.size() > 1) { // No prioritization needed if there's only one tsconfig.
+        for (Path projectPath : tsconfig) {
+          explicitlyIncludedFiles.addAll(tsParser.getOwnFiles(projectPath.toFile(), deps));
+        }
+      }
+
       // Extract TypeScript projects
       for (Path projectPath : tsconfig) {
         File projectFile = projectPath.toFile();
@@ -958,9 +968,10 @@ private Set<Path> extractTypeScript(
         // Extract all files belonging to this project which are also matched
         // by our include/exclude filters.
         List<Path> typeScriptFiles = new ArrayList<Path>();
-        for (File sourceFile : project.getSourceFiles()) {
+        for (File sourceFile : project.getAllFiles()) {
           Path sourcePath = sourceFile.toPath();
           if (!files.contains(normalizePath(sourcePath))) continue;
+          if (!project.getOwnFiles().contains(sourceFile) && explicitlyIncludedFiles.contains(sourceFile)) continue;
           if (!FileType.TYPESCRIPT.getExtensions().contains(FileUtil.extension(sourcePath))) {
             // For the time being, skip non-TypeScript files, even if the TypeScript
             // compiler can parse them for us.

javascript/extractor/src/com/semmle/js/extractor/Main.java

@@ -157,7 +157,7 @@ public void run(String[] args) {
       // Extract all files belonging to this project which are also matched
       // by our include/exclude filters.
       List<File> filesToExtract = new ArrayList<>();
-      for (File sourceFile : project.getSourceFiles()) {
+      for (File sourceFile : project.getOwnFiles()) {
         if (files.contains(normalizeFile(sourceFile))
             && !extractedFiles.contains(sourceFile.getAbsoluteFile())
             && FileType.TYPESCRIPT.getExtensions().contains(FileUtil.extension(sourceFile))) {

javascript/extractor/src/com/semmle/js/parser/ParsedProject.java

@@ -1,15 +1,17 @@
 package com.semmle.js.parser;
 
 import java.io.File;
-import java.util.LinkedHashSet;
 import java.util.Set;
 
 public class ParsedProject {
   private final File tsConfigFile;
-  private final Set<File> sourceFiles = new LinkedHashSet<>();
+  private final Set<File> ownFiles;
+  private final Set<File> allFiles;
 
-  public ParsedProject(File tsConfigFile) {
+  public ParsedProject(File tsConfigFile, Set<File> ownFiles, Set<File> allFiles) {
     this.tsConfigFile = tsConfigFile;
+    this.ownFiles = ownFiles;
+    this.allFiles = allFiles;
   }
 
   /** Returns the <tt>tsconfig.json</tt> file that defines this project. */
@@ -18,11 +20,12 @@ public File getTsConfigFile() {
   }
 
   /** Absolute paths to the files included in this project. */
-  public Set<File> getSourceFiles() {
-    return sourceFiles;
+  public Set<File> getOwnFiles() {
+    return allFiles;
   }
 
-  public void addSourceFile(File file) {
-    sourceFiles.add(file);
+  /** Absolute paths to the files included in or referenced by this project. */
+  public Set<File> getAllFiles() {
+    return allFiles;
   }
 }

javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java

@@ -15,8 +15,10 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 import com.google.gson.JsonArray;
@@ -488,6 +490,29 @@ private JsonArray mapToArray(Map<String, Path> map) {
     return result;
   }
 
+  private static Set<File> getFilesFromJsonArray(JsonArray array) {
+    Set<File> files = new LinkedHashSet<>();
+    for (JsonElement elm : array) {
+      files.add(new File(elm.getAsString()));
+    }
+    return files;
+  }
+
+  /**
+   * Returns the set of files included by the inclusion pattern in the given tsconfig.json file.
+   */
+  public Set<File> getOwnFiles(File tsConfigFile, DependencyInstallationResult deps) {
+    JsonObject request = makeLoadCommand("get-own-files", tsConfigFile, deps);
+    JsonObject response = talkToParserWrapper(request);
+    try {
+      checkResponseType(response, "file-list");
+      return getFilesFromJsonArray(response.get("ownFiles").getAsJsonArray());
+    } catch (IllegalStateException e) {
+      throw new CatastrophicError(
+          "TypeScript parser wrapper sent unexpected response: " + response, e);
+    }
+  }
+
   /**
    * Opens a new project based on a tsconfig.json file. The compiler will analyze all files in the
    * project.
@@ -497,8 +522,23 @@ private JsonArray mapToArray(Map<String, Path> map) {
    * <p>Only one project should be opened at once.
    */
   public ParsedProject openProject(File tsConfigFile, DependencyInstallationResult deps) {
+    JsonObject request = makeLoadCommand("open-project", tsConfigFile, deps);
+    JsonObject response = talkToParserWrapper(request);
+    try {
+      checkResponseType(response, "project-opened");
+      ParsedProject project = new ParsedProject(tsConfigFile,
+          getFilesFromJsonArray(response.get("ownFiles").getAsJsonArray()),
+          getFilesFromJsonArray(response.get("allFiles").getAsJsonArray()));
+      return project;
+    } catch (IllegalStateException e) {
+      throw new CatastrophicError(
+          "TypeScript parser wrapper sent unexpected response: " + response, e);
+    }
+  }
+
+  private JsonObject makeLoadCommand(String command, File tsConfigFile, DependencyInstallationResult deps) {
     JsonObject request = new JsonObject();
-    request.add("command", new JsonPrimitive("open-project"));
+    request.add("command", new JsonPrimitive(command));
     request.add("tsConfig", new JsonPrimitive(tsConfigFile.getPath()));
     request.add("packageEntryPoints", mapToArray(deps.getPackageEntryPoints()));
     request.add("packageJsonFiles", mapToArray(deps.getPackageJsonFiles()));
@@ -508,19 +548,7 @@ public ParsedProject openProject(File tsConfigFile, DependencyInstallationResult
     request.add("virtualSourceRoot", deps.getVirtualSourceRoot() == null
         ? JsonNull.INSTANCE
         : new JsonPrimitive(deps.getVirtualSourceRoot().toString()));
-    JsonObject response = talkToParserWrapper(request);
-    try {
-      checkResponseType(response, "project-opened");
-      ParsedProject project = new ParsedProject(tsConfigFile);
-      JsonArray filesJson = response.get("files").getAsJsonArray();
-      for (JsonElement elm : filesJson) {
-        project.addSourceFile(new File(elm.getAsString()));
-      }
-      return project;
-    } catch (IllegalStateException e) {
-      throw new CatastrophicError(
-          "TypeScript parser wrapper sent unexpected response: " + response, e);
-    }
+    return request;
   }
 
   /**