JS: Make this work in qltest

Author: asgerf

javascript/extractor/lib/typescript/src/common.ts

@@ -48,7 +48,7 @@ export class Project {
   public load(): void {
     const { config, host } = this;
     this.program = ts.createProgram(config.fileNames, config.options, host);
-    this.typeTable.setProgram(this.program);
+    this.typeTable.setProgram(this.program, this.virtualSourceRoot);
   }
 
   /**

javascript/extractor/lib/typescript/src/main.ts

@@ -416,14 +416,21 @@ function loadTsConfig(command: LoadCommand): LoadedConfig {
         useCaseSensitiveFileNames: true,
         readDirectory: (rootDir, extensions, excludes?, includes?, depth?) => {
             // Perform the glob matching in both real and virtual source roots.
-            let originalResults = ts.sys.readDirectory(rootDir, extensions, excludes, includes, depth)
+            let exclusions = excludes == null ? [] : [...excludes];
+            if (virtualSourceRoot.virtualSourceRoot != null) {
+                // qltest puts the virtual source root inside the real source root (.testproj).
+                // Make sure we don't find files inside the virtual source root in this pass.
+                exclusions.push(virtualSourceRoot.virtualSourceRoot);
+            }
+            let originalResults = ts.sys.readDirectory(rootDir, extensions, exclusions, includes, depth)
             let virtualDir = virtualSourceRoot.toVirtualPath(rootDir);
             if (virtualDir == null) {
                 return originalResults;
             }
             // Make sure glob matching does not to discover anything in node_modules.
-            let virtualExcludes = [ ...(excludes || []), '**/node_modules/**/*' ];
-            let virtualResults = ts.sys.readDirectory(virtualDir, extensions, virtualExcludes, includes, depth)
+            let virtualExclusions = excludes == null ? [] : [...excludes];
+            virtualExclusions.push('**/node_modules/**/*');
+            let virtualResults = ts.sys.readDirectory(virtualDir, extensions, virtualExclusions, includes, depth)
             return [ ...originalResults, ...virtualResults ];
         },
         fileExists: (path: string) => {

javascript/extractor/lib/typescript/src/type_table.ts

@@ -1,4 +1,5 @@
 import * as ts from "./typescript";
+import { VirtualSourceRoot } from "./virtual_source_root";
 
 interface AugmentedSymbol extends ts.Symbol {
   parent?: AugmentedSymbol;
@@ -379,12 +380,15 @@ export class TypeTable {
    */
   public restrictedExpansion = false;
 
+  private virtualSourceRoot: VirtualSourceRoot;
+
   /**
    * Called when a new compiler instance has started.
    */
-  public setProgram(program: ts.Program) {
+  public setProgram(program: ts.Program, virtualSourceRoot: VirtualSourceRoot) {
     this.typeChecker = program.getTypeChecker();
     this.arbitraryAstNode = program.getSourceFiles()[0];
+    this.virtualSourceRoot = virtualSourceRoot;
   }
 
   /**
@@ -703,14 +707,21 @@ export class TypeTable {
   private getSymbolString(symbol: AugmentedSymbol): string {
     let parent = symbol.parent;
     if (parent == null || parent.escapedName === ts.InternalSymbolName.Global) {
-      return "root;" + this.getSymbolDeclarationString(symbol) + ";;" + symbol.name;
+      return "root;" + this.getSymbolDeclarationString(symbol) + ";;" + this.rewriteSymbolName(symbol);
     } else if (parent.exports != null && parent.exports.get(symbol.escapedName) === symbol) {
-      return "member;;" + this.getSymbolId(parent) + ";" + symbol.name;
+      return "member;;" + this.getSymbolId(parent) + ";" + this.rewriteSymbolName(symbol);
     } else {
-      return "other;" + this.getSymbolDeclarationString(symbol) + ";" + this.getSymbolId(parent) + ";" + symbol.name;
+      return "other;" + this.getSymbolDeclarationString(symbol) + ";" + this.getSymbolId(parent) + ";" + this.rewriteSymbolName(symbol);
     }
   }
 
+  private rewriteSymbolName(symbol: AugmentedSymbol) {
+    let { virtualSourceRoot, sourceRoot } = this.virtualSourceRoot;
+    let { name } = symbol;
+    if (virtualSourceRoot == null || sourceRoot == null) return name;
+    return name.replace(virtualSourceRoot, sourceRoot);
+  }
+
   /**
    * Gets a string that distinguishes the given symbol from symbols with different
    * lexical roots, or an empty string if the symbol is not a lexical root.

javascript/extractor/lib/typescript/src/virtual_source_root.ts

@@ -7,13 +7,13 @@ import * as ts from "./typescript";
  */
 export class VirtualSourceRoot {
   constructor(
-    private sourceRoot: string | null,
+    public sourceRoot: string | null,
 
     /**
      * Directory whose folder structure mirrors the real source root, but with `node_modules` installed,
      * or undefined if no virtual source root exists.
      */
-    private virtualSourceRoot: string | null,
+    public virtualSourceRoot: string | null,
   ) {}
 
   private static translate(oldRoot: string, newRoot: string, path: string) {
@@ -25,9 +25,17 @@ export class VirtualSourceRoot {
 
   /**
    * Maps a path under the real source root to the corresponding path in the virtual source root.
+   *
+   * Returns `null` for paths already in the virtual source root.
    */
   public toVirtualPath(path: string) {
-    return VirtualSourceRoot.translate(this.sourceRoot, this.virtualSourceRoot, path);
+    let { virtualSourceRoot } = this;
+    if (path.startsWith(virtualSourceRoot)) {
+      // 'qltest' creates a virtual source root inside the real source root.
+      // Make sure such files don't appear to be inside the real source root.
+      return null;
+    }
+    return VirtualSourceRoot.translate(this.sourceRoot, virtualSourceRoot, path);
   }
 
   /**

javascript/extractor/src/com/semmle/js/extractor/Main.java

@@ -2,6 +2,8 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -31,6 +33,8 @@
 import com.semmle.util.language.LegacyLanguage;
 import com.semmle.util.process.ArgsParser;
 import com.semmle.util.process.ArgsParser.FileMode;
+import com.semmle.util.process.Env;
+import com.semmle.util.process.Env.Var;
 import com.semmle.util.trap.TrapWriter;
 
 /** The main entry point of the JavaScript extractor. */
@@ -134,12 +138,6 @@ public void run(String[] args) {
       return;
     }
 
-    TypeScriptParser tsParser = extractorState.getTypeScriptParser();
-    tsParser.setTypescriptRam(extractorConfig.getTypeScriptRam());
-    if (containsTypeScriptFiles()) {
-      tsParser.verifyInstallation(!ap.has(P_QUIET));
-    }
-
     // Sort files for determinism
     projectFiles = projectFiles.stream()
           .sorted(AutoBuild.FILE_ORDERING)
@@ -149,16 +147,30 @@ public void run(String[] args) {
         .sorted(AutoBuild.FILE_ORDERING)
         .collect(Collectors.toCollection(() -> new LinkedHashSet<>()));
 
+    // Extract HTML files first, as they may contain embedded TypeScript code
+    for (File file : files) {
+      if (FileType.forFile(file, extractorConfig) == FileType.HTML) {
+        ensureFileIsExtracted(file, ap);
+      }
+    }
+    
+    TypeScriptParser tsParser = extractorState.getTypeScriptParser();
+    tsParser.setTypescriptRam(extractorConfig.getTypeScriptRam());
+    if (containsTypeScriptFiles()) {
+      tsParser.verifyInstallation(!ap.has(P_QUIET));
+    }
+
     for (File projectFile : projectFiles) {
 
       long start = verboseLogStartTimer(ap, "Opening project " + projectFile);
-      ParsedProject project = tsParser.openProject(projectFile, DependencyInstallationResult.empty, VirtualSourceRoot.none);
+      ParsedProject project = tsParser.openProject(projectFile, DependencyInstallationResult.empty, extractorConfig.getVirtualSourceRoot());
       verboseLogEndTimer(ap, start);
       // Extract all files belonging to this project which are also matched
       // by our include/exclude filters.
       List<File> filesToExtract = new ArrayList<>();
       for (File sourceFile : project.getOwnFiles()) {
-        if (files.contains(normalizeFile(sourceFile))
+        File normalizedFile = normalizeFile(sourceFile);
+        if ((files.contains(normalizedFile) || extractorState.getSnippets().containsKey(normalizedFile.toPath()))
             && !extractedFiles.contains(sourceFile.getAbsoluteFile())
             && FileType.TYPESCRIPT.getExtensions().contains(FileUtil.extension(sourceFile))) {
           filesToExtract.add(sourceFile);
@@ -287,10 +299,14 @@ private boolean containsTypeScriptFiles() {
   }
 
   public void collectFiles(ArgsParser ap) {
-    for (File f : ap.getOneOrMoreFiles("files", FileMode.FILE_OR_DIRECTORY_MUST_EXIST))
+    for (File f : getFilesArg(ap))
       collectFiles(f, true);
   }
 
+  private List<File> getFilesArg(ArgsParser ap) {
+    return ap.getOneOrMoreFiles("files", FileMode.FILE_OR_DIRECTORY_MUST_EXIST);
+  }
+
   public void setupMatchers(ArgsParser ap) {
     Set<String> includes = new LinkedHashSet<>();
 
@@ -444,6 +460,21 @@ private static TypeScriptMode getTypeScriptMode(ArgsParser ap) {
     if (ap.has(P_TYPESCRIPT)) return TypeScriptMode.BASIC;
     return TypeScriptMode.NONE;
   }
+  
+  private Path inferSourceRoot(ArgsParser ap) {
+    List<File> files = getFilesArg(ap);
+    Path sourceRoot = files.iterator().next().toPath().toAbsolutePath().getParent();
+    for (File file : files) {
+      Path path = file.toPath().toAbsolutePath().getParent();
+      for (int i = 0; i < sourceRoot.getNameCount(); ++i) {
+        if (!(i < path.getNameCount() && path.getName(i).equals(sourceRoot.getName(i)))) {
+          sourceRoot = sourceRoot.subpath(0, i);
+          break;
+        }
+      }
+    }
+    return sourceRoot;
+  }
 
   private ExtractorConfig parseJSOptions(ArgsParser ap) {
     ExtractorConfig cfg =
@@ -466,6 +497,17 @@ private ExtractorConfig parseJSOptions(ArgsParser ap) {
                     ? UnitParser.parseOpt(ap.getString(P_TYPESCRIPT_RAM), UnitParser.MEGABYTES)
                     : 0);
     if (ap.has(P_DEFAULT_ENCODING)) cfg = cfg.withDefaultEncoding(ap.getString(P_DEFAULT_ENCODING));
+
+    // Make a usable virtual source root mapping.
+    // The concept of source root and scratch directory do not exist in the legacy extractor,
+    // so we construct these based on what we have.
+    String odasaDbDir = Env.systemEnv().getNonEmpty(Var.ODASA_DB);
+    VirtualSourceRoot virtualSourceRoot =
+        odasaDbDir == null
+            ? VirtualSourceRoot.none
+            : new VirtualSourceRoot(inferSourceRoot(ap), Paths.get(odasaDbDir, "working"));
+    cfg = cfg.withVirtualSourceRoot(virtualSourceRoot);
+
     return cfg;
   }
 

javascript/extractor/src/com/semmle/js/extractor/VirtualSourceRoot.java

@@ -43,6 +43,11 @@ private static Path translate(Path oldRoot, Path newRoot, Path file) {
   }
 
   public Path toVirtualFile(Path file) {
+    if (file.startsWith(virtualSourceRoot)) {
+      // 'qltest' creates a virtual source root inside the real source root.
+      // Make sure such files don't appear to be inside the real source root.
+      return null;
+    }
     return translate(sourceRoot, virtualSourceRoot, file);
   }
 

javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/Test.expected

@@ -1,2 +1,18 @@
 classDeclaration
+| test.vue:3:18:5:3 | class M ... er;\\n  } |
 exprType
+| other.ts:1:8:1:16 | Component | typeof default in library-tests/TypeScript/EmbeddedInScript/test.vue |
+| other.ts:1:23:1:34 | "./test.vue" | any |
+| other.ts:3:1:3:15 | new Component() | MyComponent |
+| other.ts:3:5:3:13 | Component | typeof default in library-tests/TypeScript/EmbeddedInScript/test.vue |
+| other.ts:5:17:5:19 | foo | () => void |
+| test.vue:2:15:2:19 | other | typeof library-tests/TypeScript/EmbeddedInScript/other.ts |
+| test.vue:2:26:2:34 | "./other" | any |
+| test.vue:3:24:3:34 | MyComponent | MyComponent |
+| test.vue:4:7:4:7 | x | number |
+symbols
+| other.ts:1:1:6:0 | <toplevel> | library-tests/TypeScript/EmbeddedInScript/other.ts |
+| test.vue:2:3:6:0 | <toplevel> | library-tests/TypeScript/EmbeddedInScript/test.vue |
+importTarget
+| other.ts:1:1:1:35 | import  ... t.vue"; | test.vue:2:3:6:0 | <toplevel> |
+| test.vue:2:3:2:35 | import  ... other"; | other.ts:1:1:6:0 | <toplevel> |

javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/Test.ql

@@ -3,3 +3,11 @@ import javascript
 query ClassDefinition classDeclaration() { any() }
 
 query Type exprType(Expr e) { result = e.getType() }
+
+query predicate symbols(Module mod, CanonicalName name) {
+  ast_node_symbol(mod, name)
+}
+
+query predicate importTarget(Import imprt, Module mod) {
+  imprt.getImportedModule() = mod
+}

javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/other.ts

@@ -0,0 +1,5 @@
+import Component from "./test.vue";
+
+new Component();
+
+export function foo() {};

javascript/ql/test/library-tests/TypeScript/EmbeddedInScript/test.vue

@@ -1,4 +1,5 @@
 <script lang='ts'>
+  import * as other from "./other";
   export default class MyComponent {
       x!: number;
   }