Merge pull request #3120 from asger-semmle/js/prefer-typescript-file

Approved by esbena

Author: semmle-qlci

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -565,17 +565,40 @@ private void extractSource() throws IOException {
           extractTypeScript(
               defaultExtractor, filesToExtract, tsconfigFiles, dependencyInstallationResult);
 
+    boolean hasTypeScriptFiles = extractedFiles.size() > 0;
+
     // extract remaining files
     for (Path f : filesToExtract) {
-      if (extractedFiles.add(f)) {
-        FileExtractor extractor = defaultExtractor;
-        if (!fileTypes.isEmpty()) {
-          String extension = FileUtil.extension(f);
-          if (customExtractors.containsKey(extension)) extractor = customExtractors.get(extension);
-        }
-        extract(extractor, f, null);
+      if (extractedFiles.contains(f))
+        continue;
+      if (hasTypeScriptFiles && isFileDerivedFromTypeScriptFile(f, extractedFiles)) {
+        continue;
+      }
+      extractedFiles.add(f);
+      FileExtractor extractor = defaultExtractor;
+      if (!fileTypes.isEmpty()) {
+        String extension = FileUtil.extension(f);
+        if (customExtractors.containsKey(extension)) extractor = customExtractors.get(extension);
+      }
+      extract(extractor, f, null);
+    }
+  }
+
+  /**
+   * Returns true if the given path is likely the output of compiling a TypeScript file
+   * which we have already extracted.
+   */
+  private boolean isFileDerivedFromTypeScriptFile(Path path, Set<Path> extractedFiles) {
+    String name = path.getFileName().toString();
+    if (!name.endsWith(".js"))
+      return false;
+    String stem = name.substring(0, name.length() - ".js".length());
+    for (String ext : FileType.TYPESCRIPT.getExtensions()) {
+      if (extractedFiles.contains(path.getParent().resolve(stem + ext))) {
+        return true;
       }
     }
+    return false;
   }
 
   /** Returns true if yarn is installed, otherwise prints a warning and returns false. */
@@ -613,7 +636,7 @@ private static Path tryResolveWithExtensions(Path dir, String stem, Iterable<Str
     }
     return null;
   }
-  
+
   /**
    * Returns an existing file named <code>dir/stem.ext</code> where <code>ext</code> is any TypeScript or JavaScript extension,
    * or <code>null</code> if no such file exists.
@@ -623,7 +646,7 @@ private static Path tryResolveTypeScriptOrJavaScriptFile(Path dir, String stem)
     if (resolved != null) return resolved;
     return tryResolveWithExtensions(dir, stem, FileType.JS.getExtensions());
   }
-  
+
   /**
    * Gets a child of a JSON object as a string, or <code>null</code>.
    */
@@ -658,7 +681,7 @@ protected DependencyInstallationResult installDependencies(Set<Path> filesToExtr
     if (!verifyYarnInstallation()) {
       return DependencyInstallationResult.empty;
     }
-    
+
     final Path sourceRoot = Paths.get(".").toAbsolutePath();
     final Path virtualSourceRoot = Paths.get(EnvironmentVariables.getScratchDir()).toAbsolutePath();
 
@@ -746,7 +769,7 @@ protected DependencyInstallationResult installDependencies(Set<Path> filesToExtr
     for (Path file : packageJsonFiles.keySet()) {
       Path relativePath = sourceRoot.relativize(file);
       Path virtualFile = virtualSourceRoot.resolve(relativePath);
-      
+
       try {
         Files.createDirectories(virtualFile.getParent());
         try (Writer writer = Files.newBufferedWriter(virtualFile)) {
@@ -794,7 +817,7 @@ protected DependencyInstallationResult installDependencies(Set<Path> filesToExtr
    */
   private Path guessPackageMainFile(Path packageJsonFile, JsonObject packageJson, Iterable<String> extensions) {
     Path packageDir = packageJsonFile.getParent();
-    
+
     // Try <package_dir>/index.ts.
     Path resolved = tryResolveWithExtensions(packageDir, "index", extensions);
     if (resolved != null) {

javascript/extractor/src/com/semmle/js/extractor/Main.java

@@ -1,5 +1,13 @@
 package com.semmle.js.extractor;
 
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Pattern;
+
 import com.semmle.js.extractor.ExtractorConfig.HTMLHandling;
 import com.semmle.js.extractor.ExtractorConfig.Platform;
 import com.semmle.js.extractor.ExtractorConfig.SourceType;
@@ -23,13 +31,6 @@
 import com.semmle.util.process.ArgsParser;
 import com.semmle.util.process.ArgsParser.FileMode;
 import com.semmle.util.trap.TrapWriter;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.regex.Pattern;
 
 /** The main entry point of the JavaScript extractor. */
 public class Main {
@@ -190,10 +191,29 @@ public void run(String[] args) {
 
     // Extract files that were not part of a project.
     for (File f : files) {
+      if (isFileDerivedFromTypeScriptFile(f))
+        continue;
       ensureFileIsExtracted(f, ap);
     }
   }
 
+  /**
+   * Returns true if the given path is likely the output of compiling a TypeScript file
+   * which we have already extracted.
+   */
+  private boolean isFileDerivedFromTypeScriptFile(File path) {
+    String name = path.getName();
+    if (!name.endsWith(".js"))
+      return false;
+    String stem = name.substring(0, name.length() - ".js".length());
+    for (String ext : FileType.TYPESCRIPT.getExtensions()) {
+      if (new File(path.getParent(), stem + ext).exists()) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   private void extractTypeTable(File fileHandle, TypeTable table) {
     TrapWriter trapWriter =
         extractorOutputConfig

javascript/ql/test/library-tests/Constants/Constants.expected

@@ -1,3 +1,5 @@
+| tst2.ts:1:13:1:21 | <number>1 |
+| tst2.ts:1:21:1:21 | 1 |
 | tst.js:1:1:1:3 | "a" |
 | tst.js:2:1:2:3 | "b" |
 | tst.js:2:1:2:9 | "b" + "c" |
@@ -59,5 +61,3 @@
 | tst.js:48:1:48:7 | x.p = 1 |
 | tst.js:48:7:48:7 | 1 |
 | tst.js:49:6:49:6 | 1 |
-| tst.ts:1:13:1:21 | <number>1 |
-| tst.ts:1:21:1:21 | 1 |

…t/ql/test/library-tests/Constants/tst.ts …/ql/test/library-tests/Constants/tst2.tsjavascript/ql/test/library-tests/Constants/tst.ts renamed to javascript/ql/test/library-tests/Constants/tst2.ts javascript/ql/test/library-tests/Constants/tst.ts renamed to javascript/ql/test/library-tests/Constants/tst2.ts

@@ -35,6 +35,36 @@
 | sources.js:11:14:11:16 | key | sources.js:11:14:11:16 | key |
 | sources.js:11:23:11:27 | array | sources.js:11:23:11:27 | array |
 | sources.js:11:32:11:34 | key | sources.js:11:32:11:34 | key |
+| tst2.ts:1:18:1:18 | A | tst2.ts:1:18:1:18 | A |
+| tst2.ts:2:14:2:14 | x | tst2.ts:2:14:2:14 | x |
+| tst2.ts:2:14:2:19 | x = 42 | tst2.ts:2:14:2:19 | x = 42 |
+| tst2.ts:2:18:2:19 | 42 | tst2.ts:2:18:2:19 | 42 |
+| tst2.ts:3:3:3:6 | setX | tst2.ts:3:3:3:6 | setX |
+| tst2.ts:3:3:3:8 | setX() | tst2.ts:3:3:3:8 | setX() |
+| tst2.ts:4:3:4:3 | x | tst2.ts:4:3:4:3 | x |
+| tst2.ts:7:10:7:13 | setX | tst2.ts:7:10:7:13 | setX |
+| tst2.ts:8:3:8:3 | A | tst2.ts:8:3:8:3 | A |
+| tst2.ts:8:3:8:5 | A.x | tst2.ts:8:3:8:5 | A.x |
+| tst2.ts:8:3:8:10 | A.x = 23 | tst2.ts:8:3:8:10 | A.x = 23 |
+| tst2.ts:8:5:8:5 | x | tst2.ts:8:5:8:5 | x |
+| tst2.ts:8:9:8:10 | 23 | tst2.ts:8:9:8:10 | 23 |
+| tst2.ts:11:5:11:7 | nd2 | tst2.ts:11:5:11:7 | nd2 |
+| tst2.ts:11:5:11:23 | nd2 = A.x as number | tst2.ts:11:5:11:23 | nd2 = A.x as number |
+| tst2.ts:11:11:11:11 | A | tst2.ts:11:11:11:11 | A |
+| tst2.ts:11:11:11:13 | A.x | tst2.ts:11:11:11:13 | A.x |
+| tst2.ts:11:11:11:23 | A.x as number | tst2.ts:11:11:11:23 | A.x as number |
+| tst2.ts:11:13:11:13 | x | tst2.ts:11:13:11:13 | x |
+| tst2.ts:13:7:13:16 | StringList | tst2.ts:13:7:13:16 | StringList |
+| tst2.ts:13:26:13:29 | List | tst2.ts:13:26:13:29 | List |
+| tst2.ts:13:26:13:37 | List<string> | tst2.ts:13:26:13:37 | List<string> |
+| tst2.ts:13:39:13:38 | (...arg ... rgs); } | tst2.ts:13:39:13:38 | (...arg ... rgs); } |
+| tst2.ts:13:39:13:38 | ...args | tst2.ts:13:39:13:38 | ...args |
+| tst2.ts:13:39:13:38 | args | tst2.ts:13:39:13:38 | args |
+| tst2.ts:13:39:13:38 | args | tst2.ts:13:39:13:38 | args |
+| tst2.ts:13:39:13:38 | args | tst2.ts:13:39:13:38 | args |
+| tst2.ts:13:39:13:38 | constructor | tst2.ts:13:39:13:38 | constructor |
+| tst2.ts:13:39:13:38 | super | tst2.ts:13:39:13:38 | super |
+| tst2.ts:13:39:13:38 | super(...args) | tst2.ts:13:39:13:38 | super(...args) |
 | tst.js:1:10:1:11 | fs | tst.js:1:10:1:11 | fs |
 | tst.js:1:10:1:11 | fs | tst.js:1:10:1:11 | fs |
 | tst.js:1:10:1:11 | fs | tst.js:1:10:1:11 | fs |
@@ -289,33 +319,3 @@
 | tst.js:115:1:115:12 | Array.call() | tst.js:115:1:115:12 | Array.call() |
 | tst.js:115:1:115:12 | reflective call | tst.js:115:1:115:12 | Array.call() |
 | tst.js:115:7:115:10 | call | tst.js:115:7:115:10 | call |
-| tst.ts:1:18:1:18 | A | tst.ts:1:18:1:18 | A |
-| tst.ts:2:14:2:14 | x | tst.ts:2:14:2:14 | x |
-| tst.ts:2:14:2:19 | x = 42 | tst.ts:2:14:2:19 | x = 42 |
-| tst.ts:2:18:2:19 | 42 | tst.ts:2:18:2:19 | 42 |
-| tst.ts:3:3:3:6 | setX | tst.ts:3:3:3:6 | setX |
-| tst.ts:3:3:3:8 | setX() | tst.ts:3:3:3:8 | setX() |
-| tst.ts:4:3:4:3 | x | tst.ts:4:3:4:3 | x |
-| tst.ts:7:10:7:13 | setX | tst.ts:7:10:7:13 | setX |
-| tst.ts:8:3:8:3 | A | tst.ts:8:3:8:3 | A |
-| tst.ts:8:3:8:5 | A.x | tst.ts:8:3:8:5 | A.x |
-| tst.ts:8:3:8:10 | A.x = 23 | tst.ts:8:3:8:10 | A.x = 23 |
-| tst.ts:8:5:8:5 | x | tst.ts:8:5:8:5 | x |
-| tst.ts:8:9:8:10 | 23 | tst.ts:8:9:8:10 | 23 |
-| tst.ts:11:5:11:7 | nd2 | tst.ts:11:5:11:7 | nd2 |
-| tst.ts:11:5:11:23 | nd2 = A.x as number | tst.ts:11:5:11:23 | nd2 = A.x as number |
-| tst.ts:11:11:11:11 | A | tst.ts:11:11:11:11 | A |
-| tst.ts:11:11:11:13 | A.x | tst.ts:11:11:11:13 | A.x |
-| tst.ts:11:11:11:23 | A.x as number | tst.ts:11:11:11:23 | A.x as number |
-| tst.ts:11:13:11:13 | x | tst.ts:11:13:11:13 | x |
-| tst.ts:13:7:13:16 | StringList | tst.ts:13:7:13:16 | StringList |
-| tst.ts:13:26:13:29 | List | tst.ts:13:26:13:29 | List |
-| tst.ts:13:26:13:37 | List<string> | tst.ts:13:26:13:37 | List<string> |
-| tst.ts:13:39:13:38 | (...arg ... rgs); } | tst.ts:13:39:13:38 | (...arg ... rgs); } |
-| tst.ts:13:39:13:38 | ...args | tst.ts:13:39:13:38 | ...args |
-| tst.ts:13:39:13:38 | args | tst.ts:13:39:13:38 | args |
-| tst.ts:13:39:13:38 | args | tst.ts:13:39:13:38 | args |
-| tst.ts:13:39:13:38 | args | tst.ts:13:39:13:38 | args |
-| tst.ts:13:39:13:38 | constructor | tst.ts:13:39:13:38 | constructor |
-| tst.ts:13:39:13:38 | super | tst.ts:13:39:13:38 | super |
-| tst.ts:13:39:13:38 | super(...args) | tst.ts:13:39:13:38 | super(...args) |

javascript/ql/test/library-tests/DataFlow/enclosingExpr.expected

@@ -12,6 +12,20 @@
 | sources.js:10:12:10:14 | key | sources.js:10:28:10:30 | key |
 | sources.js:11:12:11:18 | key | sources.js:11:32:11:34 | key |
 | sources.js:11:14:11:16 | key | sources.js:11:12:11:18 | key |
+| tst2.ts:1:1:1:1 | A | tst2.ts:1:18:1:18 | A |
+| tst2.ts:1:1:1:1 | A | tst2.ts:7:1:7:0 | A |
+| tst2.ts:1:8:5:1 | A | tst2.ts:7:1:7:0 | A |
+| tst2.ts:1:8:5:1 | A | tst2.ts:11:11:11:11 | A |
+| tst2.ts:1:8:5:1 | namespa ... lysed\\n} | tst2.ts:1:8:5:1 | A |
+| tst2.ts:2:14:2:19 | x | tst2.ts:4:3:4:3 | x |
+| tst2.ts:2:18:2:19 | 42 | tst2.ts:2:14:2:19 | x |
+| tst2.ts:7:1:7:0 | A | tst2.ts:8:3:8:3 | A |
+| tst2.ts:7:1:9:1 | functio ... = 23;\\n} | tst2.ts:7:10:7:13 | setX |
+| tst2.ts:7:10:7:13 | setX | tst2.ts:3:3:3:6 | setX |
+| tst2.ts:8:9:8:10 | 23 | tst2.ts:8:3:8:10 | A.x = 23 |
+| tst2.ts:11:11:11:13 | A.x | tst2.ts:11:11:11:23 | A.x as number |
+| tst2.ts:13:26:13:29 | List | tst2.ts:13:26:13:37 | List<string> |
+| tst2.ts:13:39:13:38 | args | tst2.ts:13:39:13:38 | args |
 | tst.js:1:1:1:1 | x | tst.js:28:2:28:1 | x |
 | tst.js:1:1:1:1 | x | tst.js:32:1:32:0 | x |
 | tst.js:1:10:1:11 | fs | tst.js:1:10:1:11 | fs |
@@ -147,17 +161,3 @@
 | tst.js:111:29:111:31 | o2c | tst.js:111:6:111:38 | v2c |
 | tst.js:111:36:111:38 | o2d | tst.js:111:6:111:32 | [v2a, v ...  = o2c] |
 | tst.js:115:1:115:12 | reflective call | tst.js:115:1:115:12 | Array.call() |
-| tst.ts:1:1:1:1 | A | tst.ts:1:18:1:18 | A |
-| tst.ts:1:1:1:1 | A | tst.ts:7:1:7:0 | A |
-| tst.ts:1:8:5:1 | A | tst.ts:7:1:7:0 | A |
-| tst.ts:1:8:5:1 | A | tst.ts:11:11:11:11 | A |
-| tst.ts:1:8:5:1 | namespa ... lysed\\n} | tst.ts:1:8:5:1 | A |
-| tst.ts:2:14:2:19 | x | tst.ts:4:3:4:3 | x |
-| tst.ts:2:18:2:19 | 42 | tst.ts:2:14:2:19 | x |
-| tst.ts:7:1:7:0 | A | tst.ts:8:3:8:3 | A |
-| tst.ts:7:1:9:1 | functio ... = 23;\\n} | tst.ts:7:10:7:13 | setX |
-| tst.ts:7:10:7:13 | setX | tst.ts:3:3:3:6 | setX |
-| tst.ts:8:9:8:10 | 23 | tst.ts:8:3:8:10 | A.x = 23 |
-| tst.ts:11:11:11:13 | A.x | tst.ts:11:11:11:23 | A.x as number |
-| tst.ts:13:26:13:29 | List | tst.ts:13:26:13:37 | List<string> |
-| tst.ts:13:39:13:38 | args | tst.ts:13:39:13:38 | args |

javascript/ql/test/library-tests/DataFlow/flowStep.expected

@@ -1,6 +1,8 @@
 | eval.js:2:11:2:12 | 42 | 42 |
 | sources.js:4:12:4:13 | 19 | 19 |
 | sources.js:5:4:5:5 | 23 | 23 |
+| tst2.ts:2:18:2:19 | 42 | 42 |
+| tst2.ts:8:9:8:10 | 23 | 23 |
 | tst.js:3:9:3:10 | 42 | 42 |
 | tst.js:51:11:51:12 | 42 | 42 |
 | tst.js:65:9:65:10 | 42 | 42 |
@@ -11,5 +13,3 @@
 | tst.js:103:6:103:7 | 19 | 19 |
 | tst.js:103:10:103:11 | 23 | 23 |
 | tst.js:103:14:103:14 | 0 | 0 |
-| tst.ts:2:18:2:19 | 42 | 42 |
-| tst.ts:8:9:8:10 | 23 | 23 |

javascript/ql/test/library-tests/DataFlow/getIntValue.expected

@@ -13,6 +13,18 @@
 | sources.js:10:12:10:14 | key | heap |
 | sources.js:11:12:11:18 | key | heap |
 | sources.js:11:14:11:16 | key | heap |
+| tst2.ts:2:14:2:19 | x | namespace |
+| tst2.ts:3:3:3:8 | exceptional return of setX() | call |
+| tst2.ts:3:3:3:8 | setX() | call |
+| tst2.ts:7:1:9:1 | exceptional return of function setX | call |
+| tst2.ts:8:3:8:5 | A.x | heap |
+| tst2.ts:11:11:11:13 | A.x | heap |
+| tst2.ts:13:26:13:29 | List | global |
+| tst2.ts:13:39:13:38 | args | call |
+| tst2.ts:13:39:13:38 | exceptional return of default constructor of class StringList | call |
+| tst2.ts:13:39:13:38 | exceptional return of super(...args) | call |
+| tst2.ts:13:39:13:38 | super | call |
+| tst2.ts:13:39:13:38 | super(...args) | call |
 | tst.js:1:10:1:11 | fs | import |
 | tst.js:16:1:20:9 | exceptional return of (functi ... ("arg") | call |
 | tst.js:16:2:20:1 | exceptional return of function f | call |
@@ -92,15 +104,3 @@
 | tst.js:115:1:115:10 | Array.call | heap |
 | tst.js:115:1:115:12 | Array.call() | call |
 | tst.js:115:1:115:12 | exceptional return of Array.call() | call |
-| tst.ts:2:14:2:19 | x | namespace |
-| tst.ts:3:3:3:8 | exceptional return of setX() | call |
-| tst.ts:3:3:3:8 | setX() | call |
-| tst.ts:7:1:9:1 | exceptional return of function setX | call |
-| tst.ts:8:3:8:5 | A.x | heap |
-| tst.ts:11:11:11:13 | A.x | heap |
-| tst.ts:13:26:13:29 | List | global |
-| tst.ts:13:39:13:38 | args | call |
-| tst.ts:13:39:13:38 | exceptional return of default constructor of class StringList | call |
-| tst.ts:13:39:13:38 | exceptional return of super(...args) | call |
-| tst.ts:13:39:13:38 | super | call |
-| tst.ts:13:39:13:38 | super(...args) | call |

javascript/ql/test/library-tests/DataFlow/incomplete.expected

@@ -1,8 +1,8 @@
 | sources.js:1:6:1:6 | x |
 | sources.js:3:11:3:11 | x |
 | sources.js:9:14:9:18 | array |
+| tst2.ts:13:39:13:38 | args |
 | tst.js:16:13:16:13 | a |
 | tst.js:32:12:32:12 | b |
 | tst.js:87:11:87:24 | { p: x, ...o } |
 | tst.js:98:11:98:24 | [ x, ...rest ] |
-| tst.ts:13:39:13:38 | args |

javascript/ql/test/library-tests/DataFlow/parameters.expected

@@ -19,6 +19,18 @@
 | sources.js:10:12:10:14 | key |
 | sources.js:11:12:11:18 | { key } |
 | sources.js:11:14:11:16 | key |
+| tst2.ts:1:1:1:0 | this |
+| tst2.ts:3:3:3:8 | setX() |
+| tst2.ts:7:1:7:0 | this |
+| tst2.ts:7:1:9:1 | functio ... = 23;\\n} |
+| tst2.ts:8:3:8:5 | A.x |
+| tst2.ts:11:11:11:13 | A.x |
+| tst2.ts:13:1:13:40 | class S ... ing> {} |
+| tst2.ts:13:26:13:29 | List |
+| tst2.ts:13:39:13:38 | (...arg ... rgs); } |
+| tst2.ts:13:39:13:38 | args |
+| tst2.ts:13:39:13:38 | super(...args) |
+| tst2.ts:13:39:13:38 | this |
 | tst.js:1:1:1:0 | this |
 | tst.js:1:1:1:24 | import  ... m 'fs'; |
 | tst.js:1:10:1:11 | fs |
@@ -105,15 +117,3 @@
 | tst.js:115:1:115:10 | Array.call |
 | tst.js:115:1:115:12 | Array.call() |
 | tst.js:115:1:115:12 | reflective call |
-| tst.ts:1:1:1:0 | this |
-| tst.ts:3:3:3:8 | setX() |
-| tst.ts:7:1:7:0 | this |
-| tst.ts:7:1:9:1 | functio ... = 23;\\n} |
-| tst.ts:8:3:8:5 | A.x |
-| tst.ts:11:11:11:13 | A.x |
-| tst.ts:13:1:13:40 | class S ... ing> {} |
-| tst.ts:13:26:13:29 | List |
-| tst.ts:13:39:13:38 | (...arg ... rgs); } |
-| tst.ts:13:39:13:38 | args |
-| tst.ts:13:39:13:38 | super(...args) |
-| tst.ts:13:39:13:38 | this |