Merge pull request #2432 from asger-semmle/install-typescript-deps

Approved by max-schaefer

Author: semmle-qlci

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -21,8 +21,10 @@
 import com.semmle.util.process.Env;
 import com.semmle.util.projectstructure.ProjectLayout;
 import com.semmle.util.trap.TrapWriter;
+import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.Reader;
 import java.lang.ProcessBuilder.Redirect;
 import java.net.URI;
@@ -195,6 +197,11 @@ public class AutoBuild {
   private final String defaultEncoding;
   private ExecutorService threadPool;
   private volatile boolean seenCode = false;
+  private boolean installDependencies = false;
+  private int installDependenciesTimeout;
+
+  /** The default timeout when running <code>yarn</code>, in milliseconds. */
+  public static final int INSTALL_DEPENDENCIES_DEFAULT_TIMEOUT = 10 * 60 * 1000; // 10 minutes
 
   public AutoBuild() {
     this.LGTM_SRC = toRealPath(getPathFromEnvVar("LGTM_SRC"));
@@ -204,6 +211,11 @@ public AutoBuild() {
     this.typeScriptMode =
         getEnumFromEnvVar("LGTM_INDEX_TYPESCRIPT", TypeScriptMode.class, TypeScriptMode.FULL);
     this.defaultEncoding = getEnvVar("LGTM_INDEX_DEFAULT_ENCODING");
+    this.installDependencies = Boolean.valueOf(getEnvVar("LGTM_INDEX_TYPESCRIPT_INSTALL_DEPS"));
+    this.installDependenciesTimeout =
+        Env.systemEnv()
+            .getInt(
+                "LGTM_INDEX_TYPESCRIPT_INSTALL_DEPS_TIMEOUT", INSTALL_DEPENDENCIES_DEFAULT_TIMEOUT);
     setupFileTypes();
     setupXmlMode();
     setupMatchers();
@@ -533,6 +545,10 @@ private void extractSource() throws IOException {
     List<Path> tsconfigFiles = new ArrayList<>();
     findFilesToExtract(defaultExtractor, filesToExtract, tsconfigFiles);
 
+    if (!tsconfigFiles.isEmpty() && this.installDependencies) {
+      this.installDependencies(filesToExtract);
+    }
+
     // extract TypeScript projects and files
     Set<Path> extractedFiles = extractTypeScript(defaultExtractor, filesToExtract, tsconfigFiles);
 
@@ -549,6 +565,61 @@ private void extractSource() throws IOException {
     }
   }
 
+  /** Returns true if yarn is installed, otherwise prints a warning and returns false. */
+  private boolean verifyYarnInstallation() {
+    ProcessBuilder pb = new ProcessBuilder(Arrays.asList("yarn", "-v"));
+    try {
+      Process process = pb.start();
+      boolean completed = process.waitFor(this.installDependenciesTimeout, TimeUnit.MILLISECONDS);
+      if (!completed) {
+        System.err.println("Yarn could not be launched. Timeout during 'yarn -v'.");
+        return false;
+      }
+      BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
+      String version = reader.readLine();
+      System.out.println("Found yarn version: " + version);
+      return true;
+    } catch (IOException | InterruptedException ex) {
+      System.err.println(
+          "Yarn not found. Please put 'yarn' on the PATH for automatic dependency installation.");
+      Exceptions.ignore(ex, "Continue without dependency installation");
+      return false;
+    }
+  }
+
+  protected void installDependencies(Set<Path> filesToExtract) {
+    if (!verifyYarnInstallation()) {
+      return;
+    }
+    for (Path file : filesToExtract) {
+      if (file.getFileName().toString().equals("package.json")) {
+        System.out.println("Installing dependencies from " + file);
+        ProcessBuilder pb =
+            new ProcessBuilder(
+                Arrays.asList(
+                    "yarn",
+                    "install",
+                    "--verbose",
+                    "--non-interactive",
+                    "--ignore-scripts",
+                    "--ignore-platform",
+                    "--ignore-engines",
+                    "--ignore-optional",
+                    "--no-default-rc",
+                    "--no-bin-links",
+                    "--pure-lockfile"));
+        pb.directory(file.getParent().toFile());
+        pb.redirectOutput(Redirect.INHERIT);
+        pb.redirectError(Redirect.INHERIT);
+        try {
+          pb.start().waitFor(this.installDependenciesTimeout, TimeUnit.MILLISECONDS);
+        } catch (IOException | InterruptedException ex) {
+          throw new ResourceError("Could not install dependencies from " + file, ex);
+        }
+      }
+    }
+  }
+
   private ExtractorConfig mkExtractorConfig() {
     ExtractorConfig config = new ExtractorConfig(true);
     config = config.withSourceType(getSourceType());

javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java

@@ -128,6 +128,11 @@ public void extractTypeScriptFiles(
           }
         }
 
+        @Override
+        protected void installDependencies(Set<Path> filesToExtract) {
+          // never install dependencies during testing
+        }
+
         @Override
         protected void extractXml() throws IOException {
           Files.walkFileTree(