Merge pull request #479 from asger-semmle/typescript-extractor-perf1

Approved by xiemaisi

Author: semmle-qlci

javascript/extractor/lib/typescript/src/main.ts

@@ -67,13 +67,24 @@ interface ResetCommand {
 interface QuitCommand {
     command: "quit";
 }
+interface PrepareFilesCommand {
+    command: "prepare-files";
+    filenames: string[];
+}
 type Command = ParseCommand | OpenProjectCommand | CloseProjectCommand
-    | GetTypeTableCommand | ResetCommand | QuitCommand;
+    | GetTypeTableCommand | ResetCommand | QuitCommand | PrepareFilesCommand;
 
 /** The state to be shared between commands. */
 class State {
     public project: Project = null;
     public typeTable = new TypeTable();
+
+    /** List of files that have been requested. */
+    public pendingFiles: string[] = [];
+    public pendingFileIndex = 0;
+
+    /** Next response to be delivered. */
+    public pendingResponse: string = null;
 }
 let state = new State();
 
@@ -154,16 +165,43 @@ function getSourceCode(filename: string): string {
     return code;
 }
 
-function handleParseCommand(command: ParseCommand) {
-    let filename = String(command.filename);
+function extractFile(filename: string): string {
     let {ast, code} = getAstForFile(filename);
 
     // Get the AST and augment it.
     ast_extractor.augmentAst(ast, code, state.project);
 
-    console.log(stringifyAST(
-        { type: "ast", ast, nodeFlags: ts.NodeFlags, syntaxKinds: ts.SyntaxKind },
-    ));
+    return stringifyAST({
+        type: "ast",
+        ast,
+        nodeFlags: ts.NodeFlags,
+        syntaxKinds: ts.SyntaxKind
+    });
+}
+
+function prepareNextFile() {
+    if (state.pendingResponse != null) return;
+    if (state.pendingFileIndex < state.pendingFiles.length) {
+        let nextFilename = state.pendingFiles[state.pendingFileIndex];
+        state.pendingResponse = extractFile(nextFilename);
+    }
+}
+
+function handleParseCommand(command: ParseCommand) {
+    let filename = command.filename;
+    let expectedFilename = state.pendingFiles[state.pendingFileIndex];
+    if (expectedFilename !== filename) {
+        throw new Error("File requested out of order. Expected '" + expectedFilename + "' but got '" + filename + "'");
+    }
+    ++state.pendingFileIndex;
+    let response = state.pendingResponse || extractFile(command.filename);
+    state.pendingResponse = null;
+    process.stdout.write(response + "\n", () => {
+        // Start working on the next file as soon as the old one is flushed.
+        // Note that if we didn't wait for flushing, this would block the I/O
+        // loop and delay flushing.
+        prepareNextFile();
+    });
 }
 
 /**
@@ -360,6 +398,15 @@ function handleResetCommand(command: ResetCommand) {
     }));
 }
 
+function handlePrepareFilesCommand(command: PrepareFilesCommand) {
+    state.pendingFiles = command.filenames;
+    state.pendingFileIndex = 0;
+    state.pendingResponse = null;
+    process.stdout.write('{"type":"ok"}\n', () => {
+        prepareNextFile();
+    });
+}
+
 function reset() {
     state = new State();
     state.typeTable.restrictedExpansion = getEnvironmentVariable("SEMMLE_TYPESCRIPT_NO_EXPANSION", Boolean, false);
@@ -400,6 +447,9 @@ function runReadLineInterface() {
         case "get-type-table":
             handleGetTypeTableCommand(req);
             break;
+        case "prepare-files":
+            handlePrepareFilesCommand(req);
+            break;
         case "reset":
             handleResetCommand(req);
             break;

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -494,27 +494,40 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th
 			logEndProcess();
 			// Extract all files belonging to this project which are also matched
 			// by our include/exclude filters.
+			List<File> typeScriptFiles = new ArrayList<File>();
 			for (File sourceFile : project.getSourceFiles()) {
 				Path sourcePath = sourceFile.toPath();
 				if (!filesToExtract.contains(normalizePath(sourcePath)))
 					continue;
-				if (extractedFiles.add(sourcePath)) {
-					extract(extractor, sourcePath);
+				if (!extractedFiles.contains(sourcePath)) {
+					typeScriptFiles.add(sourcePath.toFile());
 				}
 			}
+			extractTypeScriptFiles(typeScriptFiles, extractedFiles, extractor);
 			tsParser.closeProject(projectFile);
 		}
 
 		if (!tsconfigFiles.isEmpty()) {
 			// Extract all the types discovered when extracting the ASTs.
 			TypeTable typeTable = tsParser.getTypeTable();
 			extractTypeTable(tsconfigFiles.iterator().next(), typeTable);
+		}
 
-			// The TypeScript compiler instance is no longer needed.
-			tsParser.killProcess();
+		// Extract remaining TypeScript files.
+		List<File> remainingTypeScriptFiles = new ArrayList<File>();
+		for (Path f : filesToExtract) {
+			if (!extractedFiles.contains(f) && FileType.forFileExtension(f.toFile()) == FileType.TYPESCRIPT) {
+				remainingTypeScriptFiles.add(f.toFile());
+			}
+		}
+		if (!remainingTypeScriptFiles.isEmpty()) {
+			extractTypeScriptFiles(remainingTypeScriptFiles, extractedFiles, extractor);
 		}
 
-		// Extract files that were not part of a project.
+		// The TypeScript compiler instance is no longer needed.
+		tsParser.killProcess();
+
+		// Extract non-TypeScript files
 		for (Path f : filesToExtract) {
 			if (extractedFiles.add(f)) {
 				extract(extractor, f);
@@ -530,6 +543,15 @@ public void verifyTypeScriptInstallation() {
 		extractorState.getTypeScriptParser().verifyInstallation(true);
 	}
 
+	public void extractTypeScriptFiles(List<File> files, Set<Path> extractedFiles, FileExtractor extractor) throws IOException {
+		extractorState.getTypeScriptParser().prepareFiles(files);
+		for (File f : files) {
+			Path path = f.toPath();
+			extractedFiles.add(path);
+			extract(extractor, f.toPath());
+		}
+	}
+
 	private Path normalizePath(Path path) {
 		return path.toAbsolutePath().normalize();
 	}

javascript/extractor/src/com/semmle/js/extractor/Main.java

@@ -2,7 +2,9 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.LinkedHashSet;
+import java.util.List;
 import java.util.Set;
 import java.util.regex.Pattern;
 
@@ -140,16 +142,22 @@ public void run(String[] args) {
 			tsParser.verifyInstallation(!ap.has(P_QUIET));
 		}
 		for (File projectFile : projectFiles) {
+
 			long start = verboseLogStartTimer(ap, "Opening project " + projectFile);
 			ParsedProject project = tsParser.openProject(projectFile);
 			verboseLogEndTimer(ap, start);
 			// Extract all files belonging to this project which are also matched
 			// by our include/exclude filters.
+			List<File> filesToExtract = new ArrayList<>();
 			for (File sourceFile : project.getSourceFiles()) {
-				if (files.contains(normalizeFile(sourceFile))) {
-					ensureFileIsExtracted(sourceFile, ap);
+				if (files.contains(normalizeFile(sourceFile)) && !extractedFiles.contains(sourceFile.getAbsoluteFile())) {
+					filesToExtract.add(sourceFile);
 				}
 			}
+			tsParser.prepareFiles(filesToExtract);
+			for (int i = 0; i < filesToExtract.size(); ++i) {
+				ensureFileIsExtracted(filesToExtract.get(i), ap);
+			}
 			// Close the project to free memory. This does not need to be in a `finally` as
 			// the project is not a system resource.
 			tsParser.closeProject(projectFile);
@@ -159,14 +167,27 @@ public void run(String[] args) {
 			// Extract all the types discovered when extracting the ASTs.
 			TypeTable typeTable = tsParser.getTypeTable();
 			extractTypeTable(projectFiles.iterator().next(), typeTable);
+		}
 
-			// The TypeScript compiler instance is no longer needed - free up some memory.
-			if (hasSharedExtractorState) {
-				tsParser.reset(); // This is called from a test runner, so keep the process alive.
-			} else {
-				tsParser.killProcess();
+		List<File> remainingTypescriptFiles = new ArrayList<>();
+		for (File f : files) {
+			if (!extractedFiles.contains(f.getAbsoluteFile()) && FileType.forFileExtension(f) == FileType.TYPESCRIPT) {
+				remainingTypescriptFiles.add(f);
 			}
 		}
+		if (!remainingTypescriptFiles.isEmpty()) {
+			tsParser.prepareFiles(remainingTypescriptFiles);
+			for (File f : remainingTypescriptFiles) {
+				ensureFileIsExtracted(f, ap);
+			}
+		}
+
+		// The TypeScript compiler instance is no longer needed - free up some memory.
+		if (hasSharedExtractorState) {
+			tsParser.reset(); // This is called from a test runner, so keep the process alive.
+		} else {
+			tsParser.killProcess();
+		}
 
 		// Extract files that were not part of a project.
 		for (File f : files) {

javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java

@@ -1,5 +1,6 @@
 package com.semmle.js.extractor.test;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -100,6 +101,13 @@ protected void extract(FileExtractor extractor, Path file) {
 				@Override
 				public void verifyTypeScriptInstallation() {
 				}
+
+				@Override
+				public void extractTypeScriptFiles(List<File> files, Set<Path> extractedFiles, FileExtractor extractor) throws IOException {
+					for (File f : files) {
+						actual.add(f.toString());
+					}
+				}
 			}.run();
 			String expectedString = StringUtil.glue("\n", expected.stream().sorted().toArray());
 			String actualString = StringUtil.glue("\n", actual.stream().sorted().toArray());

javascript/extractor/src/com/semmle/js/parser/TypeScriptParser.java

@@ -292,6 +292,25 @@ public Result parse(File sourceFile, String source) {
 		}
 	}
 
+	/**
+	 * Informs the parser process that the following files are going to be
+	 * requested, in that order.
+	 * <p>
+	 * The parser process uses this list to start work on the next file before it is
+	 * requested.
+	 */
+	public void prepareFiles(List<File> files) {
+		JsonObject request = new JsonObject();
+		request.add("command", new JsonPrimitive("prepare-files"));
+		JsonArray filenames = new JsonArray();
+		for (File file : files) {
+			filenames.add(new JsonPrimitive(file.getAbsolutePath()));
+		}
+		request.add("filenames", filenames);
+		JsonObject response = talkToParserWrapper(request);
+		checkResponseType(response, "ok");
+	}
+
 	/**
 	 * Opens a new project based on a tsconfig.json file. The compiler will analyze
 	 * all files in the project.