Merge pull request #2619 from asger-semmle/ts-monorepo-deps

Approved by erik-krogh, max-schaefer

Author: semmle-qlci

javascript/extractor/lib/typescript/src/common.ts

@@ -1,20 +1,53 @@
 import * as ts from "./typescript";
 import { TypeTable } from "./type_table";
+import * as pathlib from "path";
+import { VirtualSourceRoot } from "./virtual_source_root";
+
+/**
+ * Extracts the package name from the prefix of an import string.
+ */
+const packageNameRex = /^(?:@[\w.-]+[/\\]+)?\w[\w.-]*(?=[/\\]|$)/;
+const extensions = ['.ts', '.tsx', '.d.ts', '.js', '.jsx'];
+
+function getPackageName(importString: string) {
+  let packageNameMatch = packageNameRex.exec(importString);
+  if (packageNameMatch == null) return null;
+  let packageName = packageNameMatch[0];
+  if (packageName.charAt(0) === '@') {
+    packageName = packageName.replace(/[/\\]+/g, '/'); // Normalize slash after the scope.
+  }
+  return packageName;
+}
 
 export class Project {
   public program: ts.Program = null;
+  private host: ts.CompilerHost;
+  private resolutionCache: ts.ModuleResolutionCache;
 
-  constructor(public tsConfig: string, public config: ts.ParsedCommandLine, public typeTable: TypeTable) {}
+  constructor(
+        public tsConfig: string,
+        public config: ts.ParsedCommandLine,
+        public typeTable: TypeTable,
+        public packageEntryPoints: Map<string, string>,
+        public virtualSourceRoot: VirtualSourceRoot) {
+
+    this.resolveModuleNames = this.resolveModuleNames.bind(this);
+
+    this.resolutionCache = ts.createModuleResolutionCache(pathlib.dirname(tsConfig), ts.sys.realpath, config.options);
+    let host = ts.createCompilerHost(config.options, true);
+    host.resolveModuleNames = this.resolveModuleNames;
+    host.trace = undefined; // Disable tracing which would otherwise go to standard out
+    this.host = host;
+  }
 
   public unload(): void {
     this.typeTable.releaseProgram();
     this.program = null;
   }
 
   public load(): void {
-    let host = ts.createCompilerHost(this.config.options, true);
-    host.trace = undefined; // Disable tracing which would otherwise go to standard out
-    this.program = ts.createProgram(this.config.fileNames, this.config.options, host);
+    const { config, host } = this;
+    this.program = ts.createProgram(config.fileNames, config.options, host);
     this.typeTable.setProgram(this.program);
   }
 
@@ -27,4 +60,73 @@ export class Project {
     this.unload();
     this.load();
   }
+
+  /**
+   * Override for module resolution in the TypeScript compiler host.
+   */
+  private resolveModuleNames(
+        moduleNames: string[],
+        containingFile: string,
+        reusedNames: string[],
+        redirectedReference: ts.ResolvedProjectReference,
+        options: ts.CompilerOptions) {
+
+    const { host, resolutionCache } = this;
+    return moduleNames.map((moduleName) => {
+      let redirected = this.redirectModuleName(moduleName, containingFile, options);
+      if (redirected != null) return redirected;
+      return ts.resolveModuleName(moduleName, containingFile, options,  host, resolutionCache).resolvedModule;
+    });
+  }
+
+  /**
+   * Returns the path that the given import string should be redirected to, or null if it should
+   * fall back to standard module resolution.
+   */
+  private redirectModuleName(moduleName: string, containingFile: string, options: ts.CompilerOptions): ts.ResolvedModule {
+    // Get a package name from the leading part of the module name, e.g. '@scope/foo' from '@scope/foo/bar'.
+    let packageName = getPackageName(moduleName);
+    if (packageName == null) return null;
+
+    // Get the overridden location of this package, if one exists.
+    let packageEntryPoint = this.packageEntryPoints.get(packageName);
+    if (packageEntryPoint == null) {
+      // The package is not overridden, but we have established that it begins with a valid package name.
+      // Do a lookup in the virtual source root (where dependencies are installed) by changing the 'containing file'.
+      let virtualContainingFile = this.virtualSourceRoot.toVirtualPath(containingFile);
+      if (virtualContainingFile != null) {
+        return ts.resolveModuleName(moduleName, virtualContainingFile, options, this.host, this.resolutionCache).resolvedModule;
+      }
+      return null;
+    }
+
+    // If the requested module name is exactly the overridden package name,
+    // return the entry point file (it is not necessarily called `index.ts`).
+    if (moduleName === packageName) {
+      return { resolvedFileName: packageEntryPoint, isExternalLibraryImport: true };
+    }
+
+    // Get the suffix after the package name, e.g. the '/bar' in '@scope/foo/bar'.
+    let suffix = moduleName.substring(packageName.length);
+
+    // Resolve the suffix relative to the package directory.
+    let packageDir = pathlib.dirname(packageEntryPoint);
+    let joinedPath = pathlib.join(packageDir, suffix);
+
+    // Add implicit '/index'
+    if (ts.sys.directoryExists(joinedPath)) {
+      joinedPath = pathlib.join(joinedPath, 'index');
+    }
+
+    // Try each recognized extension. We must not return a file whose extension is not
+    // recognized by TypeScript.
+    for (let ext of extensions) {
+      let candidate = joinedPath.endsWith(ext) ? joinedPath : (joinedPath + ext);
+      if (ts.sys.fileExists(candidate)) {
+        return { resolvedFileName: candidate, isExternalLibraryImport: true };
+      }
+    }
+
+    return null;
+  }
 }

javascript/extractor/lib/typescript/src/main.ts

@@ -39,6 +39,7 @@ import * as ast_extractor from "./ast_extractor";
 
 import { Project } from "./common";
 import { TypeTable } from "./type_table";
+import { VirtualSourceRoot } from "./virtual_source_root";
 
 interface ParseCommand {
     command: "parse";
@@ -47,6 +48,9 @@ interface ParseCommand {
 interface OpenProjectCommand {
     command: "open-project";
     tsConfig: string;
+    virtualSourceRoot: string | null;
+    packageEntryPoints: [string, string][];
+    packageJsonFiles: [string, string][];
 }
 interface CloseProjectCommand {
     command: "close-project";
@@ -242,26 +246,93 @@ function parseSingleFile(filename: string): {ast: ts.SourceFile, code: string} {
     return {ast, code};
 }
 
+/**
+ * Matches a path segment referencing a package in a node_modules folder, and extracts
+ * two capture groups: the package name, and the relative path in the package.
+ *
+ * For example `lib/node_modules/@foo/bar/src/index.js` extracts the capture groups [`@foo/bar`, `src/index.js`].
+ */
+const nodeModulesRex = /[/\\]node_modules[/\\]((?:@[\w.-]+[/\\])?\w[\w.-]*)[/\\](.*)/;
+
 function handleOpenProjectCommand(command: OpenProjectCommand) {
     Error.stackTraceLimit = Infinity;
     let tsConfigFilename = String(command.tsConfig);
     let tsConfig = ts.readConfigFile(tsConfigFilename, ts.sys.readFile);
     let basePath = pathlib.dirname(tsConfigFilename);
 
+    let packageEntryPoints = new Map(command.packageEntryPoints);
+    let packageJsonFiles = new Map(command.packageJsonFiles);
+    let virtualSourceRoot = new VirtualSourceRoot(process.cwd(), command.virtualSourceRoot);
+
+    /**
+     * Rewrites path segments of form `node_modules/PACK/suffix` to be relative to
+     * the location of package PACK in the source tree, if it exists.
+     */
+    function redirectNodeModulesPath(path: string) {
+        let nodeModulesMatch = nodeModulesRex.exec(path);
+        if (nodeModulesMatch == null) return null;
+        let packageName = nodeModulesMatch[1];
+        let packageJsonFile = packageJsonFiles.get(packageName);
+        if (packageJsonFile == null) return null;
+        let packageDir = pathlib.dirname(packageJsonFile);
+        let suffix = nodeModulesMatch[2];
+        let finalPath = pathlib.join(packageDir, suffix);
+        if (!ts.sys.fileExists(finalPath)) return null;
+        return finalPath;
+    }
+
+    /**
+     * Create the host passed to the tsconfig.json parser.
+     *
+     * We override its file system access in case there is an "extends"
+     * clause pointing into "./node_modules", which must be redirected to
+     * the location of an installed package or a checked-in package.
+     */
     let parseConfigHost: ts.ParseConfigHost = {
         useCaseSensitiveFileNames: true,
-        readDirectory: ts.sys.readDirectory,
-        fileExists: (path: string) => fs.existsSync(path),
-        readFile: ts.sys.readFile,
+        readDirectory: ts.sys.readDirectory, // No need to override traversal/glob matching
+        fileExists: (path: string) => {
+            return ts.sys.fileExists(path)
+                || virtualSourceRoot.toVirtualPathIfFileExists(path) != null
+                || redirectNodeModulesPath(path) != null;
+        },
+        readFile: (path: string) => {
+            if (!ts.sys.fileExists(path)) {
+                let virtualPath = virtualSourceRoot.toVirtualPathIfFileExists(path);
+                if (virtualPath != null) return ts.sys.readFile(virtualPath);
+                virtualPath = redirectNodeModulesPath(path);
+                if (virtualPath != null) return ts.sys.readFile(virtualPath);
+            }
+            return ts.sys.readFile(path);
+        }
     };
     let config = ts.parseJsonConfigFileContent(tsConfig.config, parseConfigHost, basePath);
-    let project = new Project(tsConfigFilename, config, state.typeTable);
+    let project = new Project(tsConfigFilename, config, state.typeTable, packageEntryPoints, virtualSourceRoot);
     project.load();
 
     state.project = project;
     let program = project.program;
     let typeChecker = program.getTypeChecker();
 
+    let diagnostics = program.getSemanticDiagnostics()
+        .filter(d => d.category === ts.DiagnosticCategory.Error);
+    if (diagnostics.length > 0) {
+        console.warn('TypeScript: reported ' + diagnostics.length + ' semantic errors.');
+    }
+    for (let diagnostic of diagnostics) {
+        let text = diagnostic.messageText;
+        if (text && typeof text !== 'string') {
+            text = text.messageText;
+        }
+        let locationStr = '';
+        let { file } = diagnostic;
+        if (file != null) {
+            let { line, character } = file.getLineAndCharacterOfPosition(diagnostic.start);
+            locationStr = `${file.fileName}:${line}:${character}`;
+        }
+        console.warn(`TypeScript: ${locationStr} ${text}`);
+    }
+
     // Associate external module names with the corresponding file symbols.
     // We need these mappings to identify which module a given external type comes from.
     // The TypeScript API lets us resolve a module name to a source file, but there is no
@@ -512,6 +583,9 @@ if (process.argv.length > 2) {
         handleOpenProjectCommand({
             command: "open-project",
             tsConfig: argument,
+            packageEntryPoints: [],
+            packageJsonFiles: [],
+            virtualSourceRoot: null,
         });
         for (let sf of state.project.program.getSourceFiles()) {
             if (pathlib.basename(sf.fileName) === "lib.d.ts") continue;

javascript/extractor/lib/typescript/src/virtual_source_root.ts

@@ -0,0 +1,39 @@
+import * as pathlib from "path";
+import * as ts from "./typescript";
+
+/**
+ * Mapping from the real source root to the virtual source root,
+ * a directory whose folder structure mirrors the real source root, but with `node_modules` installed.
+ */
+export class VirtualSourceRoot {
+  constructor(
+    private sourceRoot: string,
+
+    /**
+     * Directory whose folder structure mirrors the real source root, but with `node_modules` installed,
+     * or undefined if no virtual source root exists.
+     */
+    private virtualSourceRoot: string,
+  ) {}
+
+  /**
+   * Maps a path under the real source root to the corresponding path in the virtual source root.
+   */
+  public toVirtualPath(path: string) {
+    if (!this.virtualSourceRoot) return null;
+    let relative = pathlib.relative(this.sourceRoot, path);
+    if (relative.startsWith('..') || pathlib.isAbsolute(relative)) return null;
+    return pathlib.join(this.virtualSourceRoot, relative);
+  }
+
+  /**
+   * Maps a path under the real source root to the corresponding path in the virtual source root.
+   */
+  public toVirtualPathIfFileExists(path: string) {
+    let virtualPath = this.toVirtualPath(path);
+    if (virtualPath != null && ts.sys.fileExists(virtualPath)) {
+        return virtualPath;
+    }
+    return null;
+  }
+}