Merge pull request #526 from esben-semmle/js/flow-parsing-improvements

Approved by xiemaisi

Author: semmle-qlci

change-notes/1.19/extractor-javascript.md

@@ -26,3 +26,6 @@
   point to an installation of the `typescript` NPM package.
 
 * The extractor now supports [Optional Chaining](https://github.com/tc39/proposal-optional-chaining) expressions.
+
+* The extractor now supports additional [Flow](https://flow.org/) syntax.
+

javascript/extractor/src/com/semmle/jcorn/Parser.java

@@ -3040,14 +3040,18 @@ protected MemberDefinition<?> parseClassMember(boolean hadConstructor) {
 		return parseClassPropertyBody(pi, hadConstructor, isStatic);
 	}
 
+	protected boolean atGetterSetterName(PropertyInfo pi) {
+		return !pi.isGenerator && !pi.isAsync && pi.key instanceof Identifier && this.type != TokenType.parenL && (((Identifier) pi.key).getName().equals("get") || ((Identifier) pi.key).getName().equals("set"));
+	}
+
 	/**
 	 * Parse a method declaration in a class, assuming that its name has already been consumed.
 	 */
 	protected MemberDefinition<?> parseClassPropertyBody(PropertyInfo pi, boolean hadConstructor, boolean isStatic) {
 		pi.kind = "method";
 		boolean isGetSet = false;
 		if (!pi.computed) {
-			if (!pi.isGenerator && !pi.isAsync && pi.key instanceof Identifier && this.type != TokenType.parenL && (((Identifier) pi.key).getName().equals("get") || ((Identifier) pi.key).getName().equals("set"))) {
+			if (atGetterSetterName(pi)) {
 				isGetSet = true;
 				pi.kind = ((Identifier) pi.key).getName();
 				this.parsePropertyName(pi);

javascript/extractor/src/com/semmle/jcorn/flow/FlowParser.java

@@ -146,6 +146,10 @@ private void flowParseTypeInitialiser(TokenType tok, boolean allowLeadingPipeOrA
 		boolean oldInType = inType;
 		inType = true;
 		this.expect(tok == null ? TokenType.colon : tok);
+		if (this.type == TokenType.modulo) {// an annotation like '%checks' without a preceeding type
+			inType = oldInType;
+			return;
+		}
 		if (allowLeadingPipeOrAnd) {
 			if (this.type == TokenType.bitwiseAND || this.type == TokenType.bitwiseOR) {
 				this.next();
@@ -223,14 +227,29 @@ private void flowParseDeclareModule(Position start) {
 		while (this.type != TokenType.braceR) {
 			Position stmtStart = startLoc;
 
-			// todo: declare check
-			this.next();
+			if (this.eat(TokenType._import)) {
+				this.flowParseDeclareImport(stmtStart);
+			} else {
+				// todo: declare check
+				this.next();
 
-			this.flowParseDeclare(stmtStart);
+				this.flowParseDeclare(stmtStart);
+			}
 		}
 		this.expect(TokenType.braceR);
 	}
 
+	private void flowParseDeclareImport(Position stmtStart) {
+		String kind = flowParseImportSpecifiers();
+		if (kind == null) {
+			this.raise(stmtStart, "Imports within a `declare module` body must always be `import type` or `import typeof`.");
+		}
+		this.expect(TokenType.name);
+		this.expectContextual("from");
+		this.expect(TokenType.string);
+		this.semicolon();
+	}
+
 	private void flowParseDeclareModuleExports() {
 		this.expectContextual("module");
 		this.expect(TokenType.dot);
@@ -737,7 +756,7 @@ private void flowParsePrimaryType() {
 
 	private void flowParsePostfixType() {
 		this.flowParsePrimaryType();
-		if (this.type == TokenType.bracketL) {
+		while (this.type == TokenType.bracketL) {
 			this.expect(TokenType.bracketL);
 			this.expect(TokenType.bracketR);
 		}
@@ -807,11 +826,20 @@ protected Node parseFunctionBody(Identifier id, List<Expression> params, boolean
 			// if allowExpression is true then we're parsing an arrow function and if
 			// there's a return type then it's been handled elsewhere
 			this.flowParseTypeAnnotation();
+			this.flowParseChecksAnnotation();
 		}
 
 		return super.parseFunctionBody(id, params, isArrowFunction);
 	}
 
+	private void flowParseChecksAnnotation() {
+		// predicate functions with the special '%checks' annotation
+		if (this.type == TokenType.modulo && lookaheadIsIdent("checks", true)) {
+			this.next();
+			this.next();
+		}
+	}
+
 	// interfaces
 	@Override
 	protected Statement parseStatement(boolean declaration, boolean topLevel, Set<String> exports) {
@@ -975,24 +1003,30 @@ protected Expression processBindingListItem(Expression param) {
 		return param;
 	}
 
+	private String flowParseImportSpecifiers() {
+		String kind = null;
+		if (this.type == TokenType._typeof) {
+			kind = "typeof";
+		} else if (this.isContextual("type")) {
+			kind = "type";
+		}
+		if (kind != null) {
+			String lh = lookahead(4);
+			if (!lh.isEmpty()) {
+				int c = lh.codePointAt(0);
+				if ((Identifiers.isIdentifierStart(c, true) && !"from".equals(lh)) || c == '{' || c == '*') {
+					this.next();
+				}
+			}
+		}
+		return kind;
+	}
+
 	@Override
 	protected List<ImportSpecifier> parseImportSpecifiers() {
 		String kind = null;
 		if (flow()) {
-			if (this.type == TokenType._typeof) {
-				kind = "typeof";
-			} else if (this.isContextual("type")) {
-				kind = "type";
-			}
-			if (kind != null) {
-				String lh = lookahead(4);
-				if (!lh.isEmpty()) {
-					int c = lh.codePointAt(0);
-					if ((Identifiers.isIdentifierStart(c, true) && !"from".equals(lh)) || c == '{' || c == '*') {
-						this.next();
-					}
-				}
-			}
+			kind = flowParseImportSpecifiers();
 		}
 
 		List<ImportSpecifier> specs = super.parseImportSpecifiers();
@@ -1102,6 +1136,7 @@ protected ParenthesisedExpressions parseParenthesisedExpressions(DestructuringEr
 				boolean oldNoAnonFunctionType = noAnonFunctionType;
 				noAnonFunctionType = true;
 				flowParseTypeAnnotation();
+				flowParseChecksAnnotation();
 				noAnonFunctionType = oldNoAnonFunctionType;
 				if (this.type != TokenType.arrow)
 					unexpected();
@@ -1158,4 +1193,12 @@ protected void parsePropertyName(PropertyInfo result) {
 			this.eat(TokenType.plusMin);
 		super.parsePropertyName(result);
 	}
+
+	@Override
+	protected boolean atGetterSetterName(PropertyInfo pi) {
+		if (flow() && this.isRelational("<"))
+			return false;
+		return super.atGetterSetterName(pi);
+	}
+
 }

javascript/extractor/src/com/semmle/js/extractor/Main.java

@@ -41,7 +41,7 @@ public class Main {
 	 * such a way that it may produce different tuples for the same file under the same
 	 * {@link ExtractorConfig}.
 	 */
-	public static final String EXTRACTOR_VERSION = "2018-11-20";
+	public static final String EXTRACTOR_VERSION = "2018-11-22_a";
 
 	public static final Pattern NEWLINE = Pattern.compile("\n");
 

javascript/extractor/tests/flow/input/array-types.js

@@ -0,0 +1,4 @@
+function f(p: T) {}
+function f(p: T[]) {}
+function f(p: T[][]) {}
+function f(p: T[][][]) {}

javascript/extractor/tests/flow/input/declared-module-imports.js

@@ -0,0 +1,4 @@
+declare module "m1" {
+  import type t from "m2";
+  import typeof t from "m3";
+}

javascript/extractor/tests/flow/input/get-set-methods.js

@@ -0,0 +1,4 @@
+class C {
+    get<T>(v: T) { }
+    set<T>(v: T) { }
+}

javascript/extractor/tests/flow/input/predicate-function-annotation.js

@@ -0,0 +1,9 @@
+function f(a): boolean %checks {
+    return a;
+}
+function g(): %checks {} {
+    return b;
+}
+
+(c): boolean %checks => c;
+(d): %checks => d;