Merge pull request #506 from esben-semmle/js/optional-chaining-extractor-and-ql

JS: Optional chaining support in extractor and ql

Author: Max Schaefer

change-notes/1.19/extractor-javascript.md

@@ -24,3 +24,5 @@
 * The TypeScript compiler is now bundled with the distribution, and no longer needs to be installed manually.
   Should the compiler version need to be overridden, set the `SEMMLE_TYPESCRIPT_HOME` environment variable to
   point to an installation of the `typescript` NPM package.
+
+* The extractor now supports [Optional Chaining](https://github.com/tc39/proposal-optional-chaining) expressions.

javascript/extractor/src/com/semmle/jcorn/CustomParser.java

@@ -153,7 +153,7 @@ protected Expression parseExprAtom(DestructuringErrors refDestructuringErrors) {
 			Identifier name = this.parseIdent(true);
 			this.expect(TokenType.parenL);
 			List<Expression> args = this.parseExprList(TokenType.parenR, false, false, null);
-			CallExpression node = new CallExpression(new SourceLocation(startLoc), name, new ArrayList<>(), args);
+			CallExpression node = new CallExpression(new SourceLocation(startLoc), name, new ArrayList<>(), args, false, false);
 			return this.finishNode(node);
 		} else {
 			return super.parseExprAtom(refDestructuringErrors);
@@ -212,7 +212,7 @@ protected INode parseFunction(Position startLoc, boolean isStatement, boolean al
 			 *   A.f = function f(...) { ... };
 			 */
 			SourceLocation memloc = new SourceLocation(iface.getName() + "::" + id.getName(), iface.getLoc().getStart(), id.getLoc().getEnd());
-			MemberExpression mem = new MemberExpression(memloc, iface, new Identifier(id.getLoc(), id.getName()), false);
+			MemberExpression mem = new MemberExpression(memloc, iface, new Identifier(id.getLoc(), id.getName()), false, false, false);
 			AssignmentExpression assgn = new AssignmentExpression(result.getLoc(), "=", mem, ((FunctionDeclaration)result).asFunctionExpression());
 			return new ExpressionStatement(result.getLoc(), assgn);
 		}

javascript/extractor/src/com/semmle/jcorn/Parser.java

@@ -29,6 +29,7 @@
 import com.semmle.js.ast.BreakStatement;
 import com.semmle.js.ast.CallExpression;
 import com.semmle.js.ast.CatchClause;
+import com.semmle.js.ast.Chainable;
 import com.semmle.js.ast.ClassBody;
 import com.semmle.js.ast.ClassDeclaration;
 import com.semmle.js.ast.ClassExpression;
@@ -504,6 +505,14 @@ private Token readToken_dot() {
 		}
 	}
 
+	private Token readToken_question() { // '?'
+		int next = charAt(this.pos + 1);
+		int next2 = charAt(this.pos + 2);
+		if (this.options.esnext() && next == '.' && !('0' <= next2 && next2 <= '9')) // '?.', but not '?.X' where X is a digit
+			return this.finishOp(TokenType.questiondot, 2);
+		return this.finishOp(TokenType.question, 1);
+	}
+
 	private Token readToken_slash() { // '/'
 		int next = charAt(this.pos + 1);
 		if (this.exprAllowed) {
@@ -616,7 +625,7 @@ protected Token getTokenFromCode(int code) {
 		case 123: ++this.pos; return this.finishToken(TokenType.braceL);
 		case 125: ++this.pos; return this.finishToken(TokenType.braceR);
 		case 58: ++this.pos; return this.finishToken(TokenType.colon);
-		case 63: ++this.pos; return this.finishToken(TokenType.question);
+		case 63: return this.readToken_question();
 
 		case 96: // '`'
 			if (this.options.ecmaVersion() < 6) break;
@@ -1465,17 +1474,19 @@ protected Expression parseSubscripts(Expression base, int startPos, Position sta
 		}
 	}
 
+	private boolean isOnOptionalChain(boolean optional, Expression base) {
+		return optional || base instanceof Chainable && ((Chainable)base).isOnOptionalChain();
+	}
+
 	/**
 	 * Parse a single subscript {@code s}; if more subscripts could follow, return {@code Pair.make(s, true},
 	 * otherwise return {@code Pair.make(s, false)}.
 	 */
 	protected Pair<Expression, Boolean> parseSubscript(final Expression base, Position startLoc, boolean noCalls) {
 		boolean maybeAsyncArrow = this.options.ecmaVersion() >= 8 && base instanceof Identifier && "async".equals(((Identifier) base).getName()) && !this.canInsertSemicolon();
-		if (this.eat(TokenType.dot)) {
-			MemberExpression node = new MemberExpression(new SourceLocation(startLoc), base, this.parseIdent(true), false);
-			return Pair.make(this.finishNode(node), true);
-		} else if (this.eat(TokenType.bracketL)) {
-			MemberExpression node = new MemberExpression(new SourceLocation(startLoc), base, this.parseExpression(false, null), true);
+		boolean optional = this.eat(TokenType.questiondot);
+		if (this.eat(TokenType.bracketL)) {
+			MemberExpression node = new MemberExpression(new SourceLocation(startLoc), base, this.parseExpression(false, null), true, optional, isOnOptionalChain(optional, base));
 			this.expect(TokenType.bracketR);
 			return Pair.make(this.finishNode(node), true);
 		} else if (!noCalls && this.eat(TokenType.parenL)) {
@@ -1494,11 +1505,17 @@ protected Pair<Expression, Boolean> parseSubscript(final Expression base, Positi
 			this.checkExpressionErrors(refDestructuringErrors, true);
 			if (oldYieldPos > 0) this.yieldPos = oldYieldPos;
 			if (oldAwaitPos > 0) this.awaitPos = oldAwaitPos;
-			CallExpression node = new CallExpression(new SourceLocation(startLoc), base, new ArrayList<>(), exprList);
+			CallExpression node = new CallExpression(new SourceLocation(startLoc), base, new ArrayList<>(), exprList, optional, isOnOptionalChain(optional, base));
 			return Pair.make(this.finishNode(node), true);
 		} else if (this.type == TokenType.backQuote) {
+			if (isOnOptionalChain(optional, base)) {
+				this.raise(base, "An optional chain may not be used in a tagged template expression.");
+			}
 			TaggedTemplateExpression node = new TaggedTemplateExpression(new SourceLocation(startLoc), base, this.parseTemplate(true));
 			return Pair.make(this.finishNode(node), true);
+		} else if (optional || this.eat(TokenType.dot)) {
+			MemberExpression node = new MemberExpression(new SourceLocation(startLoc), base, this.parseIdent(true), false, optional, isOnOptionalChain(optional, base));
+			return Pair.make(this.finishNode(node), true);
 		} else {
 			return Pair.make(base, false);
 		}
@@ -1719,6 +1736,10 @@ protected Expression parseNew() {
 		int innerStartPos = this.start;
 		Position innerStartLoc = this.startLoc;
 		Expression callee = this.parseSubscripts(this.parseExprAtom(null), innerStartPos, innerStartLoc, true);
+
+		if (isOnOptionalChain(false, callee))
+			this.raise(callee, "An optional chain may not be used in a `new` expression.");
+
 		List<Expression> arguments;
 		if (this.eat(TokenType.parenL))
 			arguments = this.parseExprList(TokenType.parenR, this.options.ecmaVersion() >= 8, false, null);
@@ -2159,9 +2180,12 @@ protected INode toAssignable(INode node, boolean isBinding) {
 				return new ParenthesizedExpression(node.getLoc(), (Expression) this.toAssignable(expr, isBinding));
 			}
 
-			if (node instanceof MemberExpression)
+			if (node instanceof MemberExpression) {
+				if (isOnOptionalChain(false, (MemberExpression)node))
+					this.raise(node, "Invalid left-hand side in assignment");
 				if (!isBinding)
 					return node;
+			}
 
 			this.raise(node, "Assigning to rvalue");
 		}

javascript/extractor/src/com/semmle/jcorn/TokenType.java

@@ -76,6 +76,7 @@ public void updateContext(Parser parser, TokenType prevType) {
 	semi = new TokenType(new Properties(";").beforeExpr()),
 	colon = new TokenType(new Properties(":").beforeExpr()),
 	dot = new TokenType(new Properties(".")),
+	questiondot = new TokenType(new Properties("?.")),
 	question = new TokenType(new Properties("?").beforeExpr()),
 	arrow = new TokenType(new Properties("=>").beforeExpr()),
 	template = new TokenType(new Properties("template")),

javascript/extractor/src/com/semmle/js/ast/AST2JSON.java

@@ -215,6 +215,7 @@ public JsonElement visit(CallExpression nd, Void c) {
 		JsonObject result = this.mkNode(nd);
 		result.add("callee", visit(nd.getCallee()));
 		result.add("arguments", visit(nd.getArguments()));
+		result.add("optional", new JsonPrimitive(nd.isOptional()));
 		return result;
 	}
 
@@ -424,6 +425,7 @@ public JsonElement visit(MemberExpression nd, Void c) {
 		result.add("object", visit(nd.getObject()));
 		result.add("property", visit(nd.getProperty()));
 		result.add("computed", new JsonPrimitive(nd.isComputed()));
+		result.add("optional", new JsonPrimitive(nd.isOptional()));
 		return result;
 	}
 

javascript/extractor/src/com/semmle/js/ast/CallExpression.java

@@ -8,8 +8,8 @@
  * A function call expression such as <code>f(1, 1)</code>.
  */
 public class CallExpression extends InvokeExpression {
-	public CallExpression(SourceLocation loc, Expression callee, List<ITypeExpression> typeArguments, List<Expression> arguments) {
-		super("CallExpression", loc, callee, typeArguments, arguments);
+	public CallExpression(SourceLocation loc, Expression callee, List<ITypeExpression> typeArguments, List<Expression> arguments, Boolean optional, Boolean onOptionalChain) {
+		super("CallExpression", loc, callee, typeArguments, arguments, optional, onOptionalChain);
 	}
 
 	@Override

javascript/extractor/src/com/semmle/js/ast/Chainable.java

@@ -0,0 +1,16 @@
+package com.semmle.js.ast;
+
+/**
+ * A chainable expression, such as a member access or function call.
+ */
+public interface Chainable {
+	/**
+	 * Is this step of the chain optional?
+	 */
+	abstract boolean isOptional();
+
+	/**
+	 * Is this on an optional chain?
+	 */
+	abstract boolean isOnOptionalChain();
+}

javascript/extractor/src/com/semmle/js/ast/InvokeExpression.java

@@ -8,20 +8,24 @@
 /**
  * An invocation, that is, either a {@link CallExpression} or a {@link NewExpression}.
  */
-public abstract class InvokeExpression extends Expression implements INodeWithSymbol {
+public abstract class InvokeExpression extends Expression implements INodeWithSymbol, Chainable {
 	private final Expression callee;
 	private final List<ITypeExpression> typeArguments;
 	private final List<Expression> arguments;
+	private final boolean optional;
+	private final boolean onOptionalChain;
 	private int resolvedSignatureId = -1;
 	private int overloadIndex = -1;
 	private int symbol = -1;
 
 	public InvokeExpression(String type, SourceLocation loc, Expression callee, List<ITypeExpression> typeArguments,
-			List<Expression> arguments) {
+			List<Expression> arguments, Boolean optional, Boolean onOptionalChain) {
 		super(type, loc);
 		this.callee = callee;
 		this.typeArguments = typeArguments;
 		this.arguments = arguments;
+		this.optional = optional == Boolean.TRUE;
+		this.onOptionalChain = onOptionalChain == Boolean.TRUE;
 	}
 
 	/**
@@ -45,6 +49,16 @@ public List<Expression> getArguments() {
 		return arguments;
 	}
 
+	@Override
+	public boolean isOptional() {
+		return optional;
+	}
+
+	@Override
+	public boolean isOnOptionalChain() {
+		return onOptionalChain;
+	}
+
 	public int getResolvedSignatureId() {
 		return resolvedSignatureId;
 	}
@@ -70,4 +84,4 @@ public int getSymbol() {
 	public void setSymbol(int symbol) {
 		this.symbol = symbol;
 	}
-}
+}

javascript/extractor/src/com/semmle/js/ast/MemberExpression.java

@@ -6,16 +6,20 @@
 /**
  * A member expression, either computed (<code>e[f]</code>) or static (<code>e.f</code>).
  */
-public class MemberExpression extends Expression implements ITypeExpression, INodeWithSymbol {
+public class MemberExpression extends Expression implements ITypeExpression, INodeWithSymbol, Chainable {
 	private final Expression object, property;
 	private final boolean computed;
+	private final boolean optional;
+	private final boolean onOptionalChain;
 	private int symbol = -1;
 
-	public MemberExpression(SourceLocation loc, Expression object, Expression property, Boolean computed) {
+	public MemberExpression(SourceLocation loc, Expression object, Expression property, Boolean computed, Boolean optional, Boolean onOptionalChain) {
 		super("MemberExpression", loc);
 		this.object = object;
 		this.property = property;
 		this.computed = computed == Boolean.TRUE;
+		this.optional = optional == Boolean.TRUE;
+		this.onOptionalChain = onOptionalChain == Boolean.TRUE;
 	}
 
 	@Override
@@ -45,6 +49,16 @@ public boolean isComputed() {
 		return computed;
 	}
 
+	@Override
+	public boolean isOptional() {
+		return optional;
+	}
+
+	@Override
+	public boolean isOnOptionalChain() {
+		return onOptionalChain;
+	}
+
 	@Override
 	public int getSymbol() {
 		return symbol;

javascript/extractor/src/com/semmle/js/ast/NewExpression.java

@@ -9,7 +9,7 @@
  */
 public class NewExpression extends InvokeExpression {
 	public NewExpression(SourceLocation loc, Expression callee, List<ITypeExpression> typeArguments, List<Expression> arguments) {
-		super("NewExpression", loc, callee, typeArguments, arguments);
+		super("NewExpression", loc, callee, typeArguments, arguments, false, false);
 	}
 
 	@Override