Merge pull request #3500 from yoff/UnmatchableDollar

Python: Regexp: Handle repetions {n} (with no ,)

Author: RasmusWL

python/ql/src/semmle/python/regex.qll

@@ -497,8 +497,12 @@ abstract class RegexString extends Expr {
             this.getChar(endin) = "}" and
             end > start and
             exists(string multiples | multiples = this.getText().substring(start + 1, endin) |
+                multiples.regexpMatch("0+") and maybe_empty = true
+                or
                 multiples.regexpMatch("0*,[0-9]*") and maybe_empty = true
                 or
+                multiples.regexpMatch("0*[1-9][0-9]*") and maybe_empty = false
+                or
                 multiples.regexpMatch("0*[1-9][0-9]*,[0-9]*") and maybe_empty = false
             ) and
             not exists(int mid |
@@ -643,9 +647,13 @@ abstract class RegexString extends Expr {
         start = 0 and end = this.getText().length()
         or
         exists(int y | this.lastPart(start, y) |
-            this.emptyMatchAtEndGroup(end, y) or
-            this.qualifiedItem(end, y, true) or
+            this.emptyMatchAtEndGroup(end, y)
+            or
+            this.qualifiedItem(end, y, true)
+            or
             this.specialCharacter(end, y, "$")
+            or
+            y = end + 2 and this.escapingChar(end) and this.getChar(end + 1) = "Z"
         )
         or
         exists(int x |

python/ql/test/library-tests/regex/Characters.expected

@@ -110,7 +110,6 @@
 | ax{3,} | 5 | 6 |
 | ax{3} | 0 | 1 |
 | ax{3} | 1 | 2 |
-| ax{3} | 2 | 3 |
 | ax{3} | 3 | 4 |
 | ax{3} | 4 | 5 |
 | ax{,3} | 0 | 1 |

python/ql/test/library-tests/regex/FirstLast.expected

@@ -84,6 +84,8 @@
 | ax{3,} | last | 1 | 6 |
 | ax{3,} | last | 5 | 6 |
 | ax{3} | first | 0 | 1 |
+| ax{3} | last | 1 | 2 |
+| ax{3} | last | 1 | 5 |
 | ax{3} | last | 4 | 5 |
 | ax{,3} | first | 0 | 1 |
 | ax{,3} | last | 0 | 1 |

python/ql/test/library-tests/regex/Qualified.expected

@@ -11,4 +11,5 @@
 | ^[A-Z_]+$(?<!not-this) | 1 | 8 | false |
 | ax{01,3} | 1 | 8 | false |
 | ax{3,} | 1 | 6 | false |
+| ax{3} | 1 | 5 | false |
 | ax{,3} | 1 | 6 | true |

python/ql/test/library-tests/regex/Regex.expected

@@ -207,9 +207,9 @@
 | ax{3,} | sequence | 0 | 6 |
 | ax{3} | char | 0 | 1 |
 | ax{3} | char | 1 | 2 |
-| ax{3} | char | 2 | 3 |
 | ax{3} | char | 3 | 4 |
 | ax{3} | char | 4 | 5 |
+| ax{3} | qualified | 1 | 5 |
 | ax{3} | sequence | 0 | 5 |
 | ax{,3} | char | 0 | 1 |
 | ax{,3} | char | 1 | 2 |

python/ql/test/query-tests/Expressions/Regex/DuplicateCharacterInSet.expected

@@ -1,3 +1,3 @@
-| test.py:41:12:41:18 | Str | This regular expression includes duplicate character 'A' in a set of characters. |
-| test.py:42:12:42:19 | Str | This regular expression includes duplicate character '0' in a set of characters. |
-| test.py:43:12:43:21 | Str | This regular expression includes duplicate character '-' in a set of characters. |
+| test.py:46:12:46:18 | Str | This regular expression includes duplicate character 'A' in a set of characters. |
+| test.py:47:12:47:19 | Str | This regular expression includes duplicate character '0' in a set of characters. |
+| test.py:48:12:48:21 | Str | This regular expression includes duplicate character '-' in a set of characters. |

python/ql/test/query-tests/Expressions/Regex/UnmatchableCaret.expected

@@ -1,4 +1,4 @@
 | test.py:4:12:4:19 | Str | This regular expression includes an unmatchable caret at offset 1. |
 | test.py:5:12:5:23 | Str | This regular expression includes an unmatchable caret at offset 5. |
 | test.py:6:12:6:21 | Str | This regular expression includes an unmatchable caret at offset 2. |
-| test.py:74:12:74:27 | Str | This regular expression includes an unmatchable caret at offset 8. |
+| test.py:79:12:79:27 | Str | This regular expression includes an unmatchable caret at offset 8. |

python/ql/test/query-tests/Expressions/Regex/UnmatchableDollar.expected

@@ -1,4 +1,4 @@
 | test.py:29:12:29:19 | Str | This regular expression includes an unmatchable dollar at offset 3. |
 | test.py:30:12:30:23 | Str | This regular expression includes an unmatchable dollar at offset 3. |
 | test.py:31:12:31:20 | Str | This regular expression includes an unmatchable dollar at offset 2. |
-| test.py:75:12:75:26 | Str | This regular expression includes an unmatchable dollar at offset 3. |
+| test.py:80:12:80:26 | Str | This regular expression includes an unmatchable dollar at offset 3. |

python/ql/test/query-tests/Expressions/Regex/test.py

@@ -30,12 +30,17 @@
 re.compile(b"abc$ (?s)")
 re.compile(b"\[$]  ")
 
-#Likely false positives for unmatchable dollar
-re.compile(b"[$]  ")
-re.compile(b"\$  ")
-re.compile(b"abc$(?m)")
-re.compile(b"abc$()")
-
+#Not unmatchable dollar
+re.match(b"[$]  ", b"$  ")
+re.match(b"\$  ", b"$  ")
+re.match(b"abc$(?m)", b"abc")
+re.match(b"abc$()", b"abc")
+re.match(b"((a$)|b)*", b"bba")
+re.match(b"((a$)|b){4}", b"bbba") # Inspired by FP report here: https://github.com/github/codeql/issues/2403
+re.match(b"((a$).*)", b"a")
+re.match("(\Aab$|\Aba$)$\Z", "ab")
+re.match(b"((a$\Z)|b){4}", b"bbba")
+re.match(b"(a){00}b", b"b")
 
 #Duplicate character in set
 re.compile(b"[AA]")