Shared: Post-processing query for inline test expectations

Author: hvitved

shared/util/codeql/util/test/InlineExpectationsTest.qll

@@ -134,8 +134,20 @@ module Make<InlineExpectationsTestSig Impl> {
      * predicate for an active test will be ignored. This makes it possible to write multiple tests in
      * different `.ql` files that all query the same source code.
      */
+    bindingset[result]
     string getARelevantTag();
 
+    /**
+     * Holds if expected tag `expectedTag` matches actual tag `actualTag`.
+     *
+     * This is normally defined as `expectedTag = actualTag`.
+     */
+    bindingset[expectedTag, actualTag]
+    default predicate tagMatches(string expectedTag, string actualTag) { expectedTag = actualTag }
+
+    bindingset[expectedTag]
+    default predicate tagIsOptional(string expectedTag) { none() }
+
     /**
      * Returns the actual results of the query that is being tested. Each result consist of the
      * following values:
@@ -200,13 +212,13 @@ module Make<InlineExpectationsTestSig Impl> {
         not exists(ActualTestResult actualResult | expectation.matchesActualResult(actualResult)) and
         expectation.getTag() = TestImpl::getARelevantTag() and
         element = expectation and
-        (
-          expectation instanceof GoodTestExpectation and
-          message = "Missing result: " + expectation.getExpectationText()
-          or
-          expectation instanceof FalsePositiveTestExpectation and
-          message = "Fixed spurious result: " + expectation.getExpectationText()
-        )
+        not expectation.isOptional()
+      |
+        expectation instanceof GoodTestExpectation and
+        message = "Missing result: " + expectation.getExpectationText()
+        or
+        expectation instanceof FalsePositiveTestExpectation and
+        message = "Fixed spurious result: " + expectation.getExpectationText()
       )
       or
       exists(InvalidTestExpectation expectation |
@@ -311,9 +323,11 @@ module Make<InlineExpectationsTestSig Impl> {
 
       predicate matchesActualResult(ActualTestResult actualResult) {
         onSameLine(pragma[only_bind_into](this), actualResult) and
-        this.getTag() = actualResult.getTag() and
+        TestImpl::tagMatches(this.getTag(), actualResult.getTag()) and
         this.getValue() = actualResult.getValue()
       }
+
+      predicate isOptional() { TestImpl::tagIsOptional(tag) }
     }
 
     // Note: These next three classes correspond to all the possible values of type `TColumn`.
@@ -385,6 +399,7 @@ module Make<InlineExpectationsTestSig Impl> {
    * ```
    */
   module MergeTests<TestSig TestImpl1, TestSig TestImpl2> implements TestSig {
+    bindingset[result]
     string getARelevantTag() {
       result = TestImpl1::getARelevantTag() or result = TestImpl2::getARelevantTag()
     }
@@ -408,6 +423,7 @@ module Make<InlineExpectationsTestSig Impl> {
   module MergeTests3<TestSig TestImpl1, TestSig TestImpl2, TestSig TestImpl3> implements TestSig {
     private module M = MergeTests<MergeTests<TestImpl1, TestImpl2>, TestImpl3>;
 
+    bindingset[result]
     string getARelevantTag() { result = M::getARelevantTag() }
 
     predicate hasActualResult(Impl::Location location, string element, string tag, string value) {
@@ -427,6 +443,7 @@ module Make<InlineExpectationsTestSig Impl> {
   {
     private module M = MergeTests<MergeTests3<TestImpl1, TestImpl2, TestImpl3>, TestImpl4>;
 
+    bindingset[result]
     string getARelevantTag() { result = M::getARelevantTag() }
 
     predicate hasActualResult(Impl::Location location, string element, string tag, string value) {
@@ -448,6 +465,7 @@ module Make<InlineExpectationsTestSig Impl> {
     private module M =
       MergeTests<MergeTests4<TestImpl1, TestImpl2, TestImpl3, TestImpl4>, TestImpl5>;
 
+    bindingset[result]
     string getARelevantTag() { result = M::getARelevantTag() }
 
     predicate hasActualResult(Impl::Location location, string element, string tag, string value) {
@@ -590,3 +608,105 @@ private string expectationPattern() {
     result = tags + "(?:=" + value + ")?"
   )
 }
+
+/**
+ * Holds if alerts should be restricted to those whose alert messages
+ * that regexp matches `alertMessage`.
+ */
+extensible predicate testPostProcessingRestrictAlertsTo(string alertMessage);
+
+/**
+ * Provides logic for creating a `@kind test-postprocess` query that checks
+ * inline test expectations using `$ Alert` markers.
+ */
+module TestPostProcessing {
+  external predicate queryResults(string relation, int row, int column, string data);
+
+  external predicate queryRelations(string relation);
+
+  signature module InputSig<InlineExpectationsTestSig Input> {
+    string getRelativeUrl(Input::Location location);
+  }
+
+  module Make<InlineExpectationsTestSig Input, InputSig<Input> Input2> {
+    private import InlineExpectationsTest as InlineExpectationsTest
+    private import InlineExpectationsTest::Make<Input>
+
+    private module TestInput implements TestSig {
+      bindingset[result]
+      string getARelevantTag() { any() }
+
+      bindingset[expectedTag, actualTag]
+      predicate tagMatches(string expectedTag, string actualTag) {
+        expectedTag = "Alert"
+        or
+        exists(string alertTagRegex, string s |
+          alertTagRegex = "Alert\\[(.*)\\]" and
+          s = expectedTag.regexpCapture(alertTagRegex, 1) and
+          actualTag.regexpMatch(".*" + s + ".*")
+        )
+      }
+
+      bindingset[expectedTag]
+      predicate tagIsOptional(string expectedTag) {
+        not expectedTag.regexpMatch("Alert(\\[.*\\])?")
+        or
+        exists(string alertTagRegex, string s, string alertMessage |
+          alertTagRegex = "Alert\\[(.*)\\]" and
+          s = expectedTag.regexpCapture(alertTagRegex, 1) and
+          testPostProcessingRestrictAlertsTo(alertMessage) and
+          not alertMessage.regexpMatch(".*" + s + ".*")
+        )
+      }
+
+      predicate hasActualResult(Input::Location location, string element, string tag, string value) {
+        exists(int row, string loc |
+          queryResults("#select", row, 0, loc) and
+          queryResults("#select", row, 2, element) and
+          tag = element and
+          value = "" and
+          Input2::getRelativeUrl(location) = loc
+        )
+      }
+    }
+
+    private module Test = MakeTest<TestInput>;
+
+    private newtype TTestFailure =
+      MkTestFailure(Test::FailureLocatable f, string message) { Test::testFailures(f, message) }
+
+    private predicate rankedTestFailures(int i, MkTestFailure f) {
+      f =
+        rank[i](MkTestFailure f0, Test::FailureLocatable fl, string message, string filename,
+          int startLine, int startColumn, int endLine, int endColumn |
+          f0 = MkTestFailure(fl, message) and
+          fl.getLocation().hasLocationInfo(filename, startLine, startColumn, endLine, endColumn)
+        |
+          f0 order by filename, startLine, startColumn, endLine, endColumn, message
+        )
+    }
+
+    query predicate results(string relation, int row, int column, string data) {
+      queryResults(relation, row, column, data)
+      or
+      exists(MkTestFailure f, Test::FailureLocatable fl, string message |
+        relation = "testFailures" and
+        rankedTestFailures(row, f) and
+        f = MkTestFailure(fl, message)
+      |
+        column = 0 and data = Input2::getRelativeUrl(fl.getLocation())
+        or
+        column = 1 and data = fl.toString()
+        or
+        column = 2 and data = message
+      )
+    }
+
+    query predicate resultRelations(string relation) {
+      queryRelations(relation)
+      or
+      Test::testFailures(_, _) and
+      relation = "testFailures"
+    }
+  }
+}

shared/util/ext/default-test-post-processing-alert-filter.yml

@@ -0,0 +1,7 @@
+# Make sure that the `testPostProcessingRestrictAlertsTo` predicate has at least one definition
+# to avoid errors about undefined extensionals.
+extensions:
+  - addsTo:
+      pack: codeql/util
+      extensible: testPostProcessingRestrictAlertsTo
+    data: []