C++: Document InlineExpectationsTest

Author: Dave Bartolomeo

cpp/ql/test/TestUtilities/InlineExpectationsTest.qll

@@ -1,11 +1,116 @@
+/**
+ * Provides a library for writing QL tests whose success or failure is based on expected results
+ * embedded in the test source code as comments, rather than a `.expected` file.
+ *
+ * To create a new inline expectations test:
+ * - Declare a class that extends `InlineExpectationsTest`. In the characteristic predicate of the
+ * new class, bind `this` to a unique string (usually the name of the test).
+ * - Override the `hasActualResult()` predicate to produce the actual results of the query. For each
+ * result, specify a `Location`, a text description of the element for which the result was
+ * reported, a short string to serve as the tag to identify expected results for this test, and the
+ * expected value of the result.
+ * - Override `getARelevantTag()` to return the set of tags that can be produced by
+ * `hasActualResult()`. Often this is just a single tag.
+ *
+ * Example:
+ * ```
+ * class ConstantValueTest extends InlineExpectationsTest {
+ *   ConstantValueTest() { this = "ConstantValueTest" }
+ *
+ *   override string getARelevantTag() {
+ *     // We only use one tag for this test.
+ *     result = "const"
+ *   }
+ *
+ *   override predicate hasActualResult(
+ *     Location location, string element, string tag, string valuesasas
+ *   ) {
+ *     exists(Expr e |
+ *       tag = "const" and // The tag for this test.
+ *       valuesasas = e.getValue() and // The expected value. Will only hold for constant expressions.
+ *       location = e.getLocation() and // The location of the result to be reported.
+ *       element = e.toString() // The display text for the result.
+ *     )
+ *   }
+ * }
+ * ```
+ *
+ * There is no need to write a `select` clause or query predicate. All of the differences between
+ * expected results and actual results will be reported in the `failures()` query predicate.
+ *
+ * To annotate the test source code with an expected result, place a C++-style (`//`) comment on the
+ * same line as the expected result, with text of the following format as the body of the comment:
+ *
+ * `// $tag=expected-value`
+ *
+ * Where `tag` is the value of the `tag` parameter from `hasActualResult()`, and `expected-value` is
+ * the value of the `value` parameter from `hasActualResult()`. Multiple expectations may be placed
+ * in the same comment, as long as each is prefixed by a `$`. Any actual result that appears on a
+ * line that does not contain a matching expected result comment will be reported with a message of
+ * the form "Unexpected result: tag=value". Any expected result comment for which there is no
+ * matching actual result will be reported with a message of the form
+ * "Missing result: tag=expected-value".
+ *
+ * Example:
+ * ```
+ * int i = x + 5;  // $const=5
+ * int j = y + (7 - 3)  // $const=7 $const=3 $const=4  // The result of the subtraction is a constant.
+ * ```
+ *
+ * For tests that contain known false positives and false negatives, it is possible to further
+ * annotate that a particular expected result is known to be a false positive, or that a particular
+ * missing result is known to be a false negative:
+ *
+ * `// $f+:tag=expected-value`  // False positive
+ * `// $f-:tag=expected-value`  // False negative
+ *
+ * A false positive expectation is treated as any other expected result, except that if there is no
+ * matching actual result, the message will be of the form "Fixed false positive: tag=value". A
+ * false negative expectation is treated as if there were no expected result, except that if a
+ * matching expected result is found, the message will be of the form
+ * "Fixed false negative: tag=value".
+ *
+ * If the same result value is expected for two or more tags on the same line, there is a shorthand
+ * notation available:
+ *
+ * `// $tag1,tag2=expected-value`
+ *
+ * is equivalent to:
+ *
+ * `// $tag1=expected-value $tag2=expected-value`
+ */
+
 import cpp
 
+/**
+ * Base class for tests with inline expectations. The test extends this class to provide the actual
+ * results of the query, which are then compared with the expected results in comments to produce a
+ * list of failure messages that point out where the actual results differ from the expected
+ * results.
+ */
 abstract class InlineExpectationsTest extends string {
   bindingset[this]
   InlineExpectationsTest() { any() }
 
+  /**
+   * Returns all tags that can be generated by this test. Most tests will only ever produce a single
+   * tag. Any expected result comments for a tag that is not returned by the `getARelevantTag()`
+   * predicate for an active test will be ignored. This makes it possible to write multiple tests in
+   * different `.ql` files that all query the same source code.
+   */
   abstract string getARelevantTag();
 
+  /**
+   * Returns the actual results of the query that is being tested. Each result consist of the
+   * following values:
+   * - `location` - The source code location of the result. Any expected result comment must appear
+   *   on the start line of this location.
+   * - `element` - Display text for the element on which the result is reported.
+   * - `tag` - The tag that marks this result as coming from this test. This must be one of the tags
+   *   returned by `getARelevantTag()`.
+   * - `value` - The value of the result, which will be matched against the value associated with
+   *   `tag` in any expected result comment on that line.
+   */
   abstract predicate hasActualResult(Location location, string element, string tag, string value);
 
   final predicate hasFailureMessage(FailureLocatable element, string message) {