Merge pull request #3770 from tausbn/python-add-a-bunch-of-documentation

Python: Add a bunch of documentation.

Author: RasmusWL

python/ql/src/Expressions/CallArgs.qll

@@ -1,3 +1,5 @@
+/** INTERNAL - Methods used by queries that test whether functions are invoked correctly.  */
+
 import python
 import Testing.Mox
 
@@ -71,36 +73,38 @@ private int positional_arg_count_for_call(Call call, Value callable) {
     )
 }
 
+/** Gets the number of arguments in `call`. */
 int arg_count_objectapi(Call call) {
     result = count(call.getAnArg()) + varargs_length_objectapi(call) + count(call.getAKeyword())
 }
 
+/** Gets the number of arguments in `call`. */
 int arg_count(Call call) {
     result = count(call.getAnArg()) + varargs_length(call) + count(call.getAKeyword())
 }
 
-/* Gets a call corresponding to the given class or function*/
+/** Gets a call corresponding to the given class or function. */
 private ControlFlowNode get_a_call_objectapi(Object callable) {
     result = callable.(ClassObject).getACall()
     or
     result = callable.(FunctionObject).getACall()
 }
 
-/* Gets a call corresponding to the given class or function*/
+/** Gets a call corresponding to the given class or function. */
 private ControlFlowNode get_a_call(Value callable) {
     result = callable.(ClassValue).getACall()
     or
     result = callable.(FunctionValue).getACall()
 }
 
-/* Gets the function object corresponding to the given class or function*/
+/** Gets the function object corresponding to the given class or function. */
 FunctionObject get_function_or_initializer_objectapi(Object func_or_cls) {
     result = func_or_cls.(FunctionObject)
     or
     result = func_or_cls.(ClassObject).declaredAttribute("__init__")
 }
 
-/* Gets the function object corresponding to the given class or function*/
+/** Gets the function object corresponding to the given class or function. */
 FunctionValue get_function_or_initializer(Value func_or_cls) {
     result = func_or_cls.(FunctionValue)
     or

python/ql/src/Expressions/Formatting/AdvancedFormatting.qll

@@ -1,5 +1,6 @@
 import python
 
+/** A string constant that looks like it may be used in string formatting operations. */
 library class PossibleAdvancedFormatString extends StrConst {
     PossibleAdvancedFormatString() { this.getText().matches("%{%}%") }
 
@@ -51,6 +52,7 @@ library class PossibleAdvancedFormatString extends StrConst {
     predicate isExplicitlyNumbered() { exists(this.fieldId(_, _).toInt()) }
 }
 
+/** Holds if the formatting string `fmt` contains a sequence of braces `{` of length `len`, beginning at index `index`. */
 predicate brace_sequence(PossibleAdvancedFormatString fmt, int index, int len) {
     exists(string text | text = fmt.getText() |
         text.charAt(index) = "{" and not text.charAt(index - 1) = "{" and len = 1
@@ -61,10 +63,12 @@ predicate brace_sequence(PossibleAdvancedFormatString fmt, int index, int len) {
     )
 }
 
+/** Holds if index `index` in the format string `fmt` contains an escaped brace `{`. */
 predicate escaped_brace(PossibleAdvancedFormatString fmt, int index) {
     exists(int len | brace_sequence(fmt, index, len) | len % 2 = 0)
 }
 
+/** Holds if index `index` in the format string `fmt` contains a left brace `{` that acts as an escape character. */
 predicate escaping_brace(PossibleAdvancedFormatString fmt, int index) {
     escaped_brace(fmt, index + 1)
 }
@@ -105,15 +109,18 @@ private predicate advanced_format_call(Call format_expr, PossibleAdvancedFormatS
     )
 }
 
+/** A string constant that has the `format` method applied to it. */
 class AdvancedFormatString extends PossibleAdvancedFormatString {
     AdvancedFormatString() { advanced_format_call(_, this, _) }
 }
 
+/** A string formatting operation that uses the `format` method. */
 class AdvancedFormattingCall extends Call {
     AdvancedFormattingCall() { advanced_format_call(this, _, _) }
 
     /** Count of the arguments actually provided */
     int providedArgCount() { advanced_format_call(this, _, result) }
 
+    /** Gets a formatting string for this call. */
     AdvancedFormatString getAFormat() { advanced_format_call(this, result, _) }
 }

python/ql/src/Expressions/IsComparisons.qll

@@ -1,12 +1,16 @@
+/** INTERNAL - Helper predicates for queries that inspect the comparison of objects using `is`. */
+
 import python
 
+/** Holds if the comparison `comp` uses `is` or `is not` (represented as `op`) to compare its `left` and `right` arguments. */
 predicate comparison_using_is(Compare comp, ControlFlowNode left, Cmpop op, ControlFlowNode right) {
     exists(CompareNode fcomp | fcomp = comp.getAFlowNode() |
         fcomp.operands(left, op, right) and
         (op instanceof Is or op instanceof IsNot)
     )
 }
 
+/** Holds if the class `c` overrides the default notion of equality or comparison. */
 predicate overrides_eq_or_cmp(ClassValue c) {
     major_version() = 2 and c.hasAttribute("__eq__")
     or
@@ -19,12 +23,14 @@ predicate overrides_eq_or_cmp(ClassValue c) {
     major_version() = 2 and c.hasAttribute("__cmp__")
 }
 
+/** Holds if the class `cls` is likely to only have a single instance throughout the program. */
 predicate probablySingleton(ClassValue cls) {
     strictcount(Value inst | inst.getClass() = cls) = 1
     or
     cls = Value::named("None").getClass()
 }
 
+/** Holds if using `is` to compare instances of the class `c` is likely to cause unexpected behavior. */
 predicate invalid_to_use_is_portably(ClassValue c) {
     overrides_eq_or_cmp(c) and
     // Exclude type/builtin-function/bool as it is legitimate to compare them using 'is' but they implement __eq__
@@ -35,6 +41,7 @@ predicate invalid_to_use_is_portably(ClassValue c) {
     not probablySingleton(c)
 }
 
+/** Holds if the control flow node `f` points to either `True`, `False`, or `None`. */
 predicate simple_constant(ControlFlowNode f) {
     exists(Value val | f.pointsTo(val) |
         val = Value::named("True") or val = Value::named("False") or val = Value::named("None")
@@ -66,10 +73,12 @@ private predicate universally_interned_value(Expr e) {
     e.(StrConst).getText() = ""
 }
 
+/** Holds if the expression `e` points to an interned constant in CPython. */
 predicate cpython_interned_constant(Expr e) {
     exists(Expr const | e.pointsTo(_, const) | cpython_interned_value(const))
 }
 
+/** Holds if the expression `e` points to a value that can be reasonably expected to be interned across all implementations of Python. */
 predicate universally_interned_constant(Expr e) {
     exists(Expr const | e.pointsTo(_, const) | universally_interned_value(const))
 }
@@ -92,6 +101,9 @@ private predicate comparison_one_type(Compare comp, Cmpop op, ClassValue cls) {
     )
 }
 
+/**
+* Holds if using `is` or `is not` as the operator `op` in the comparison `comp` would be invalid when applied to the class `cls`.
+ */
 predicate invalid_portable_is_comparison(Compare comp, Cmpop op, ClassValue cls) {
     // OK to use 'is' when defining '__eq__'
     not exists(Function eq | eq.getName() = "__eq__" or eq.getName() = "__ne__" |

python/ql/src/Expressions/RedundantComparison.qll

@@ -1,5 +1,8 @@
+/** Helper functions for queries that test redundant comparisons. */
+
 import python
 
+/** A comparison where the left and right hand sides appear to be identical. */
 class RedundantComparison extends Compare {
     RedundantComparison() {
         exists(Expr left, Expr right |
@@ -8,6 +11,15 @@ class RedundantComparison extends Compare {
         )
     }
 
+    /** Holds if this comparison could be redundant due to a missing `self.`, for example
+     * ```python
+     * foo == foo
+     * ```
+     * instead of
+     * ```python
+     * self.foo == foo
+     * ```
+     */
     predicate maybeMissingSelf() {
         exists(Name left |
             this.compares(left, _, _) and

python/ql/src/Lexical/CommentedOutCode.qll

@@ -191,10 +191,19 @@ class CommentedOutCodeBlock extends @py_comment {
     /** The length of this comment block (in comments) */
     int length() { result = count(Comment c | this.contains(c)) }
 
-    predicate hasLocationInfo(string filepath, int bl, int bc, int el, int ec) {
-        this.(Comment).getLocation().hasLocationInfo(filepath, bl, bc, _, _) and
+    /**
+     * Holds if this element is at the specified location.
+     * The location spans column `startcolumn` of line `startline` to
+     * column `endcolumn` of line `endline` in file `filepath`.
+     * For more information, see
+     * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+     */
+    predicate hasLocationInfo(
+        string filepath, int startline, int startcolumn, int endline, int endcolumn
+    ) {
+        this.(Comment).getLocation().hasLocationInfo(filepath, startline, startcolumn, _, _) and
         exists(Comment end | commented_out_code_block(this, end) |
-            end.getLocation().hasLocationInfo(_, _, _, el, ec)
+            end.getLocation().hasLocationInfo(_, _, _, endline, endcolumn)
         )
     }
 

python/ql/src/Metrics/Internal/Extents.qll

@@ -15,9 +15,17 @@ import python
  * including the body (if any), as opposed to the location of its name only.
  */
 class RangeFunction extends Function {
-    predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
-        super.getLocation().hasLocationInfo(path, sl, sc, _, _) and
-        this.getBody().getLastItem().getLocation().hasLocationInfo(path, _, _, el, ec)
+    /**
+     * Holds if this element is at the specified location.
+     * The location spans column `startcolumn` of line `startline` to
+     * column `endcolumn` of line `endline` in file `filepath`.
+     * For more information, see
+     * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+     */
+    predicate hasLocationInfo(
+        string filepath, int startline, int startcolumn, int endline, int endcolumn
+    ) {        super.getLocation().hasLocationInfo(filepath, startline, startcolumn, _, _) and
+        this.getBody().getLastItem().getLocation().hasLocationInfo(filepath, _, _, endline, endcolumn)
     }
 }
 
@@ -26,8 +34,16 @@ class RangeFunction extends Function {
  * including the body (if any), as opposed to the location of its name only.
  */
 class RangeClass extends Class {
-    predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
-        super.getLocation().hasLocationInfo(path, sl, sc, _, _) and
-        this.getBody().getLastItem().getLocation().hasLocationInfo(path, _, _, el, ec)
+    /**
+     * Holds if this element is at the specified location.
+     * The location spans column `startcolumn` of line `startline` to
+     * column `endcolumn` of line `endline` in file `filepath`.
+     * For more information, see
+     * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+     */
+    predicate hasLocationInfo(
+        string filepath, int startline, int startcolumn, int endline, int endcolumn
+    ) {        super.getLocation().hasLocationInfo(filepath, startline, startcolumn, _, _) and
+        this.getBody().getLastItem().getLocation().hasLocationInfo(filepath, _, _, endline, endcolumn)
     }
 }

python/ql/src/Resources/FileOpen.qll

@@ -1,3 +1,5 @@
+/** Contains predicates concerning when and where files are opened and closed. */
+
 import python
 import semmle.python.GuardedControlFlow
 import semmle.python.pointsto.Filters
@@ -113,19 +115,22 @@ predicate close_method_call(CallNode call, ControlFlowNode self) {
     call.getFunction().(AttrNode).getObject("close") = self
 }
 
+/** Holds if `close` is a function that appears to close files that are passed to it as an argument. */
 predicate function_closes_file(FunctionValue close) {
     close = Value::named("os.close")
     or
     function_should_close_parameter(close.getScope())
 }
 
+/** INTERNAL - Helper predicate for `function_closes_file` */
 predicate function_should_close_parameter(Function func) {
     exists(EssaDefinition def |
         closes_file(def) and
         def.getSourceVariable().(Variable).getScope() = func
     )
 }
 
+/** Holds if the function `f` opens a file, either directly or indirectly. */
 predicate function_opens_file(FunctionValue f) {
     f = Value::named("open")
     or
@@ -140,6 +145,7 @@ predicate function_opens_file(FunctionValue f) {
     )
 }
 
+/** Holds if the variable `v` refers to a file opened at `open` which is subsequently returned from a function. */
 predicate file_is_returned(EssaVariable v, ControlFlowNode open) {
     exists(NameNode n, Return ret |
         var_is_open(v, open) and

python/ql/src/analysis/DefinitionTracking.qll

@@ -468,7 +468,13 @@ Definition getUniqueDefinition(Expr use) {
 /** Helper class to get suitable locations for attributes */
 class NiceLocationExpr extends @py_expr {
     string toString() { result = this.(Expr).toString() }
-
+    /**
+     * Holds if this element is at the specified location.
+     * The location spans column `bc` of line `bl` to
+     * column `ec` of line `el` in file `f`.
+     * For more information, see
+     * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+     */
     predicate hasLocationInfo(string f, int bl, int bc, int el, int ec) {
         /* Attribute location for x.y is that of 'y' so that url does not overlap with that of 'x' */
         exists(int abl, int abc | this.(Attribute).getLocation().hasLocationInfo(f, abl, abc, el, ec) |

python/ql/src/external/DefectFilter.qll

@@ -26,7 +26,9 @@ class DefectResult extends int {
 
     /** Gets the file in which this query result was reported. */
     File getFile() {
-        exists(string path | defectResults(this, _, path, _, _, _, _, _) and result.getAbsolutePath() = path)
+        exists(string path |
+            defectResults(this, _, path, _, _, _, _, _) and result.getAbsolutePath() = path
+        )
     }
 
     /** Gets the file path in which this query result was reported. */
@@ -47,8 +49,17 @@ class DefectResult extends int {
     /** Gets the message associated with this query result. */
     string getMessage() { defectResults(this, _, _, _, _, _, _, result) }
 
-    predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
-        defectResults(this, _, path, sl, sc, el, ec, _)
+    /**
+     * Holds if this element is at the specified location.
+     * The location spans column `startcolumn` of line `startline` to
+     * column `endcolumn` of line `endline` in file `filepath`.
+     * For more information, see
+     * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+     */
+    predicate hasLocationInfo(
+        string filepath, int startline, int startcolumn, int endline, int endcolumn
+    ) {
+        defectResults(this, _, filepath, startline, startcolumn, endline, endcolumn, _)
     }
 
     /** Gets the URL corresponding to the location of this query result. */

python/ql/src/external/ExternalArtifact.qll

@@ -1,3 +1,7 @@
+/**
+ * Provides classes for working with external data.
+ */
+
 import python
 
 class ExternalDefect extends @externalDefect {
@@ -27,23 +31,38 @@ class ExternalMetric extends @externalMetric {
     string toString() { result = getQueryPath() + ": " + getLocation() + " - " + getValue() }
 }
 
+/**
+ * An external data item.
+ */
 class ExternalData extends @externalDataElement {
+    /** Gets the path of the file this data was loaded from. */
     string getDataPath() { externalData(this, result, _, _) }
 
+    /**
+     * Gets the path of the file this data was loaded from, with its
+     * extension replaced by `.ql`.
+     */
     string getQueryPath() { result = getDataPath().regexpReplaceAll("\\.[^.]*$", ".ql") }
 
+    /** Gets the number of fields in this data item. */
     int getNumFields() { result = 1 + max(int i | externalData(this, _, i, _) | i) }
 
+    /** Gets the value of the field at position `index` of this data item. */
     string getField(int index) { externalData(this, _, index, result) }
 
+    /** Gets the integer value of the field at position `index` of this data item. */
     int getFieldAsInt(int index) { result = getField(index).toInt() }
 
+    /** Gets the floating-point value of the field at position `index` of this data item. */
     float getFieldAsFloat(int index) { result = getField(index).toFloat() }
 
+    /** Gets the value of the field at position `index` of this data item, interpreted as a date. */
     date getFieldAsDate(int index) { result = getField(index).toDate() }
 
+    /** Gets a textual representation of this data item. */
     string toString() { result = getQueryPath() + ": " + buildTupleString(0) }
 
+    /** Gets a textual representation of this data item, starting with the field at position `start`. */
     private string buildTupleString(int start) {
         start = getNumFields() - 1 and result = getField(start)
         or