Python: Highlight problem with flow summaries and TAttributeContent

Author: RasmusWL

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll

@@ -10,6 +10,7 @@ import LocalSources
 private import semmle.python.essa.SsaCompute
 private import semmle.python.dataflow.new.internal.ImportStar
 private import FlowSummaryImpl as FlowSummaryImpl
+private import semmle.python.frameworks.data.ModelsAsData
 
 /**
  * IPA type for data flow nodes.
@@ -597,7 +598,30 @@ newtype TContent =
   /** An element of a dictionary under any key. */
   TDictionaryElementAnyContent() or
   /** An object attribute. */
-  TAttributeContent(string attr) { attr = any(Attribute a).getName() }
+  TAttributeContent(string attr) {
+    attr = any(Attribute a).getName()
+    or
+    // Flow summaries that target attributes rely on a TAttributeContent being
+    // available. However, since the code above only constructs a TAttributeContent
+    // based on the attribute names seen in the DB, we can end up in a scenario where
+    // flow summaries don't work due to missing TAttributeContent. To get around this,
+    // we need to add the attribute names used by flow summaries. This needs to be done
+    // both for the summaries written in QL and the ones written in data-extension
+    // files.
+    //
+    // 1) Summaries in QL. Sadly the following code leads to non-monotonic recursion
+    //   name = any(AccessPathToken a).getAnArgument("Attribute")
+    // instead we use a qltest to alert if we write a new summary in QL that uses an
+    // attribute -- see
+    // python/ql/test/experimental/dataflow/summaries-checks/missing-attribute-content.ql
+    none() // to be filled out in next commit
+    or
+    //
+    // 2) summaries in data-extension files
+    exists(string input, string output | ModelOutput::relevantSummaryModel(_, _, input, output, _) |
+      attr = [input, output].regexpFind("(?<=(^|\\.)Attribute\\[)[^\\]]+(?=\\])", _, _).trim()
+    )
+  }
 
 /**
  * A data-flow value can have associated content.

python/ql/test/experimental/dataflow/summaries-checks/dummy.py

@@ -0,0 +1 @@
+# an empty file, since we want the test to run on an empty db

python/ql/test/experimental/dataflow/summaries-checks/invalid-spec.expected

@@ -0,0 +1,14 @@
+| compiled re.Match | Argument[self].Attribute[pattern] | Attribute[pattern] |
+| compiled re.Match | ReturnValue.Attribute[re].Attribute[pattern] | Attribute[pattern] |
+| compiled re.Match | ReturnValue.Attribute[re].Attribute[pattern] | Attribute[re] |
+| compiled re.Match | ReturnValue.Attribute[string] | Attribute[string] |
+| compiled re.subn | ReturnValue.TupleElement[0] | TupleElement[0] |
+| re.Match | ReturnValue.Attribute[re].Attribute[pattern] | Attribute[pattern] |
+| re.Match | ReturnValue.Attribute[re].Attribute[pattern] | Attribute[re] |
+| re.Match | ReturnValue.Attribute[string] | Attribute[string] |
+| re.Match.expand | Argument[self].Attribute[string] | Attribute[string] |
+| re.Match.group | Argument[self].Attribute[string] | Attribute[string] |
+| re.Match.groupdict | Argument[self].Attribute[string] | Attribute[string] |
+| re.Match.groups | Argument[self].Attribute[string] | Attribute[string] |
+| re.Pattern | ReturnValue.Attribute[pattern] | Attribute[pattern] |
+| re.subn | ReturnValue.TupleElement[0] | TupleElement[0] |

python/ql/test/experimental/dataflow/summaries-checks/invalid-spec.ql

@@ -0,0 +1,8 @@
+import python
+import semmle.python.dataflow.new.FlowSummary
+import semmle.python.dataflow.new.internal.FlowSummaryImpl
+
+query predicate invalidSpecComponent(SummarizedCallable sc, string s, string c) {
+  (sc.propagatesFlowExt(s, _, _) or sc.propagatesFlowExt(_, s, _)) and
+  Private::External::invalidSpecComponent(s, c)
+}

python/ql/test/experimental/dataflow/summaries-checks/missing-attribute-content.expected

@@ -0,0 +1,3 @@
+| The attribute "pattern" is not a valid TAttributeContent, please add it to the hardcoded list of TAttributeContent in the dataflow library. |
+| The attribute "re" is not a valid TAttributeContent, please add it to the hardcoded list of TAttributeContent in the dataflow library. |
+| The attribute "string" is not a valid TAttributeContent, please add it to the hardcoded list of TAttributeContent in the dataflow library. |

python/ql/test/experimental/dataflow/summaries-checks/missing-attribute-content.ql

@@ -0,0 +1,11 @@
+import python
+import semmle.python.dataflow.new.FlowSummary
+import semmle.python.dataflow.new.internal.FlowSummaryImpl
+
+from SummarizedCallable sc, string s, string c, string attr
+where
+  (sc.propagatesFlowExt(s, _, _) or sc.propagatesFlowExt(_, s, _)) and
+  Private::External::invalidSpecComponent(s, c) and
+  c = "Attribute[" + attr + "]"
+select "The attribute \"" + attr +
+    "\" is not a valid TAttributeContent, please add it to the hardcoded list of TAttributeContent in the dataflow library."