Merge pull request #9025 from michaelnebel/csharp/generatedrefactor

C#: Provenance column in Models as Data CSV format.

Author: michaelnebel

csharp/ql/lib/semmle/code/csharp/dataflow/ExternalFlow.qll

@@ -69,6 +69,12 @@
  *    sources "remote" indicates a default remote flow source, and for summaries
  *    "taint" indicates a default additional taint step and "value" indicates a
  *    globally applicable value-preserving step.
+ * 9. The `provenance` column is a tag to indicate the origin of the summary.
+ *    There are two supported values: "generated" and "manual". "generated" means that
+ *    the model has been emitted by the model generator tool and "manual" means
+ *    that the model has been written by hand. This information is used in a heuristic
+ *    for dataflow analysis to determine, if a model or source code should be used for
+ *    determining flow.
  */
 
 import csharp
@@ -163,17 +169,10 @@ private predicate sinkModel(string row) { any(SinkModelCsv s).row(row) }
 
 private predicate summaryModel(string row) { any(SummaryModelCsv s).row(row) }
 
-bindingset[input]
-private predicate getKind(string input, string kind, boolean generated) {
-  input.splitAt(":", 0) = "generated" and kind = input.splitAt(":", 1) and generated = true
-  or
-  not input.matches("%:%") and kind = input and generated = false
-}
-
 /** Holds if a source model exists for the given parameters. */
 predicate sourceModel(
   string namespace, string type, boolean subtypes, string name, string signature, string ext,
-  string output, string kind, boolean generated
+  string output, string kind, string provenance
 ) {
   exists(string row |
     sourceModel(row) and
@@ -185,14 +184,15 @@ predicate sourceModel(
     row.splitAt(";", 4) = signature and
     row.splitAt(";", 5) = ext and
     row.splitAt(";", 6) = output and
-    exists(string k | row.splitAt(";", 7) = k and getKind(k, kind, generated))
+    row.splitAt(";", 7) = kind and
+    row.splitAt(";", 8) = provenance
   )
 }
 
 /** Holds if a sink model exists for the given parameters. */
 predicate sinkModel(
   string namespace, string type, boolean subtypes, string name, string signature, string ext,
-  string input, string kind, boolean generated
+  string input, string kind, string provenance
 ) {
   exists(string row |
     sinkModel(row) and
@@ -204,14 +204,15 @@ predicate sinkModel(
     row.splitAt(";", 4) = signature and
     row.splitAt(";", 5) = ext and
     row.splitAt(";", 6) = input and
-    exists(string k | row.splitAt(";", 7) = k and getKind(k, kind, generated))
+    row.splitAt(";", 7) = kind and
+    row.splitAt(";", 8) = provenance
   )
 }
 
 /** Holds if a summary model exists for the given parameters. */
 predicate summaryModel(
   string namespace, string type, boolean subtypes, string name, string signature, string ext,
-  string input, string output, string kind, boolean generated
+  string input, string output, string kind, string provenance
 ) {
   exists(string row |
     summaryModel(row) and
@@ -224,7 +225,8 @@ predicate summaryModel(
     row.splitAt(";", 5) = ext and
     row.splitAt(";", 6) = input and
     row.splitAt(";", 7) = output and
-    exists(string k | row.splitAt(";", 8) = k and getKind(k, kind, generated))
+    row.splitAt(";", 8) = kind and
+    row.splitAt(";", 9) = provenance
   )
 }
 
@@ -259,25 +261,25 @@ predicate modelCoverage(string namespace, int namespaces, string kind, string pa
     part = "source" and
     n =
       strictcount(string subns, string type, boolean subtypes, string name, string signature,
-        string ext, string output, boolean generated |
+        string ext, string output, string provenance |
         canonicalNamespaceLink(namespace, subns) and
-        sourceModel(subns, type, subtypes, name, signature, ext, output, kind, generated)
+        sourceModel(subns, type, subtypes, name, signature, ext, output, kind, provenance)
       )
     or
     part = "sink" and
     n =
       strictcount(string subns, string type, boolean subtypes, string name, string signature,
-        string ext, string input, boolean generated |
+        string ext, string input, string provenance |
         canonicalNamespaceLink(namespace, subns) and
-        sinkModel(subns, type, subtypes, name, signature, ext, input, kind, generated)
+        sinkModel(subns, type, subtypes, name, signature, ext, input, kind, provenance)
       )
     or
     part = "summary" and
     n =
       strictcount(string subns, string type, boolean subtypes, string name, string signature,
-        string ext, string input, string output, boolean generated |
+        string ext, string input, string output, string provenance |
         canonicalNamespaceLink(namespace, subns) and
-        summaryModel(subns, type, subtypes, name, signature, ext, input, output, kind, generated)
+        summaryModel(subns, type, subtypes, name, signature, ext, input, output, kind, provenance)
       )
   )
 }
@@ -286,12 +288,16 @@ predicate modelCoverage(string namespace, int namespaces, string kind, string pa
 module CsvValidation {
   /** Holds if some row in a CSV-based flow model appears to contain typos. */
   query predicate invalidModelRow(string msg) {
-    exists(string pred, string namespace, string type, string name, string signature, string ext |
-      sourceModel(namespace, type, _, name, signature, ext, _, _, _) and pred = "source"
+    exists(
+      string pred, string namespace, string type, string name, string signature, string ext,
+      string provenance
+    |
+      sourceModel(namespace, type, _, name, signature, ext, _, _, provenance) and pred = "source"
       or
-      sinkModel(namespace, type, _, name, signature, ext, _, _, _) and pred = "sink"
+      sinkModel(namespace, type, _, name, signature, ext, _, _, provenance) and pred = "sink"
       or
-      summaryModel(namespace, type, _, name, signature, ext, _, _, _, _) and pred = "summary"
+      summaryModel(namespace, type, _, name, signature, ext, _, _, _, provenance) and
+      pred = "summary"
     |
       not namespace.regexpMatch("[a-zA-Z0-9_\\.]+") and
       msg = "Dubious namespace \"" + namespace + "\" in " + pred + " model."
@@ -307,6 +313,9 @@ module CsvValidation {
       or
       not ext.regexpMatch("|Attribute") and
       msg = "Unrecognized extra API graph element \"" + ext + "\" in " + pred + " model."
+      or
+      not provenance = ["manual", "generated"] and
+      msg = "Unrecognized provenance description \"" + provenance + "\" in " + pred + " model."
     )
     or
     exists(string pred, AccessPath input, string part |
@@ -338,18 +347,18 @@ module CsvValidation {
     )
     or
     exists(string pred, string row, int expect |
-      sourceModel(row) and expect = 8 and pred = "source"
+      sourceModel(row) and expect = 9 and pred = "source"
       or
-      sinkModel(row) and expect = 8 and pred = "sink"
+      sinkModel(row) and expect = 9 and pred = "sink"
       or
-      summaryModel(row) and expect = 9 and pred = "summary"
+      summaryModel(row) and expect = 10 and pred = "summary"
     |
       exists(int cols |
         cols = 1 + max(int n | exists(row.splitAt(";", n))) and
         cols != expect and
         msg =
           "Wrong number of columns in " + pred + " model row, expected " + expect + ", got " + cols +
-            "."
+            " in " + row + "."
       )
       or
       exists(string b |
@@ -359,23 +368,20 @@ module CsvValidation {
       )
     )
     or
-    exists(string row, string k, string kind | summaryModel(row) |
-      k = row.splitAt(";", 8) and
-      getKind(k, kind, _) and
+    exists(string row, string kind | summaryModel(row) |
+      kind = row.splitAt(";", 8) and
       not kind = ["taint", "value"] and
       msg = "Invalid kind \"" + kind + "\" in summary model."
     )
     or
-    exists(string row, string k, string kind | sinkModel(row) |
-      k = row.splitAt(";", 7) and
-      getKind(k, kind, _) and
+    exists(string row, string kind | sinkModel(row) |
+      kind = row.splitAt(";", 7) and
       not kind = ["code", "sql", "xss", "remote", "html"] and
       msg = "Invalid kind \"" + kind + "\" in sink model."
     )
     or
-    exists(string row, string k, string kind | sourceModel(row) |
-      k = row.splitAt(";", 7) and
-      getKind(k, kind, _) and
+    exists(string row, string kind | sourceModel(row) |
+      kind = row.splitAt(";", 7) and
       not kind = "local" and
       msg = "Invalid kind \"" + kind + "\" in source model."
     )

csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll

@@ -1116,13 +1116,13 @@ module Private {
       preservesValue = false and result = "taint"
     }
 
-    private string renderGenerated(RelevantSummarizedCallable c) {
-      if c.(SummarizedCallable).isAutoGenerated() then result = "generated:" else result = ""
+    private string renderProvenance(RelevantSummarizedCallable c) {
+      if c.(SummarizedCallable).isAutoGenerated() then result = "generated" else result = "manual"
     }
 
     /**
      * A query predicate for outputting flow summaries in semi-colon separated format in QL tests.
-     * The syntax is: "namespace;type;overrides;name;signature;ext;inputspec;outputspec;(generated:)?kind",
+     * The syntax is: "namespace;type;overrides;name;signature;ext;inputspec;outputspec;kind;provenance"",
      * ext is hardcoded to empty.
      */
     query predicate summary(string csv) {
@@ -1133,7 +1133,7 @@ module Private {
         c.relevantSummary(input, output, preservesValue) and
         csv =
           c.getCallableCsv() + getComponentStackCsv(input) + ";" + getComponentStackCsv(output) +
-            ";" + renderGenerated(c) + renderKind(preservesValue)
+            ";" + renderKind(preservesValue) + ";" + renderProvenance(c)
       )
     }
   }

csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImplSpecific.qll

@@ -91,16 +91,25 @@ DataFlowType getCallbackReturnType(DataFlowType t, ReturnKind rk) {
   )
 }
 
+bindingset[provenance]
+private boolean isGenerated(string provenance) {
+  provenance = "generated" and result = true
+  or
+  provenance != "generated" and result = false
+}
+
 /**
  * Holds if an external flow summary exists for `c` with input specification
  * `input`, output specification `output`, kind `kind`, and a flag `generated`
  * stating whether the summary is autogenerated.
  */
 predicate summaryElement(Callable c, string input, string output, string kind, boolean generated) {
   exists(
-    string namespace, string type, boolean subtypes, string name, string signature, string ext
+    string namespace, string type, boolean subtypes, string name, string signature, string ext,
+    string provenance
   |
-    summaryModel(namespace, type, subtypes, name, signature, ext, input, output, kind, generated) and
+    summaryModel(namespace, type, subtypes, name, signature, ext, input, output, kind, provenance) and
+    generated = isGenerated(provenance) and
     c = interpretElement(namespace, type, subtypes, name, signature, ext)
   )
 }
@@ -112,9 +121,11 @@ predicate summaryElement(Callable c, string input, string output, string kind, b
  */
 predicate sourceElement(Element e, string output, string kind, boolean generated) {
   exists(
-    string namespace, string type, boolean subtypes, string name, string signature, string ext
+    string namespace, string type, boolean subtypes, string name, string signature, string ext,
+    string provenance
   |
-    sourceModel(namespace, type, subtypes, name, signature, ext, output, kind, generated) and
+    sourceModel(namespace, type, subtypes, name, signature, ext, output, kind, provenance) and
+    generated = isGenerated(provenance) and
     e = interpretElement(namespace, type, subtypes, name, signature, ext)
   )
 }
@@ -126,9 +137,11 @@ predicate sourceElement(Element e, string output, string kind, boolean generated
  */
 predicate sinkElement(Element e, string input, string kind, boolean generated) {
   exists(
-    string namespace, string type, boolean subtypes, string name, string signature, string ext
+    string namespace, string type, boolean subtypes, string name, string signature, string ext,
+    string provenance
   |
-    sinkModel(namespace, type, subtypes, name, signature, ext, input, kind, generated) and
+    sinkModel(namespace, type, subtypes, name, signature, ext, input, kind, provenance) and
+    generated = isGenerated(provenance) and
     e = interpretElement(namespace, type, subtypes, name, signature, ext)
   )
 }

csharp/ql/lib/semmle/code/csharp/frameworks/EntityFramework.qll

@@ -240,7 +240,7 @@ module EntityFramework {
   private class SystemDataEntityDbSetSqlQuerySinkModelCsv extends SinkModelCsv {
     override predicate row(string row) {
       row =
-        "System.Data.Entity;DbSet;false;SqlQuery;(System.String,System.Object[]);;Argument[0];sql"
+        "System.Data.Entity;DbSet;false;SqlQuery;(System.String,System.Object[]);;Argument[0];sql;manual"
     }
   }
 
@@ -249,14 +249,14 @@ module EntityFramework {
     override predicate row(string row) {
       row =
         [
-          "System.Data.Entity;Database;false;SqlQuery;(System.Type,System.String,System.Object[]);;Argument[1];sql",
-          "System.Data.Entity;Database;false;SqlQuery<>;(System.String,System.Object[]);;Argument[0];sql",
-          "System.Data.Entity;Database;false;ExecuteSqlCommand;(System.String,System.Object[]);;Argument[0];sql",
-          "System.Data.Entity;Database;false;ExecuteSqlCommand;(System.Data.Entity.TransactionalBehavior,System.String,System.Object[]);;Argument[1];sql",
-          "System.Data.Entity;Database;false;ExecuteSqlCommandAsync;(System.Data.Entity.TransactionalBehavior,System.String,System.Threading.CancellationToken,System.Object[]);;Argument[1];sql",
-          "System.Data.Entity;Database;false;ExecuteSqlCommandAsync;(System.String,System.Threading.CancellationToken,System.Object[]);;Argument[0];sql",
-          "System.Data.Entity;Database;false;ExecuteSqlCommandAsync;(System.String,System.Object[]);;Argument[0];sql",
-          "System.Data.Entity;Database;false;ExecuteSqlCommandAsync;(System.Data.Entity.TransactionalBehavior,System.String,System.Object[]);;Argument[1];sql"
+          "System.Data.Entity;Database;false;SqlQuery;(System.Type,System.String,System.Object[]);;Argument[1];sql;manual",
+          "System.Data.Entity;Database;false;SqlQuery<>;(System.String,System.Object[]);;Argument[0];sql;manual",
+          "System.Data.Entity;Database;false;ExecuteSqlCommand;(System.String,System.Object[]);;Argument[0];sql;manual",
+          "System.Data.Entity;Database;false;ExecuteSqlCommand;(System.Data.Entity.TransactionalBehavior,System.String,System.Object[]);;Argument[1];sql;manual",
+          "System.Data.Entity;Database;false;ExecuteSqlCommandAsync;(System.Data.Entity.TransactionalBehavior,System.String,System.Threading.CancellationToken,System.Object[]);;Argument[1];sql;manual",
+          "System.Data.Entity;Database;false;ExecuteSqlCommandAsync;(System.String,System.Threading.CancellationToken,System.Object[]);;Argument[0];sql;manual",
+          "System.Data.Entity;Database;false;ExecuteSqlCommandAsync;(System.String,System.Object[]);;Argument[0];sql;manual",
+          "System.Data.Entity;Database;false;ExecuteSqlCommandAsync;(System.Data.Entity.TransactionalBehavior,System.String,System.Object[]);;Argument[1];sql;manual"
         ]
     }
   }
@@ -266,7 +266,7 @@ module EntityFramework {
     override predicate row(string row) {
       row =
         [
-          "Microsoft.EntityFrameworkCore;RelationalQueryableExtensions;false;FromSqlRaw<>;(Microsoft.EntityFrameworkCore.DbSet<TEntity>,System.String,System.Object[]);;Argument[1];sql",
+          "Microsoft.EntityFrameworkCore;RelationalQueryableExtensions;false;FromSqlRaw<>;(Microsoft.EntityFrameworkCore.DbSet<TEntity>,System.String,System.Object[]);;Argument[1];sql;manual",
         ]
     }
   }
@@ -276,11 +276,11 @@ module EntityFramework {
     override predicate row(string row) {
       row =
         [
-          "Microsoft.EntityFrameworkCore;RelationalDatabaseFacadeExtensions;false;ExecuteSqlRaw;(Microsoft.EntityFrameworkCore.Infrastructure.DatabaseFacade,System.String,System.Collections.Generic.IEnumerable<System.Object>);;Argument[1];sql",
-          "Microsoft.EntityFrameworkCore;RelationalDatabaseFacadeExtensions;false;ExecuteSqlRaw;(Microsoft.EntityFrameworkCore.Infrastructure.DatabaseFacade,System.String,System.Object[]);;Argument[1];sql",
-          "Microsoft.EntityFrameworkCore;RelationalDatabaseFacadeExtensions;false;ExecuteSqlRawAsync;(Microsoft.EntityFrameworkCore.Infrastructure.DatabaseFacade,System.String,System.Threading.CancellationToken);;Argument[1];sql",
-          "Microsoft.EntityFrameworkCore;RelationalDatabaseFacadeExtensions;false;ExecuteSqlRawAsync;(Microsoft.EntityFrameworkCore.Infrastructure.DatabaseFacade,System.String,System.Object[]);;Argument[1];sql",
-          "Microsoft.EntityFrameworkCore;RelationalDatabaseFacadeExtensions;false;ExecuteSqlRawAsync;(Microsoft.EntityFrameworkCore.Infrastructure.DatabaseFacade,System.String,System.Collections.Generic.IEnumerable<System.Object>,System.Threading.CancellationToken);;Argument[1];sql",
+          "Microsoft.EntityFrameworkCore;RelationalDatabaseFacadeExtensions;false;ExecuteSqlRaw;(Microsoft.EntityFrameworkCore.Infrastructure.DatabaseFacade,System.String,System.Collections.Generic.IEnumerable<System.Object>);;Argument[1];sql;manual",
+          "Microsoft.EntityFrameworkCore;RelationalDatabaseFacadeExtensions;false;ExecuteSqlRaw;(Microsoft.EntityFrameworkCore.Infrastructure.DatabaseFacade,System.String,System.Object[]);;Argument[1];sql;manual",
+          "Microsoft.EntityFrameworkCore;RelationalDatabaseFacadeExtensions;false;ExecuteSqlRawAsync;(Microsoft.EntityFrameworkCore.Infrastructure.DatabaseFacade,System.String,System.Threading.CancellationToken);;Argument[1];sql;manual",
+          "Microsoft.EntityFrameworkCore;RelationalDatabaseFacadeExtensions;false;ExecuteSqlRawAsync;(Microsoft.EntityFrameworkCore.Infrastructure.DatabaseFacade,System.String,System.Object[]);;Argument[1];sql;manual",
+          "Microsoft.EntityFrameworkCore;RelationalDatabaseFacadeExtensions;false;ExecuteSqlRawAsync;(Microsoft.EntityFrameworkCore.Infrastructure.DatabaseFacade,System.String,System.Collections.Generic.IEnumerable<System.Object>,System.Threading.CancellationToken);;Argument[1];sql;manual",
         ]
     }
   }