Data flow: Prune parameter-self flow in stage 1

Author: hvitved

shared/dataflow/codeql/dataflow/internal/DataFlowImpl.qll

@@ -561,13 +561,14 @@ module MakeImpl<LocationSig Location, InputSig<Location> Lang> {
         cc = true
         or
         // flow out of a callable
-        fwdFlowOut(_, node, false) and
+        fwdFlowOut(_, _, node, false) and
         cc = false
         or
         // flow through a callable
-        exists(DataFlowCall call |
-          fwdFlowOutFromArg(call, node) and
-          fwdFlowIsEntered(call, cc)
+        exists(DataFlowCall call, ReturnKindExtOption kind, ReturnKindExtOption disallow |
+          fwdFlowOutFromArg(call, kind, node) and
+          fwdFlowIsEntered(call, disallow, cc) and
+          kind != disallow
         )
       }
 
@@ -597,7 +598,13 @@ module MakeImpl<LocationSig Location, InputSig<Location> Lang> {
        * Holds if an argument to `call` is reached in the flow covered by `fwdFlow`.
        */
       pragma[nomagic]
-      private predicate fwdFlowIsEntered(DataFlowCall call, Cc cc) { fwdFlowIn(call, _, cc, _) }
+      private predicate fwdFlowIsEntered(DataFlowCall call, ReturnKindExtOption disallow, Cc cc) {
+        exists(ParamNodeEx p | fwdFlowIn(call, _, cc, p) |
+          if allowParameterReturnInSelfEx(p)
+          then disallow.isNone()
+          else p.isParameterOf(_, disallow.asSome().(ParamUpdateReturnKind).getPosition())
+        )
+      }
 
       pragma[nomagic]
       private predicate fwdFlowInReducedViableImplInSomeCallContext(
@@ -618,7 +625,7 @@ module MakeImpl<LocationSig Location, InputSig<Location> Lang> {
       pragma[nomagic]
       private DataFlowCallable viableImplInSomeFwdFlowCallContextExt(DataFlowCall call) {
         exists(DataFlowCall ctx |
-          fwdFlowIsEntered(ctx, _) and
+          fwdFlowIsEntered(ctx, _, _) and
           result = viableImplInCallContextExt(call, ctx)
         )
       }
@@ -666,17 +673,18 @@ module MakeImpl<LocationSig Location, InputSig<Location> Lang> {
 
       // inline to reduce the number of iterations
       pragma[inline]
-      private predicate fwdFlowOut(DataFlowCall call, NodeEx out, Cc cc) {
-        exists(ReturnPosition pos |
-          fwdFlowReturnPosition(pos, cc) and
-          viableReturnPosOutEx(call, pos, out) and
-          not fullBarrier(out)
-        )
+      private predicate fwdFlowOut(DataFlowCall call, ReturnPosition pos, NodeEx out, Cc cc) {
+        fwdFlowReturnPosition(pos, cc) and
+        viableReturnPosOutEx(call, pos, out) and
+        not fullBarrier(out)
       }
 
       pragma[nomagic]
-      private predicate fwdFlowOutFromArg(DataFlowCall call, NodeEx out) {
-        fwdFlowOut(call, out, true)
+      private predicate fwdFlowOutFromArg(DataFlowCall call, ReturnKindExtOption kind, NodeEx out) {
+        exists(ReturnPosition pos |
+          fwdFlowOut(call, pos, out, true) and
+          kind.asSome() = pos.getKind()
+        )
       }
 
       private predicate stateStepFwd(FlowState state1, FlowState state2) {
@@ -750,7 +758,7 @@ module MakeImpl<LocationSig Location, InputSig<Location> Lang> {
         )
         or
         // flow into a callable
-        revFlowIn(_, node, false) and
+        revFlowIn(_, _, node, false) and
         toReturn = false
         or
         // flow out of a callable
@@ -761,9 +769,10 @@ module MakeImpl<LocationSig Location, InputSig<Location> Lang> {
         )
         or
         // flow through a callable
-        exists(DataFlowCall call |
-          revFlowInToReturn(call, node) and
-          revFlowIsReturned(call, toReturn)
+        exists(DataFlowCall call, ReturnKindExtOption kind, ReturnKindExtOption disallow |
+          revFlowIsReturned(call, kind, toReturn) and
+          revFlowInToReturn(call, disallow, node) and
+          kind != disallow
         )
       }
 
@@ -824,16 +833,20 @@ module MakeImpl<LocationSig Location, InputSig<Location> Lang> {
 
       // inline to reduce the number of iterations
       pragma[inline]
-      private predicate revFlowIn(DataFlowCall call, ArgNodeEx arg, boolean toReturn) {
-        exists(ParamNodeEx p |
-          revFlow(p, toReturn) and
-          viableParamArgNodeCandFwd1(call, p, arg)
-        )
+      private predicate revFlowIn(DataFlowCall call, ParamNodeEx p, ArgNodeEx arg, boolean toReturn) {
+        revFlow(p, toReturn) and
+        viableParamArgNodeCandFwd1(call, p, arg)
       }
 
       pragma[nomagic]
-      private predicate revFlowInToReturn(DataFlowCall call, ArgNodeEx arg) {
-        revFlowIn(call, arg, true)
+      private predicate revFlowInToReturn(
+        DataFlowCall call, ReturnKindExtOption disallow, ArgNodeEx arg
+      ) {
+        exists(ParamNodeEx p | revFlowIn(call, p, arg, true) |
+          if allowParameterReturnInSelfEx(p)
+          then disallow.isNone()
+          else p.isParameterOf(_, disallow.asSome().(ParamUpdateReturnKind).getPosition())
+        )
       }
 
       /**
@@ -842,10 +855,12 @@ module MakeImpl<LocationSig Location, InputSig<Location> Lang> {
        * reaching an argument of `call`.
        */
       pragma[nomagic]
-      private predicate revFlowIsReturned(DataFlowCall call, boolean toReturn) {
+      private predicate revFlowIsReturned(
+        DataFlowCall call, ReturnKindExtOption kind, boolean toReturn
+      ) {
         exists(NodeEx out |
           revFlow(out, toReturn) and
-          fwdFlowOutFromArg(call, out)
+          fwdFlowOutFromArg(call, kind, out)
         )
       }
 
@@ -947,10 +962,13 @@ module MakeImpl<LocationSig Location, InputSig<Location> Lang> {
 
       pragma[nomagic]
       predicate callMayFlowThroughRev(DataFlowCall call) {
-        exists(ArgNodeEx arg, boolean toReturn |
+        exists(
+          ArgNodeEx arg, ReturnKindExtOption kind, ReturnKindExtOption disallow, boolean toReturn
+        |
           revFlow(arg, toReturn) and
-          revFlowInToReturn(call, arg) and
-          revFlowIsReturned(call, toReturn)
+          revFlowIsReturned(call, pragma[only_bind_into](kind), toReturn) and
+          revFlowInToReturn(call, pragma[only_bind_into](disallow), arg) and
+          kind != disallow
         )
       }
 

shared/dataflow/codeql/dataflow/internal/DataFlowImplCommon.qll

@@ -2417,6 +2417,10 @@ module MakeImplCommon<LocationSig Location, InputSig<Location> Lang> {
     override string toString() { result = "param update " + pos }
   }
 
+  module ReturnKindExtOption = Option<ReturnKindExt>;
+
+  class ReturnKindExtOption = ReturnKindExtOption::Option;
+
   /** A callable tagged with a relevant return kind. */
   class ReturnPosition extends TReturnPosition0 {
     private DataFlowCallable c;