Merge branch 'main' into swift-to-string

Author: MathiasVP

.codeqlmanifest.json

@@ -12,6 +12,7 @@ on:
     paths:
       - "javascript/ql/experimental/adaptivethreatmodeling/**"
       - .github/workflows/js-ml-tests.yml
+  workflow_dispatch:
 
 defaults:
   run:

.github/workflows/js-ml-tests.yml

@@ -0,0 +1,32 @@
+provide:
+  - "*/ql/src/qlpack.yml"
+  - "*/ql/lib/qlpack.yml"
+  - "*/ql/test/qlpack.yml"
+  - "*/ql/examples/qlpack.yml"
+  - "*/ql/consistency-queries/qlpack.yml"
+  - "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml"
+  - "go/ql/config/legacy-support/qlpack.yml"
+  - "go/build/codeql-extractor-go/codeql-extractor.yml"
+  - "javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml"
+  # This pack is explicitly excluded from the workspace since most users
+  # will want to use a version of this pack from the package cache. Internal
+  # users can uncomment the following line and place a custom ML model
+  # in the corresponding pack to test a custom ML model within their local
+  # checkout.
+  # - "javascript/ql/experimental/adaptivethreatmodeling/model/qlpack.yml"
+  - "javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml"
+  - "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml"
+  - "csharp/ql/campaigns/Solorigate/lib/qlpack.yml"
+  - "csharp/ql/campaigns/Solorigate/src/qlpack.yml"
+  - "csharp/ql/campaigns/Solorigate/test/qlpack.yml"
+  - "misc/legacy-support/*/qlpack.yml"
+  - "misc/suite-helpers/qlpack.yml"
+  - "ruby/extractor-pack/codeql-extractor.yml"
+  - "swift/extractor-pack/codeql-extractor.yml"
+  - "ql/extractor-pack/codeql-extractor.ym"
+
+versionPolicies:
+  default:
+    requireChangeNotes: true
+    committedPrereleaseSuffix: dev
+    committedVersion: nextPatchRelease

codeql-workspace.yml

@@ -4,10 +4,14 @@
 import java.io.File;
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Objects;
 import java.util.regex.Pattern;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
 
 import com.github.codeql.Logger;
 import static com.github.codeql.ClassNamesKt.getIrDeclBinaryName;
@@ -547,6 +551,51 @@ else if (majorVersion==0)
 					(tcv.majorVersion == majorVersion && tcv.minorVersion == minorVersion &&
 					tcv.lastModified < lastModified);
 		}
+
+        private static Map<String, Map<String, Long>> jarFileEntryTimeStamps = new HashMap<>();
+
+        private static Map<String, Long> getZipFileEntryTimeStamps(String path, Logger log) {
+            try {
+                Map<String, Long> result = new HashMap<>();
+                ZipFile zf = new ZipFile(path);
+                Enumeration<? extends ZipEntry> entries = zf.entries();
+                while (entries.hasMoreElements()) {
+                    ZipEntry ze = entries.nextElement();
+                    result.put(ze.getName(), ze.getLastModifiedTime().toMillis());
+                }
+                return result;
+            } catch(IOException e) {
+                log.warn("Failed to get entry timestamps from " + path, e);
+                return null;
+            }
+        }
+
+        private static long getVirtualFileTimeStamp(VirtualFile vf, Logger log) {
+            if (vf.getFileSystem().getProtocol().equals("jar")) {
+                String[] parts = vf.getPath().split("!/");
+                if (parts.length == 2) {
+                    String jarFilePath = parts[0];
+                    String entryPath = parts[1];
+                    if (!jarFileEntryTimeStamps.containsKey(jarFilePath)) {
+                        jarFileEntryTimeStamps.put(jarFilePath, getZipFileEntryTimeStamps(jarFilePath, log));
+                    }
+                    Map<String, Long> entryTimeStamps = jarFileEntryTimeStamps.get(jarFilePath);
+                    if (entryTimeStamps != null) {
+                        Long entryTimeStamp = entryTimeStamps.get(entryPath);
+                        if (entryTimeStamp != null)
+                            return entryTimeStamp;
+                        else
+                            log.warn("Couldn't find timestamp for jar file " + jarFilePath + " entry " + entryPath);
+                    }
+                } else {
+                    log.warn("Expected JAR-file path " + vf.getPath() + " to have exactly one '!/' separator");
+                }
+            }
+
+            // For all files except for jar files, and a fallback in case of I/O problems reading a jar file:
+            return vf.getTimeStamp();
+        }
+
 		private static TrapClassVersion fromSymbol(IrDeclaration sym, Logger log) {
 			VirtualFile vf = sym instanceof IrClass ? getIrClassVirtualFile((IrClass)sym) :
 					sym.getParent() instanceof IrClass ? getIrClassVirtualFile((IrClass)sym.getParent()) :
@@ -583,7 +632,7 @@ public void visit(int version, int access, java.lang.String name, java.lang.Stri
 				};
 				(new ClassReader(vf.contentsToByteArray())).accept(versionGetter, ClassReader.SKIP_CODE | ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES);
 
-				return new TrapClassVersion(versionStore[0] & 0xffff, versionStore[0] >> 16, vf.getTimeStamp(), "kotlin");
+				return new TrapClassVersion(versionStore[0] & 0xffff, versionStore[0] >> 16, getVirtualFileTimeStamp(vf, log), "kotlin");
 			}
 			catch(IllegalAccessException e) {
 				log.warn("Failed to read class file version information", e);

java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java

@@ -9,5 +9,7 @@ private class MyConsistencyConfiguration extends ConsistencyConfiguration {
     n instanceof BlockArgumentNode
     or
     n instanceof SummaryNode
+    or
+    n instanceof HashSplatArgumentsNode
   }
 }

ruby/ql/consistency-queries/DataFlowConsistency.ql

@@ -109,7 +109,7 @@ class NamedParameter extends Parameter, TNamedParameter {
   final VariableAccess getDefiningAccess() {
     result = this.getVariable().getDefiningAccess()
     or
-    result = this.(SimpleParameterSynthImpl).getDefininingAccess()
+    result = this.(SimpleParameterSynthImpl).getDefiningAccess()
   }
 
   override AstNode getAChild(string pred) {

ruby/ql/lib/codeql/ruby/ast/Parameter.qll

@@ -116,7 +116,7 @@ class VariableAccess extends Expr instanceof VariableAccessImpl {
   predicate isImplicitWrite() {
     implicitWriteAccess(toGenerated(this))
     or
-    this = any(SimpleParameterSynthImpl p).getDefininingAccess()
+    this = any(SimpleParameterSynthImpl p).getDefiningAccess()
     or
     this = any(HashPattern p).getValue(_)
     or

ruby/ql/lib/codeql/ruby/ast/Variable.qll

@@ -38,7 +38,7 @@ class SimpleParameterRealImpl extends SimpleParameterImpl, TSimpleParameterReal
 class SimpleParameterSynthImpl extends SimpleParameterImpl, TSimpleParameterSynth {
   SimpleParameterSynthImpl() { this = TSimpleParameterSynth(_, _) }
 
-  LocalVariableAccessSynth getDefininingAccess() { synthChild(this, 0, result) }
+  LocalVariableAccessSynth getDefiningAccess() { synthChild(this, 0, result) }
 
   override LocalVariable getVariableImpl() { result = TLocalVariableSynth(this, _) }
 

ruby/ql/lib/codeql/ruby/ast/internal/Parameter.qll

@@ -259,7 +259,8 @@ private module Cached {
       exists(any(Call c).getKeywordArgument(name))
       or
       FlowSummaryImplSpecific::ParsePositions::isParsedKeywordParameterPosition(_, name)
-    }
+    } or
+    THashSplatArgumentPosition()
 
   cached
   newtype TParameterPosition =
@@ -278,6 +279,7 @@ private module Cached {
       or
       FlowSummaryImplSpecific::ParsePositions::isParsedKeywordArgumentPosition(_, name)
     } or
+    THashSplatParameterPosition() or
     TAnyParameterPosition()
 }
 
@@ -476,6 +478,9 @@ class ParameterPosition extends TParameterPosition {
   /** Holds if this position represents a keyword parameter named `name`. */
   predicate isKeyword(string name) { this = TKeywordParameterPosition(name) }
 
+  /** Holds if this position represents a hash-splat parameter. */
+  predicate isHashSplat() { this = THashSplatParameterPosition() }
+
   /**
    * Holds if this position represents any parameter. This includes both positional
    * and named parameters.
@@ -494,6 +499,8 @@ class ParameterPosition extends TParameterPosition {
     or
     exists(string name | this.isKeyword(name) and result = "keyword " + name)
     or
+    this.isHashSplat() and result = "**"
+    or
     this.isAny() and result = "any"
   }
 }
@@ -512,6 +519,12 @@ class ArgumentPosition extends TArgumentPosition {
   /** Holds if this position represents a keyword argument named `name`. */
   predicate isKeyword(string name) { this = TKeywordArgumentPosition(name) }
 
+  /**
+   * Holds if this position represents a synthesized argument containing all keyword
+   * arguments wrapped in a hash.
+   */
+  predicate isHashSplat() { this = THashSplatArgumentPosition() }
+
   /** Gets a textual representation of this position. */
   string toString() {
     this.isSelf() and result = "self"
@@ -521,6 +534,8 @@ class ArgumentPosition extends TArgumentPosition {
     exists(int pos | this.isPositional(pos) and result = "position " + pos)
     or
     exists(string name | this.isKeyword(name) and result = "keyword " + name)
+    or
+    this.isHashSplat() and result = "**"
   }
 }
 
@@ -539,5 +554,7 @@ predicate parameterMatch(ParameterPosition ppos, ArgumentPosition apos) {
   or
   exists(string name | ppos.isKeyword(name) and apos.isKeyword(name))
   or
+  ppos.isHashSplat() and apos.isHashSplat()
+  or
   ppos.isAny() and exists(apos)
 }

ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowDispatch.qll

@@ -179,6 +179,7 @@ private class Argument extends CfgNodes::ExprCfgNode {
       this = call.getArgument(i) and
       not this.getExpr() instanceof BlockArgument and
       not this.getExpr().(Pair).getKey().getConstantValue().isSymbol(_) and
+      not this.getExpr() instanceof HashSplatExpr and
       arg.isPositional(i)
     )
     or
@@ -189,6 +190,10 @@ private class Argument extends CfgNodes::ExprCfgNode {
     )
     or
     this = call.getReceiver() and arg.isSelf()
+    or
+    this = call.getAnArgument() and
+    this.getExpr() instanceof HashSplatExpr and
+    arg.isHashSplat()
   }
 
   /** Holds if this expression is the `i`th argument of `c`. */
@@ -216,7 +221,8 @@ private module Cached {
     TNormalParameterNode(Parameter p) {
       p instanceof SimpleParameter or
       p instanceof OptionalParameter or
-      p instanceof KeywordParameter
+      p instanceof KeywordParameter or
+      p instanceof HashSplatParameter
     } or
     TSelfParameterNode(MethodBase m) or
     TBlockParameterNode(MethodBase m) or
@@ -232,6 +238,9 @@ private module Cached {
     } or
     TSummaryParameterNode(FlowSummaryImpl::Public::SummarizedCallable c, ParameterPosition pos) {
       FlowSummaryImpl::Private::summaryParameterNodeRange(c, pos)
+    } or
+    THashSplatArgumentsNode(CfgNodes::ExprNodes::CallCfgNode c) {
+      exists(Argument arg | arg.isArgumentOf(c, any(ArgumentPosition pos | pos.isKeyword(_))))
     }
 
   class TParameterNode =
@@ -389,6 +398,8 @@ predicate nodeIsHidden(Node n) {
   n instanceof SummaryParameterNode
   or
   n instanceof SynthReturnNode
+  or
+  n instanceof HashSplatArgumentsNode
 }
 
 /** An SSA definition, viewed as a node in a data flow graph. */
@@ -473,6 +484,9 @@ private module ParameterNodes {
           c.getAParameter() = kp and
           pos.isKeyword(kp.getName())
         )
+      or
+      parameter = c.getAParameter().(HashSplatParameter) and
+      pos.isHashSplat()
     }
 
     override CfgScope getCfgScope() { result = parameter.getCallable() }
@@ -651,6 +665,40 @@ private module ArgumentNodes {
       FlowSummaryImpl::Private::summaryArgumentNode(call, this, pos)
     }
   }
+
+  /**
+   * A data-flow node that represents all keyword arguments wrapped in a hash.
+   *
+   * The callee is responsible for filtering out the keyword arguments that are
+   * part of the method signature, such that those cannot end up in the hash-splat
+   * parameter.
+   */
+  class HashSplatArgumentsNode extends ArgumentNode, THashSplatArgumentsNode {
+    CfgNodes::ExprNodes::CallCfgNode c;
+
+    HashSplatArgumentsNode() { this = THashSplatArgumentsNode(c) }
+
+    override predicate argumentOf(DataFlowCall call, ArgumentPosition pos) {
+      this.sourceArgumentOf(call.asCall(), pos)
+    }
+
+    override predicate sourceArgumentOf(CfgNodes::ExprNodes::CallCfgNode call, ArgumentPosition pos) {
+      call = c and
+      pos.isHashSplat()
+    }
+  }
+
+  private class HashSplatArgumentsNodeImpl extends NodeImpl, THashSplatArgumentsNode {
+    CfgNodes::ExprNodes::CallCfgNode c;
+
+    HashSplatArgumentsNodeImpl() { this = THashSplatArgumentsNode(c) }
+
+    override CfgScope getCfgScope() { result = c.getExpr().getCfgScope() }
+
+    override Location getLocationImpl() { result = c.getLocation() }
+
+    override string toStringImpl() { result = "**" }
+  }
 }
 
 import ArgumentNodes
@@ -807,6 +855,13 @@ predicate jumpStep(Node pred, Node succ) {
   succ.asExpr().getExpr().(ConstantReadAccess).getValue() = pred.asExpr().getExpr()
 }
 
+private ContentSet getKeywordContent(string name) {
+  exists(ConstantValue::ConstantSymbolValue key |
+    result.isSingleton(TKnownElementContent(key)) and
+    key.isSymbol(name)
+  )
+}
+
 /**
  * Holds if data can flow from `node1` to `node2` via an assignment to
  * content `c`.
@@ -845,6 +900,14 @@ predicate storeStep(Node node1, ContentSet c, Node node2) {
         c.isSingleton(TUnknownPairValueContent())
       )
     )
+  or
+  // Wrap all keyword arguments in a synthesized hash-splat argument node
+  exists(CfgNodes::ExprNodes::CallCfgNode call, ArgumentPosition keywordPos, string name |
+    node2 = THashSplatArgumentsNode(call) and
+    node1.asExpr().(Argument).isArgumentOf(call, keywordPos) and
+    keywordPos.isKeyword(name) and
+    c = getKeywordContent(name)
+  )
 }
 
 /**
@@ -870,6 +933,19 @@ predicate readStep(Node node1, ContentSet c, Node node2) {
  */
 predicate clearsContent(Node n, ContentSet c) {
   FlowSummaryImpl::Private::Steps::summaryClearsContent(n, c)
+  or
+  // Filter out keyword arguments that are part of the method signature from
+  // the hash-splat parameter
+  exists(
+    DataFlowCallable callable, ParameterPosition hashSplatPos, ParameterNodeImpl keywordParam,
+    ParameterPosition keywordPos, string name
+  |
+    n.(ParameterNodes::NormalParameterNode).isParameterOf(callable, hashSplatPos) and
+    hashSplatPos.isHashSplat() and
+    keywordParam.isParameterOf(callable, keywordPos) and
+    keywordPos.isKeyword(name) and
+    c = getKeywordContent(name)
+  )
 }
 
 /**