Merge branch 'main' into rb-redosMod

Author: erik-krogh

.github/workflows/compile-queries.yml

@@ -2,49 +2,51 @@ name: "Compile all queries using the latest stable CodeQL CLI"
 
 on:
   push:
-    branches: [main] # makes sure the cache gets populated
-  pull_request:
-    branches:
+    branches:  # makes sure the cache gets populated - running on the branches people tend to merge into.
       - main
       - "rc/*"
+      - "codeql-cli-*"
+  pull_request:
 
 jobs:
   compile-queries:
     runs-on: ubuntu-latest-xl
 
     steps:
       - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
       # calculate the merge-base with main, in a way that works both on PRs and pushes to main. 
       - name: Calculate merge-base
         if: ${{ github.event_name == 'pull_request' }}
         env:
           BASE_BRANCH: ${{ github.base_ref }}
         run: |
-          MERGE_BASE=$(git merge-base --fork-point origin/$BASE_BRANCH)
-          echo "merge-base=$MERGE_BASE" >> $GITHUB_ENV
-      - name: Calculate merge-base - branch
-        if: ${{ github.event_name != 'pull_request' }}
-        # using github.sha instead, since we're directly on a branch, and not in a PR
-        run: |
-          MERGE_BASE=${{ github.sha }}
+          MERGE_BASE=$(git cat-file commit $GITHUB_SHA | grep '^parent ' | head -1 | cut -f 2 -d " ")
           echo "merge-base=$MERGE_BASE" >> $GITHUB_ENV
-      - name: Cache CodeQL query compilation
+      - name: Read CodeQL query compilation - PR
+        if: ${{ github.event_name == 'pull_request' }}
         uses: actions/cache@v3
         with:
           path: '*/ql/src/.cache'
-          # current GH HEAD first, merge-base second, generic third
-          key: codeql-stable-compile-${{ github.sha }}
+          key: codeql-compile-pr-${{ github.sha }} # deliberately not using the `compile-compile-main` keys here.
           restore-keys: |
-            codeql-stable-compile-${{ env.merge-base }}
-            codeql-stable-compile-
+            codeql-compile-${{ github.base_ref }}-${{ env.merge-base }}
+            codeql-compile-${{ github.base_ref }}-
+            codeql-compile-main-
+      - name: Fill CodeQL query compilation cache - main
+        if: ${{ github.event_name != 'pull_request' }}
+        uses: actions/cache@v3
+        with:
+          path: '*/ql/src/.cache'
+          key: codeql-compile-${{ github.ref_name }}-${{ github.sha }} # just fill on main
+          restore-keys: | # restore from another random commit, to speed up compilation.
+            codeql-compile-${{ github.ref_name }}- 
+            codeql-compile-main-
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql
         with:
           channel: 'release'
       - name: check formatting
-        run: codeql query format */ql/{src,lib,test}/**/*.{qll,ql} --check-only
+        run: find */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 codeql query format --check-only
       - name: compile queries - check-only
         # run with --check-only if running in a PR (github.sha != main)
         if : ${{ github.event_name == 'pull_request' }}

.github/workflows/js-ml-tests.yml

@@ -23,19 +23,6 @@ defaults:
     working-directory: javascript/ql/experimental/adaptivethreatmodeling
 
 jobs:
-  qlformat:
-    name: Check QL formatting
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: ./.github/actions/fetch-codeql
-
-      - name: Check QL formatting
-        run: |
-          find . "(" -name "*.ql" -or -name "*.qll" ")" -print0 | \
-            xargs -0 codeql query format --check-only
-
   qlcompile:
     name: Check QL compilation
     runs-on: ubuntu-latest

.github/workflows/ql-for-ql-build.yml

@@ -24,13 +24,13 @@ jobs:
       - uses: actions/checkout@v3
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
+        uses: github/codeql-action/init@77a8d2d10c0b403a8b4aadbd223dc489ecd22683
         with:
           languages: javascript # does not matter
       - name: Get CodeQL version
         id: get-codeql-version
         run: |
-          echo "::set-output name=version::$("${CODEQL}" --version | head -n 1 | rev | cut -d " " -f 1 | rev)"
+          echo "version=$("${CODEQL}" --version | head -n 1 | rev | cut -d " " -f 1 | rev)" >> $GITHUB_OUTPUT
         shell: bash
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
@@ -133,7 +133,7 @@ jobs:
         env:
           CONF: ./ql-for-ql-config.yml
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
+        uses: github/codeql-action/init@77a8d2d10c0b403a8b4aadbd223dc489ecd22683
         with:
           languages: ql
           db-location: ${{ runner.temp }}/db
@@ -145,7 +145,7 @@ jobs:
           PACK: ${{ runner.temp }}/pack
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
+        uses: github/codeql-action/analyze@77a8d2d10c0b403a8b4aadbd223dc489ecd22683
         with:
           category: "ql-for-ql"
       - name: Copy sarif file to CWD

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -25,7 +25,7 @@ jobs:
 
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
+        uses: github/codeql-action/init@77a8d2d10c0b403a8b4aadbd223dc489ecd22683
         with:
           languages: javascript # does not matter
       - uses: actions/cache@v3

.github/workflows/ql-for-ql-tests.yml

@@ -22,7 +22,7 @@ jobs:
       - uses: actions/checkout@v3
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@71a8b35ff4c80fcfcd05bc1cd932fe3c08f943ca
+        uses: github/codeql-action/init@77a8d2d10c0b403a8b4aadbd223dc489ecd22683
         with:
           languages: javascript # does not matter
       - uses: actions/cache@v3

.github/workflows/ruby-qltest.yml

@@ -28,13 +28,6 @@ defaults:
     working-directory: ruby
 
 jobs:
-  qlformat:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
-      - name: Check QL formatting
-        run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only
   qlcompile:
     runs-on: ubuntu-latest
     steps:

.github/workflows/swift.yml

@@ -51,12 +51,14 @@ jobs:
       - uses: actions/checkout@v3
       - uses: ./swift/actions/create-extractor-pack
       - uses: ./swift/actions/run-quick-tests
+      - uses: ./swift/actions/print-unextracted
   build-and-test-linux:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
       - uses: ./swift/actions/create-extractor-pack
       - uses: ./swift/actions/run-quick-tests
+      - uses: ./swift/actions/print-unextracted
   qltests-linux:
     needs: build-and-test-linux
     runs-on: ubuntu-latest
@@ -110,12 +112,9 @@ jobs:
         with:
           name: swift-generated-cpp-files
           path: swift/generated-cpp-files/**
-  qlformat:
+  database-upgrade-scripts:
     runs-on: ubuntu-latest
-    needs: changes
-    if: ${{ needs.changes.outputs.ql == 'true' }}
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/fetch-codeql
-      - name: Check QL formatting
-        run: find swift/ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql query format --check-only
+      - uses: ./swift/actions/database-upgrade-scripts

config/identical-files.json

@@ -94,8 +94,8 @@
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/rangeanalysis/SsaReadPositionCommon.qll"
   ],
   "Model as Data Generation Java/C# - CaptureModels": [
-    "java/ql/src/utils/model-generator/internal/CaptureModels.qll",
-    "csharp/ql/src/utils/model-generator/internal/CaptureModels.qll"
+    "java/ql/src/utils/modelgenerator/internal/CaptureModels.qll",
+    "csharp/ql/src/utils/modelgenerator/internal/CaptureModels.qll"
   ],
   "Sign Java/C#": [
     "java/ql/lib/semmle/code/java/dataflow/internal/rangeanalysis/Sign.qll",

cpp/ql/lib/experimental/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll

@@ -136,6 +136,18 @@ module Consistency {
     msg = "Local flow step does not preserve enclosing callable."
   }
 
+  query predicate readStepIsLocal(Node n1, Node n2, string msg) {
+    readStep(n1, _, n2) and
+    nodeGetEnclosingCallable(n1) != nodeGetEnclosingCallable(n2) and
+    msg = "Read step does not preserve enclosing callable."
+  }
+
+  query predicate storeStepIsLocal(Node n1, Node n2, string msg) {
+    storeStep(n1, _, n2) and
+    nodeGetEnclosingCallable(n1) != nodeGetEnclosingCallable(n2) and
+    msg = "Store step does not preserve enclosing callable."
+  }
+
   private DataFlowType typeRepr() { result = getNodeType(_) }
 
   query predicate compatibleTypesReflexive(DataFlowType t, string msg) {

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplConsistency.qll

@@ -136,6 +136,18 @@ module Consistency {
     msg = "Local flow step does not preserve enclosing callable."
   }
 
+  query predicate readStepIsLocal(Node n1, Node n2, string msg) {
+    readStep(n1, _, n2) and
+    nodeGetEnclosingCallable(n1) != nodeGetEnclosingCallable(n2) and
+    msg = "Read step does not preserve enclosing callable."
+  }
+
+  query predicate storeStepIsLocal(Node n1, Node n2, string msg) {
+    storeStep(n1, _, n2) and
+    nodeGetEnclosingCallable(n1) != nodeGetEnclosingCallable(n2) and
+    msg = "Store step does not preserve enclosing callable."
+  }
+
   private DataFlowType typeRepr() { result = getNodeType(_) }
 
   query predicate compatibleTypesReflexive(DataFlowType t, string msg) {